Cleanup, sweep-up and tuneup

Signed-off-by: quobix <dave@quobix.com>

datamodel/low/extraction_functions.go

@@ -129,10 +129,8 @@ func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.S
 								u.Path = filepath.Join(p, explodedRefValue[0])
 								rv = fmt.Sprintf("%s#%s", u.String(), explodedRefValue[1])
 							}
-
 						}
 					}
-
 				}
 			}
 		} else {
@@ -159,7 +157,6 @@ func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.S
 							// check for a config baseURL and use that if it exists.
 							if idx.GetConfig().BaseURL != nil {
 								u := *idx.GetConfig().BaseURL
-
 								abs, _ := filepath.Abs(filepath.Join(u.Path, rv))
 								u.Path = abs
 								rv = u.String()
@@ -375,7 +372,6 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root
 					vn = ref
 					idx = fIdx
 					ctx = nCtx
-					//referenceValue = rVal
 					if err != nil {
 						circError = err
 					}
@@ -412,8 +408,6 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root
 		}
 		for _, node := range vn.Content {
 			localReferenceValue := ""
-			//localIsReference := false
-
 			foundCtx := ctx
 			foundIndex := idx
 
@@ -421,7 +415,6 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root
 				refg, fIdx, err, nCtx := LocateRefEnd(ctx, node, idx, 0)
 				if refg != nil {
 					node = refg
-					//localIsReference = true
 					localReferenceValue = rv
 					foundIndex = fIdx
 					foundCtx = nCtx
@@ -853,6 +846,10 @@ func GenerateHashString(v any) string {
 	return fmt.Sprintf(HASH, sha256.Sum256([]byte(fmt.Sprint(v))))
 }
 
+// LocateRefEnd will perform a complete lookup for a $ref node. This function searches the entire index for
+// the reference being supplied. If there is a match found, the reference *yaml.Node is returned.
+// the function operates recursively and will keep iterating through references until it finds a non-reference
+// node.
 func LocateRefEnd(ctx context.Context, root *yaml.Node, idx *index.SpecIndex, depth int) (*yaml.Node, *index.SpecIndex, error, context.Context) {
 	depth++
 	if depth > 100 {
@@ -862,17 +859,9 @@ func LocateRefEnd(ctx context.Context, root *yaml.Node, idx *index.SpecIndex, de
 	if err != nil {
 		return ref, fIdx, err, nCtx
 	}
-	if ref != nil {
 	if rf, _, _ := utils.IsNodeRefValue(ref); rf {
 		return LocateRefEnd(nCtx, ref, fIdx, depth)
 	} else {
 		return ref, fIdx, err, nCtx
 	}
-	} else {
-		if root.Content[1].Value == "" {
-			return nil, nil, fmt.Errorf("reference at line %d, column %d is empty, it cannot be resolved",
-				root.Content[1].Line, root.Content[1].Column), ctx
-		}
-		return nil, nil, fmt.Errorf("reference cannot be found: %s", root.Content[1].Value), ctx
-	}
 }

datamodel/low/extraction_functions_test.go

@@ -11,6 +11,7 @@ import (
 	"gopkg.in/yaml.v3"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -1983,3 +1984,121 @@ func TestLocateRefNode_DoARealLookup(t *testing.T) {
 	assert.Nil(t, e)
 	assert.NotNil(t, c)
 }
+
+func TestLocateRefEndNoRef_NoName(t *testing.T) {
+
+	r := &yaml.Node{Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "$ref"}, {Kind: yaml.ScalarNode, Value: ""}}}
+	n, i, e, c := LocateRefEnd(nil, r, nil, 0)
+	assert.Nil(t, n)
+	assert.Nil(t, i)
+	assert.Error(t, e)
+	assert.Nil(t, c)
+}
+
+func TestLocateRefEndNoRef(t *testing.T) {
+
+	r := &yaml.Node{Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "$ref"}, {Kind: yaml.ScalarNode, Value: "cake"}}}
+	n, i, e, c := LocateRefEnd(context.Background(), r, index.NewSpecIndexWithConfig(r, index.CreateClosedAPIIndexConfig()), 0)
+	assert.Nil(t, n)
+	assert.NotNil(t, i)
+	assert.Error(t, e)
+	assert.NotNil(t, c)
+}
+
+func TestLocateRefEnd_TooDeep(t *testing.T) {
+	r := &yaml.Node{Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "$ref"}, {Kind: yaml.ScalarNode, Value: ""}}}
+	n, i, e, c := LocateRefEnd(nil, r, nil, 100)
+	assert.Nil(t, n)
+	assert.Nil(t, i)
+	assert.Error(t, e)
+	assert.Nil(t, c)
+}
+
+func TestLocateRefEnd_Loop(t *testing.T) {
+
+	yml, _ := os.ReadFile("../../test_specs/first.yaml")
+	var bsn yaml.Node
+	_ = yaml.Unmarshal(yml, &bsn)
+
+	cf := index.CreateOpenAPIIndexConfig()
+	cf.BasePath = "../../test_specs"
+
+	localFSConfig := &index.LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		FileFilters:   []string{"first.yaml", "second.yaml", "third.yaml", "fourth.yaml"},
+		DirFS:         os.DirFS(cf.BasePath),
+	}
+	localFs, _ := index.NewLocalFSWithConfig(localFSConfig)
+	rolo := index.NewRolodex(cf)
+	rolo.AddLocalFS(cf.BasePath, localFs)
+	rolo.SetRootNode(&bsn)
+	rolo.IndexTheRolodex()
+	idx := rolo.GetRootIndex()
+	loop := yaml.Node{
+		Kind: yaml.MappingNode,
+		Content: []*yaml.Node{
+			{
+				Kind:  yaml.ScalarNode,
+				Value: "$ref",
+			},
+			{
+				Kind:  yaml.ScalarNode,
+				Value: "third.yaml#/properties/property/properties/statistics",
+			},
+		},
+	}
+
+	wd, _ := os.Getwd()
+	cp, _ := filepath.Abs(filepath.Join(wd, "../../test_specs/first.yaml"))
+	ctx := context.WithValue(context.Background(), index.CurrentPathKey, cp)
+	n, i, e, c := LocateRefEnd(ctx, &loop, idx, 0)
+	assert.NotNil(t, n)
+	assert.NotNil(t, i)
+	assert.Nil(t, e)
+	assert.NotNil(t, c)
+}
+
+func TestLocateRefEnd_Empty(t *testing.T) {
+
+	yml, _ := os.ReadFile("../../test_specs/first.yaml")
+	var bsn yaml.Node
+	_ = yaml.Unmarshal(yml, &bsn)
+
+	cf := index.CreateOpenAPIIndexConfig()
+	cf.BasePath = "../../test_specs"
+
+	localFSConfig := &index.LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		FileFilters:   []string{"first.yaml", "second.yaml", "third.yaml", "fourth.yaml"},
+		DirFS:         os.DirFS(cf.BasePath),
+	}
+	localFs, _ := index.NewLocalFSWithConfig(localFSConfig)
+	rolo := index.NewRolodex(cf)
+	rolo.AddLocalFS(cf.BasePath, localFs)
+	rolo.SetRootNode(&bsn)
+	rolo.IndexTheRolodex()
+	idx := rolo.GetRootIndex()
+	loop := yaml.Node{
+		Kind: yaml.MappingNode,
+		Content: []*yaml.Node{
+			{
+				Kind:  yaml.ScalarNode,
+				Value: "$ref",
+			},
+			{
+				Kind:  yaml.ScalarNode,
+				Value: "",
+			},
+		},
+	}
+
+	wd, _ := os.Getwd()
+	cp, _ := filepath.Abs(filepath.Join(wd, "../../test_specs/first.yaml"))
+	ctx := context.WithValue(context.Background(), index.CurrentPathKey, cp)
+	n, i, e, c := LocateRefEnd(ctx, &loop, idx, 0)
+	assert.Nil(t, n)
+	assert.Nil(t, i)
+	assert.Error(t, e)
+	assert.Equal(t, "reference at line 0, column 0 is empty, it cannot be resolved", e.Error())
+	assert.NotNil(t, c)
+}

index/index_model.go

@@ -266,7 +266,6 @@ type SpecIndex struct {
 	circularReferences                  []*CircularReferenceResult // only available when the resolver has been used.
 	allowCircularReferences             bool                       // decide if you want to error out, or allow circular references, default is false.
 	config                              *SpecIndexConfig           // configuration for the index
-	httpClient                          *http.Client
 	componentIndexChan                  chan bool
 	polyComponentIndexChan              chan bool
 	resolver                            *Resolver

index/index_utils.go

@@ -5,9 +5,7 @@ package index
 
 import (
 	"gopkg.in/yaml.v3"
-	"net/http"
 	"strings"
-	"time"
 )
 
 func isHttpMethod(val string) bool {
@@ -68,7 +66,6 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) {
 	index.polymorphicRefs = make(map[string]*Reference)
 	index.refsWithSiblings = make(map[string]Reference)
 	index.opServersRefs = make(map[string]map[string][]*Reference)
-	index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second}
 	index.componentIndexChan = make(chan bool)
 	index.polyComponentIndexChan = make(chan bool)
 }

index/rolodex.go

@@ -17,14 +17,13 @@ import (
 	"time"
 )
 
-type HasIndex interface {
-	GetIndex() *SpecIndex
-}
-
+// CanBeIndexed is an interface that allows a file to be indexed.
 type CanBeIndexed interface {
 	Index(config *SpecIndexConfig) (*SpecIndex, error)
 }
 
+// RolodexFile is an interface that represents a file in the rolodex. It combines multiple `fs` interfaces
+// like `fs.FileInfo` and `fs.File` into one interface, so the same struct can be used for everything.
 type RolodexFile interface {
 	GetContent() string
 	GetFileExtension() FileExtension
@@ -40,6 +39,8 @@ type RolodexFile interface {
 	Mode() os.FileMode
 }
 
+// RolodexFS is an interface that represents a RolodexFS, is the same interface as `fs.FS`, except it
+// also exposes a GetFiles() signature, to extract all files in the FS.
 type RolodexFS interface {
 	Open(name string) (fs.File, error)
 	GetFiles() map[string]RolodexFile
@@ -226,7 +227,6 @@ func (r *Rolodex) IndexTheRolodex() error {
 
 	// now that we have indexed all the files, we can build the index.
 	r.indexes = indexBuildQueue
-	//if !r.indexConfig.AvoidBuildIndex {
 
 	sort.Slice(indexBuildQueue, func(i, j int) bool {
 		return indexBuildQueue[i].specAbsolutePath < indexBuildQueue[j].specAbsolutePath

index/rolodex_remote_loader.go

@@ -411,7 +411,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 	if len(remoteFile.data) > 0 {
 		i.logger.Debug("successfully loaded file", "file", absolutePath)
 	}
-	//i.seekRelatives(remoteFile)
+
 	// remove from processing
 	i.ProcessingFiles.Delete(remoteParsedURL.Path)
 	i.Files.Store(absolutePath, remoteFile)

index/search_index.go

@@ -35,7 +35,6 @@ func (index *SpecIndex) SearchIndexForReferenceWithContext(ctx context.Context,
 func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx context.Context, searchRef *Reference) (*Reference, *SpecIndex, context.Context) {
 
 	if v, ok := index.cache.Load(searchRef.FullDefinition); ok {
-		//return v.(*Reference), index, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation)
 		return v.(*Reference), v.(*Reference).Index, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation)
 	}