From f3094d0b140d1e52fb68eb715f9bb0897423d02a Mon Sep 17 00:00:00 2001 From: quobix Date: Sat, 4 Nov 2023 09:38:33 -0400 Subject: [PATCH] Cleanup, sweep-up and tuneup Signed-off-by: quobix --- datamodel/low/extraction_functions.go | 25 ++--- datamodel/low/extraction_functions_test.go | 119 +++++++++++++++++++++ index/index_model.go | 1 - index/index_utils.go | 3 - index/rolodex.go | 10 +- index/rolodex_remote_loader.go | 2 +- index/search_index.go | 1 - 7 files changed, 132 insertions(+), 29 deletions(-) diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index 986b1ca..6cc1e20 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -129,10 +129,8 @@ func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.S u.Path = filepath.Join(p, explodedRefValue[0]) rv = fmt.Sprintf("%s#%s", u.String(), explodedRefValue[1]) } - } } - } } } else { @@ -159,7 +157,6 @@ func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.S // check for a config baseURL and use that if it exists. if idx.GetConfig().BaseURL != nil { u := *idx.GetConfig().BaseURL - abs, _ := filepath.Abs(filepath.Join(u.Path, rv)) u.Path = abs rv = u.String() @@ -375,7 +372,6 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root vn = ref idx = fIdx ctx = nCtx - //referenceValue = rVal if err != nil { circError = err } @@ -412,8 +408,6 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root } for _, node := range vn.Content { localReferenceValue := "" - //localIsReference := false - foundCtx := ctx foundIndex := idx @@ -421,7 +415,6 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root refg, fIdx, err, nCtx := LocateRefEnd(ctx, node, idx, 0) if refg != nil { node = refg - //localIsReference = true localReferenceValue = rv foundIndex = fIdx foundCtx = nCtx @@ -853,6 +846,10 @@ func GenerateHashString(v any) string { return fmt.Sprintf(HASH, sha256.Sum256([]byte(fmt.Sprint(v)))) } +// LocateRefEnd will perform a complete lookup for a $ref node. This function searches the entire index for +// the reference being supplied. If there is a match found, the reference *yaml.Node is returned. +// the function operates recursively and will keep iterating through references until it finds a non-reference +// node. func LocateRefEnd(ctx context.Context, root *yaml.Node, idx *index.SpecIndex, depth int) (*yaml.Node, *index.SpecIndex, error, context.Context) { depth++ if depth > 100 { @@ -862,17 +859,9 @@ func LocateRefEnd(ctx context.Context, root *yaml.Node, idx *index.SpecIndex, de if err != nil { return ref, fIdx, err, nCtx } - if ref != nil { - if rf, _, _ := utils.IsNodeRefValue(ref); rf { - return LocateRefEnd(nCtx, ref, fIdx, depth) - } else { - return ref, fIdx, err, nCtx - } + if rf, _, _ := utils.IsNodeRefValue(ref); rf { + return LocateRefEnd(nCtx, ref, fIdx, depth) } else { - if root.Content[1].Value == "" { - return nil, nil, fmt.Errorf("reference at line %d, column %d is empty, it cannot be resolved", - root.Content[1].Line, root.Content[1].Column), ctx - } - return nil, nil, fmt.Errorf("reference cannot be found: %s", root.Content[1].Value), ctx + return ref, fIdx, err, nCtx } } diff --git a/datamodel/low/extraction_functions_test.go b/datamodel/low/extraction_functions_test.go index 6200e4a..88254f4 100644 --- a/datamodel/low/extraction_functions_test.go +++ b/datamodel/low/extraction_functions_test.go @@ -11,6 +11,7 @@ import ( "gopkg.in/yaml.v3" "net/url" "os" + "path/filepath" "strings" "testing" @@ -1983,3 +1984,121 @@ func TestLocateRefNode_DoARealLookup(t *testing.T) { assert.Nil(t, e) assert.NotNil(t, c) } + +func TestLocateRefEndNoRef_NoName(t *testing.T) { + + r := &yaml.Node{Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "$ref"}, {Kind: yaml.ScalarNode, Value: ""}}} + n, i, e, c := LocateRefEnd(nil, r, nil, 0) + assert.Nil(t, n) + assert.Nil(t, i) + assert.Error(t, e) + assert.Nil(t, c) +} + +func TestLocateRefEndNoRef(t *testing.T) { + + r := &yaml.Node{Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "$ref"}, {Kind: yaml.ScalarNode, Value: "cake"}}} + n, i, e, c := LocateRefEnd(context.Background(), r, index.NewSpecIndexWithConfig(r, index.CreateClosedAPIIndexConfig()), 0) + assert.Nil(t, n) + assert.NotNil(t, i) + assert.Error(t, e) + assert.NotNil(t, c) +} + +func TestLocateRefEnd_TooDeep(t *testing.T) { + r := &yaml.Node{Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: "$ref"}, {Kind: yaml.ScalarNode, Value: ""}}} + n, i, e, c := LocateRefEnd(nil, r, nil, 100) + assert.Nil(t, n) + assert.Nil(t, i) + assert.Error(t, e) + assert.Nil(t, c) +} + +func TestLocateRefEnd_Loop(t *testing.T) { + + yml, _ := os.ReadFile("../../test_specs/first.yaml") + var bsn yaml.Node + _ = yaml.Unmarshal(yml, &bsn) + + cf := index.CreateOpenAPIIndexConfig() + cf.BasePath = "../../test_specs" + + localFSConfig := &index.LocalFSConfig{ + BaseDirectory: cf.BasePath, + FileFilters: []string{"first.yaml", "second.yaml", "third.yaml", "fourth.yaml"}, + DirFS: os.DirFS(cf.BasePath), + } + localFs, _ := index.NewLocalFSWithConfig(localFSConfig) + rolo := index.NewRolodex(cf) + rolo.AddLocalFS(cf.BasePath, localFs) + rolo.SetRootNode(&bsn) + rolo.IndexTheRolodex() + idx := rolo.GetRootIndex() + loop := yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "$ref", + }, + { + Kind: yaml.ScalarNode, + Value: "third.yaml#/properties/property/properties/statistics", + }, + }, + } + + wd, _ := os.Getwd() + cp, _ := filepath.Abs(filepath.Join(wd, "../../test_specs/first.yaml")) + ctx := context.WithValue(context.Background(), index.CurrentPathKey, cp) + n, i, e, c := LocateRefEnd(ctx, &loop, idx, 0) + assert.NotNil(t, n) + assert.NotNil(t, i) + assert.Nil(t, e) + assert.NotNil(t, c) +} + +func TestLocateRefEnd_Empty(t *testing.T) { + + yml, _ := os.ReadFile("../../test_specs/first.yaml") + var bsn yaml.Node + _ = yaml.Unmarshal(yml, &bsn) + + cf := index.CreateOpenAPIIndexConfig() + cf.BasePath = "../../test_specs" + + localFSConfig := &index.LocalFSConfig{ + BaseDirectory: cf.BasePath, + FileFilters: []string{"first.yaml", "second.yaml", "third.yaml", "fourth.yaml"}, + DirFS: os.DirFS(cf.BasePath), + } + localFs, _ := index.NewLocalFSWithConfig(localFSConfig) + rolo := index.NewRolodex(cf) + rolo.AddLocalFS(cf.BasePath, localFs) + rolo.SetRootNode(&bsn) + rolo.IndexTheRolodex() + idx := rolo.GetRootIndex() + loop := yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + { + Kind: yaml.ScalarNode, + Value: "$ref", + }, + { + Kind: yaml.ScalarNode, + Value: "", + }, + }, + } + + wd, _ := os.Getwd() + cp, _ := filepath.Abs(filepath.Join(wd, "../../test_specs/first.yaml")) + ctx := context.WithValue(context.Background(), index.CurrentPathKey, cp) + n, i, e, c := LocateRefEnd(ctx, &loop, idx, 0) + assert.Nil(t, n) + assert.Nil(t, i) + assert.Error(t, e) + assert.Equal(t, "reference at line 0, column 0 is empty, it cannot be resolved", e.Error()) + assert.NotNil(t, c) +} diff --git a/index/index_model.go b/index/index_model.go index 3928c16..c5882eb 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -266,7 +266,6 @@ type SpecIndex struct { circularReferences []*CircularReferenceResult // only available when the resolver has been used. allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. config *SpecIndexConfig // configuration for the index - httpClient *http.Client componentIndexChan chan bool polyComponentIndexChan chan bool resolver *Resolver diff --git a/index/index_utils.go b/index/index_utils.go index 55b76f0..8703c43 100644 --- a/index/index_utils.go +++ b/index/index_utils.go @@ -5,9 +5,7 @@ package index import ( "gopkg.in/yaml.v3" - "net/http" "strings" - "time" ) func isHttpMethod(val string) bool { @@ -68,7 +66,6 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) { index.polymorphicRefs = make(map[string]*Reference) index.refsWithSiblings = make(map[string]Reference) index.opServersRefs = make(map[string]map[string][]*Reference) - index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second} index.componentIndexChan = make(chan bool) index.polyComponentIndexChan = make(chan bool) } diff --git a/index/rolodex.go b/index/rolodex.go index 8fa3313..2d8bc6c 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -17,14 +17,13 @@ import ( "time" ) -type HasIndex interface { - GetIndex() *SpecIndex -} - +// CanBeIndexed is an interface that allows a file to be indexed. type CanBeIndexed interface { Index(config *SpecIndexConfig) (*SpecIndex, error) } +// RolodexFile is an interface that represents a file in the rolodex. It combines multiple `fs` interfaces +// like `fs.FileInfo` and `fs.File` into one interface, so the same struct can be used for everything. type RolodexFile interface { GetContent() string GetFileExtension() FileExtension @@ -40,6 +39,8 @@ type RolodexFile interface { Mode() os.FileMode } +// RolodexFS is an interface that represents a RolodexFS, is the same interface as `fs.FS`, except it +// also exposes a GetFiles() signature, to extract all files in the FS. type RolodexFS interface { Open(name string) (fs.File, error) GetFiles() map[string]RolodexFile @@ -226,7 +227,6 @@ func (r *Rolodex) IndexTheRolodex() error { // now that we have indexed all the files, we can build the index. r.indexes = indexBuildQueue - //if !r.indexConfig.AvoidBuildIndex { sort.Slice(indexBuildQueue, func(i, j int) bool { return indexBuildQueue[i].specAbsolutePath < indexBuildQueue[j].specAbsolutePath diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index 731990a..7628e05 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -411,7 +411,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { if len(remoteFile.data) > 0 { i.logger.Debug("successfully loaded file", "file", absolutePath) } - //i.seekRelatives(remoteFile) + // remove from processing i.ProcessingFiles.Delete(remoteParsedURL.Path) i.Files.Store(absolutePath, remoteFile) diff --git a/index/search_index.go b/index/search_index.go index f2b2747..50a1f40 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -35,7 +35,6 @@ func (index *SpecIndex) SearchIndexForReferenceWithContext(ctx context.Context, func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx context.Context, searchRef *Reference) (*Reference, *SpecIndex, context.Context) { if v, ok := index.cache.Load(searchRef.FullDefinition); ok { - //return v.(*Reference), index, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation) return v.(*Reference), v.(*Reference).Index, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation) }