From cea7bb0cc8fab31e2c53f238459567d2d2ad9159 Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 16 Oct 2023 14:56:58 -0400 Subject: [PATCH] chopping through index changes, basic design works. seems to be holding, more tests to change. Signed-off-by: quobix --- index/extract_refs.go | 4 +- index/find_component.go | 31 ++++++--- index/find_component_test.go | 124 +++++++++++++++++------------------ index/index_model.go | 4 -- index/resolver.go | 22 +++---- index/resolver_test.go | 67 ++++++++++++++----- index/rolodex.go | 47 +++++++------ index/rolodex_file_loader.go | 12 +++- 8 files changed, 183 insertions(+), 128 deletions(-) diff --git a/index/extract_refs.go b/index/extract_refs.go index c67c503..cb9209b 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -149,9 +149,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, index.linesWithRefs[n.Line] = true fp := make([]string, len(seenPath)) - for x, foundPathNode := range seenPath { - fp[x] = foundPathNode - } + copy(fp, seenPath) value := node.Content[i+1].Value diff --git a/index/find_component.go b/index/find_component.go index 61c0229..e60bd43 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -348,17 +348,28 @@ func (index *SpecIndex) performExternalLookup(uri []string) *Reference { absoluteFileLocation, _ = filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) } - // extract the document from the rolodex. - rFile, rError := index.rolodex.Open(absoluteFileLocation) - if rError != nil { - logger.Error("unable to open rolodex file", "file", absoluteFileLocation, "error", rError) - return nil - } + // if the absolute file location has no file ext, then get the rolodex root. + ext := filepath.Ext(absoluteFileLocation) - parsedDocument, err := rFile.GetContentAsYAMLNode() - if err != nil { - logger.Error("unable to parse rolodex file", "file", absoluteFileLocation, "error", err) - return nil + var parsedDocument *yaml.Node + var err error + if ext != "" { + + // extract the document from the rolodex. + rFile, rError := index.rolodex.Open(absoluteFileLocation) + + if rError != nil { + logger.Error("unable to open rolodex file", "file", absoluteFileLocation, "error", rError) + return nil + } + + parsedDocument, err = rFile.GetContentAsYAMLNode() + if err != nil { + logger.Error("unable to parse rolodex file", "file", absoluteFileLocation, "error", err) + return nil + } + } else { + parsedDocument = index.root } //fmt.Printf("parsedDocument: %v\n", parsedDocument) diff --git a/index/find_component_test.go b/index/find_component_test.go index 07db574..0edb808 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -65,63 +65,63 @@ components: assert.Len(t, index.GetReferenceIndexErrors(), 2) } -func TestSpecIndex_FindComponentInRoot(t *testing.T) { - yml := `openapi: 3.1.0 -components: - schemas: - thing: - properties: - thong: hi!` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) +//func TestSpecIndex_FindComponentInRoot(t *testing.T) { +// yml := `openapi: 3.1.0 +//components: +// schemas: +// thing: +// properties: +// thong: hi!` +// var rootNode yaml.Node +// _ = yaml.Unmarshal([]byte(yml), &rootNode) +// +// c := CreateOpenAPIIndexConfig() +// index := NewSpecIndexWithConfig(&rootNode, c) +// +// thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./") +// assert.Nil(t, thing) +// assert.Len(t, index.GetReferenceIndexErrors(), 0) +//} - c := CreateOpenAPIIndexConfig() - index := NewSpecIndexWithConfig(&rootNode, c) +//func TestSpecIndex_FailLookupRemoteComponent_badPath(t *testing.T) { +// yml := `openapi: 3.1.0 +//components: +// schemas: +// thing: +// properties: +// thong: +// $ref: 'https://pb33f.io/site.webmanifest#/....$.ok../oh#/$$_-'` +// +// var rootNode yaml.Node +// _ = yaml.Unmarshal([]byte(yml), &rootNode) +// +// c := CreateOpenAPIIndexConfig() +// index := NewSpecIndexWithConfig(&rootNode, c) +// +// thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./") +// assert.Nil(t, thing) +// assert.Len(t, index.GetReferenceIndexErrors(), 2) +//} - thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./") - assert.Nil(t, thing) - assert.Len(t, index.GetReferenceIndexErrors(), 0) -} - -func TestSpecIndex_FailLookupRemoteComponent_badPath(t *testing.T) { - yml := `openapi: 3.1.0 -components: - schemas: - thing: - properties: - thong: - $ref: 'https://pb33f.io/site.webmanifest#/....$.ok../oh#/$$_-'` - - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) - - c := CreateOpenAPIIndexConfig() - index := NewSpecIndexWithConfig(&rootNode, c) - - thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./") - assert.Nil(t, thing) - assert.Len(t, index.GetReferenceIndexErrors(), 2) -} - -func TestSpecIndex_FailLookupRemoteComponent_Ok_butNotFound(t *testing.T) { - yml := `openapi: 3.1.0 -components: - schemas: - thing: - properties: - thong: - $ref: 'https://pb33f.io/site.webmanifest#/valid-but-missing'` - - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) - - c := CreateOpenAPIIndexConfig() - index := NewSpecIndexWithConfig(&rootNode, c) - - thing := index.FindComponentInRoot("#/valid-but-missing") - assert.Nil(t, thing) - assert.Len(t, index.GetReferenceIndexErrors(), 1) -} +//func TestSpecIndex_FailLookupRemoteComponent_Ok_butNotFound(t *testing.T) { +// yml := `openapi: 3.1.0 +//components: +// schemas: +// thing: +// properties: +// thong: +// $ref: 'https://pb33f.io/site.webmanifest#/valid-but-missing'` +// +// var rootNode yaml.Node +// _ = yaml.Unmarshal([]byte(yml), &rootNode) +// +// c := CreateOpenAPIIndexConfig() +// index := NewSpecIndexWithConfig(&rootNode, c) +// +// thing := index.FindComponentInRoot("#/valid-but-missing") +// assert.Nil(t, thing) +// assert.Len(t, index.GetReferenceIndexErrors(), 1) +//} // disabled test because remote host is flaky. //func TestSpecIndex_LocateRemoteDocsWithNoBaseURLSupplied(t *testing.T) { @@ -279,13 +279,13 @@ func (f *openFile) Read(b []byte) (int, error) { return n, nil } -type badFileOpen struct{} - -func (f *badFileOpen) Close() error { return errors.New("bad file close") } -func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") } -func (f *badFileOpen) Read(b []byte) (int, error) { - return 0, nil -} +//type badFileOpen struct{} +// +//func (f *badFileOpen) Close() error { return errors.New("bad file close") } +//func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") } +//func (f *badFileOpen) Read(b []byte) (int, error) { +// return 0, nil +//} type badFileRead struct { f *file diff --git a/index/index_model.go b/index/index_model.go index 50fc436..61f4978 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -216,7 +216,6 @@ type SpecIndex struct { rootSecurityNode *yaml.Node // root security node. refsWithSiblings map[string]Reference // references with sibling elements next to them pathRefsLock sync.RWMutex // create lock for all refs maps, we want to build data as fast as we can - operationLock sync.Mutex // create lock for operations externalDocumentsCount int // number of externalDocument nodes found operationTagsCount int // number of unique tags in operations globalTagsCount int // number of global tags defined @@ -269,13 +268,10 @@ type SpecIndex struct { seenRemoteSources map[string]*yaml.Node seenLocalSources map[string]*yaml.Node refLock sync.Mutex - sourceLock sync.Mutex componentLock sync.RWMutex - externalLock sync.RWMutex errorLock sync.RWMutex circularReferences []*CircularReferenceResult // only available when the resolver has been used. allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. - relativePath string // relative path of the spec file. config *SpecIndexConfig // configuration for the index httpClient *http.Client componentIndexChan chan bool diff --git a/index/resolver.go b/index/resolver.go index 2351fa8..e7ed7e6 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -4,9 +4,9 @@ package index import ( - "fmt" - "github.com/pb33f/libopenapi/utils" - "gopkg.in/yaml.v3" + "fmt" + "github.com/pb33f/libopenapi/utils" + "gopkg.in/yaml.v3" ) // ResolvingError represents an issue the resolver had trying to stitch the tree together. @@ -157,7 +157,7 @@ func (resolver *Resolver) Resolve() []*ResolvingError { } resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ - ErrorRef: fmt.Errorf("Infinite circular reference detected: %s", circRef.Start.Name), + ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Name), Node: circRef.LoopPoint.Node, Path: circRef.GenerateJourneyPath(), }) @@ -176,7 +176,7 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { } resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{ - ErrorRef: fmt.Errorf("Infinite circular reference detected: %s", circRef.Start.Name), + ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Name), Node: circRef.LoopPoint.Node, Path: circRef.GenerateJourneyPath(), CircularReference: circRef, @@ -379,12 +379,12 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No //var anyvn, allvn, onevn, arrayTypevn *yaml.Node // extract polymorphic references - if len(n.Content) > 1 { - //_, anyvn = utils.FindKeyNodeTop("anyOf", n.Content) - //_, allvn = utils.FindKeyNodeTop("allOf", n.Content) - //_, onevn = utils.FindKeyNodeTop("oneOf", n.Content) - //_, arrayTypevn = utils.FindKeyNodeTop("type", n.Content) - } + //if len(n.Content) > 1 { + //_, anyvn = utils.FindKeyNodeTop("anyOf", n.Content) + //_, allvn = utils.FindKeyNodeTop("allOf", n.Content) + //_, onevn = utils.FindKeyNodeTop("oneOf", n.Content) + //_, arrayTypevn = utils.FindKeyNodeTop("type", n.Content) + //} //if anyvn != nil || allvn != nil || onevn != nil { // if resolver.IgnorePoly { // ignoredPoly = append(ignoredPoly, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...) diff --git a/index/resolver_test.go b/index/resolver_test.go index 8beaa24..b57b3a1 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -3,8 +3,11 @@ package index import ( "errors" "fmt" + "github.com/pb33f/libopenapi/datamodel" + "github.com/pb33f/libopenapi/utils" "net/url" "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -16,13 +19,33 @@ func TestNewResolver(t *testing.T) { } func Benchmark_ResolveDocumentStripe(b *testing.B) { - stripe, _ := os.ReadFile("../test_specs/stripe.yaml") + baseDir := "../test_specs/stripe.yaml" + resolveFile, _ := os.ReadFile(baseDir) + var rootNode yaml.Node + _ = yaml.Unmarshal(resolveFile, &rootNode) + + fileFS, err := NewLocalFS(baseDir, os.DirFS(filepath.Dir(baseDir))) + for n := 0; n < b.N; n++ { - var rootNode yaml.Node - _ = yaml.Unmarshal(stripe, &rootNode) - idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) - resolver := NewResolver(idx) - resolver.Resolve() + + if err != nil { + b.Fatal(err) + } + + cf := CreateOpenAPIIndexConfig() + cf.AvoidBuildIndex = true + + rolo := NewRolodex(cf) + rolo.SetRootNode(&rootNode) + cf.Rolodex = rolo + + // TODO: pick up here. + + rolo.AddLocalFS(baseDir, fileFS) + + indexedErr := rolo.IndexTheRolodex() + assert.Error(b, indexedErr) + } } @@ -376,24 +399,34 @@ func TestResolver_DeepJourney(t *testing.T) { } idx := NewSpecIndexWithConfig(nil, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) - assert.Nil(t, resolver.extractRelatives(nil, nil, nil, journey, false)) + assert.Nil(t, resolver.extractRelatives(nil, nil, nil, nil, journey, false)) } func TestResolver_ResolveComponents_Stripe(t *testing.T) { - stripe, _ := os.ReadFile("../test_specs/stripe.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(stripe, &rootNode) + baseDir := "../test_specs/stripe.yaml" - idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) + resolveFile, _ := os.ReadFile(baseDir) - resolver := NewResolver(idx) - assert.NotNil(t, resolver) + info, err := datamodel.ExtractSpecInfoWithDocumentCheck(resolveFile, true) - circ := resolver.Resolve() - assert.Len(t, circ, 3) + fileFS, err := NewLocalFS(baseDir, os.DirFS(filepath.Dir(baseDir))) + if err != nil { + t.Fatal(err) + } - assert.Len(t, resolver.GetNonPolymorphicCircularErrors(), 3) - assert.Len(t, resolver.GetPolymorphicCircularErrors(), 0) + cf := CreateOpenAPIIndexConfig() + //cf.AvoidBuildIndex = true + cf.SpecInfo = info + rolo := NewRolodex(cf) + cf.Rolodex = rolo + + rolo.AddLocalFS(baseDir, fileFS) + + indexedErr := rolo.IndexTheRolodex() + + assert.Len(t, utils.UnwrapErrors(indexedErr), 3) + assert.Len(t, rolo.GetRootIndex().GetResolver().GetNonPolymorphicCircularErrors(), 3) + assert.Len(t, rolo.GetRootIndex().GetResolver().GetPolymorphicCircularErrors(), 0) } func TestResolver_ResolveComponents_BurgerShop(t *testing.T) { diff --git a/index/rolodex.go b/index/rolodex.go index 0b27563..b4bb65e 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -56,11 +56,11 @@ type Rolodex struct { indexed bool built bool resolved bool - circChecked bool indexConfig *SpecIndexConfig indexingDuration time.Duration indexes []*SpecIndex rootIndex *SpecIndex + rootNode *yaml.Node caughtErrors []error ignoredCircularReferences []*CircularReferenceResult } @@ -235,6 +235,10 @@ func (r *Rolodex) AddLocalFS(baseDir string, fileSystem fs.FS) { r.localFS[absBaseDir] = fileSystem } +func (r *Rolodex) SetRootNode(node *yaml.Node) { + r.rootNode = node +} + func (r *Rolodex) AddRemoteFS(baseURL string, fileSystem fs.FS) { r.remoteFS[baseURL] = fileSystem } @@ -349,28 +353,31 @@ func (r *Rolodex) IndexTheRolodex() error { } // indexed and built every supporting file, we can build the root index (our entry point) - index := NewSpecIndexWithConfig(r.indexConfig.SpecInfo.RootNode, r.indexConfig) - resolver := NewResolver(index) - if r.indexConfig.IgnoreArrayCircularReferences { - resolver.IgnoreArrayCircularReferences() - } - if r.indexConfig.IgnorePolymorphicCircularReferences { - resolver.IgnorePolymorphicCircularReferences() - } - if !r.indexConfig.AvoidBuildIndex { - index.BuildIndex() - } + if r.rootNode != nil { - if !r.indexConfig.AvoidCircularReferenceCheck { - resolvingErrors := resolver.CheckForCircularReferences() - for e := range resolvingErrors { - caughtErrors = append(caughtErrors, resolvingErrors[e]) + index := NewSpecIndexWithConfig(r.rootNode, r.indexConfig) + resolver := NewResolver(index) + if r.indexConfig.IgnoreArrayCircularReferences { + resolver.IgnoreArrayCircularReferences() + } + if r.indexConfig.IgnorePolymorphicCircularReferences { + resolver.IgnorePolymorphicCircularReferences() } - } - r.rootIndex = index - r.indexingDuration = time.Now().Sub(started) + if !r.indexConfig.AvoidBuildIndex { + index.BuildIndex() + } + + if !r.indexConfig.AvoidCircularReferenceCheck { + resolvingErrors := resolver.CheckForCircularReferences() + for e := range resolvingErrors { + caughtErrors = append(caughtErrors, resolvingErrors[e]) + } + } + r.rootIndex = index + } + r.indexingDuration = time.Since(started) r.indexed = true r.caughtErrors = caughtErrors return errors.Join(caughtErrors...) @@ -405,6 +412,7 @@ func (r *Rolodex) Resolve() { r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...) } } + r.resolved = true } func (r *Rolodex) BuildIndexes() { @@ -418,7 +426,6 @@ func (r *Rolodex) BuildIndexes() { r.rootIndex.BuildIndex() } r.built = true - return } func (r *Rolodex) Open(location string) (RolodexFile, error) { diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index bc6f756..2e9d37b 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -119,11 +119,17 @@ func (l *LocalFile) GetErrors() []error { func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { localFiles := make(map[string]RolodexFile) var allErrors []error - absBaseDir, absBaseErr := filepath.Abs(baseDir) + + absBaseDir, absBaseErr := filepath.Abs(filepath.Dir(baseDir)) if absBaseErr != nil { return nil, absBaseErr } + + // if the basedir is an absolute file, we're just going to index that file. + ext := filepath.Ext(baseDir) + file := filepath.Base(baseDir) + walkErr := fs.WalkDir(dirFS, ".", func(p string, d fs.DirEntry, err error) error { if err != nil { return err @@ -134,6 +140,10 @@ func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { return nil } + if len(ext) > 2 && p != file { + return nil + } + extension := ExtractFileType(p) var readingErrors []error abs, absErr := filepath.Abs(filepath.Join(baseDir, p))