From 8b795c6321a7ff25b05bb1762f7a5097c1f1b22c Mon Sep 17 00:00:00 2001 From: quobix Date: Sun, 15 Oct 2023 12:34:54 -0400 Subject: [PATCH] working through rolodex design and using it externally via vacuum this is some complex and messy work. Signed-off-by: quobix --- document.go | 34 +++++++++++++++-------- document_test.go | 2 ++ index/resolver.go | 15 +++++++++- index/resolver_test.go | 8 +++--- index/rolodex.go | 53 ++++++++++++++++++++++++++++++------ index/rolodex_file_loader.go | 9 ++++-- 6 files changed, 93 insertions(+), 28 deletions(-) diff --git a/document.go b/document.go index 2211d62..8ee30f3 100644 --- a/document.go +++ b/document.go @@ -36,6 +36,9 @@ type Document interface { // GetVersion will return the exact version of the OpenAPI specification set for the document. GetVersion() string + // GetRolodex will return the Rolodex instance that was used to load the document. + GetRolodex() *index.Rolodex + // GetSpecInfo will return the *datamodel.SpecInfo instance that contains all specification information. GetSpecInfo() *datamodel.SpecInfo @@ -102,6 +105,7 @@ type Document interface { } type document struct { + rolodex *index.Rolodex version string info *datamodel.SpecInfo config *datamodel.DocumentConfiguration @@ -161,6 +165,10 @@ func NewDocumentWithConfiguration(specByteArray []byte, configuration *datamodel return d, err } +func (d *document) GetRolodex() *index.Rolodex { + return d.rolodex +} + func (d *document) GetVersion() string { return d.version } @@ -281,15 +289,15 @@ func (d *document) BuildV3Model() (*DocumentModel[v3high.Document], []error) { if d.highOpenAPI3Model != nil { return d.highOpenAPI3Model, nil } - var errors []error + var errs []error if d.info == nil { - errors = append(errors, fmt.Errorf("unable to build document, no specification has been loaded")) - return nil, errors + errs = append(errs, fmt.Errorf("unable to build document, no specification has been loaded")) + return nil, errs } if d.info.SpecFormat != datamodel.OAS3 { - errors = append(errors, fmt.Errorf("unable to build openapi document, "+ + errs = append(errs, fmt.Errorf("unable to build openapi document, "+ "supplied spec is a different version (%v). Try 'BuildV2Model()'", d.info.SpecFormat)) - return nil, errors + return nil, errs } var lowDoc *v3low.Document @@ -300,24 +308,26 @@ func (d *document) BuildV3Model() (*DocumentModel[v3high.Document], []error) { } } - lowDoc, errors = v3low.CreateDocumentFromConfig(d.info, d.config) + var docErr error + lowDoc, docErr = v3low.CreateDocumentFromConfig(d.info, d.config) + d.rolodex = lowDoc.Rolodex // Do not short-circuit on circular reference errors, so the client // has the option of ignoring them. - for _, err := range errors { - if refErr, ok := err.(*index.ResolvingError); ok { + for _, err := range utils.UnwrapErrors(docErr) { + var refErr *index.ResolvingError + if errors.As(err, &refErr) { if refErr.CircularReference == nil { - return nil, errors + return nil, errs } - } else { - return nil, errors } } highDoc := v3high.NewDocument(lowDoc) + d.highOpenAPI3Model = &DocumentModel[v3high.Document]{ Model: *highDoc, Index: lowDoc.Index, } - return d.highOpenAPI3Model, errors + return d.highOpenAPI3Model, errs } // CompareDocuments will accept a left and right Document implementing struct, build a model for the correct diff --git a/document_test.go b/document_test.go index 2c9be68..e9999ef 100644 --- a/document_test.go +++ b/document_test.go @@ -822,6 +822,7 @@ components: assert.Len(t, errs, 0) assert.Len(t, m.Index.GetCircularReferences(), 0) + assert.Len(t, m.Index.GetResolver().GetIgnoredCircularPolyReferences(), 1) } @@ -856,5 +857,6 @@ components: assert.Len(t, errs, 0) assert.Len(t, m.Index.GetCircularReferences(), 0) + assert.Len(t, m.Index.GetResolver().GetIgnoredCircularArrayReferences(), 1) } diff --git a/index/resolver.go b/index/resolver.go index 5070b02..3b14b8a 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -53,10 +53,23 @@ func NewResolver(index *SpecIndex) *Resolver { if index == nil { return nil } - return &Resolver{ + r := &Resolver{ + specIndex: index, resolvedRoot: index.GetRootNode(), } + index.resolver = r + return r +} + +// GetIgnoredCircularPolyReferences returns all ignored circular references that are polymorphic +func (resolver *Resolver) GetIgnoredCircularPolyReferences() []*CircularReferenceResult { + return resolver.ignoredPolyReferences +} + +// GetIgnoredCircularArrayReferences returns all ignored circular references that are arrays +func (resolver *Resolver) GetIgnoredCircularArrayReferences() []*CircularReferenceResult { + return resolver.ignoredArrayReferences } // GetResolvingErrors returns all errors found during resolving diff --git a/index/resolver_test.go b/index/resolver_test.go index 6fda4ea..8beaa24 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -7,7 +7,7 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" ) @@ -318,9 +318,9 @@ func TestResolver_CheckForCircularReferences_DigitalOcean(t *testing.T) { baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") idx := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - AllowRemoteLookup: true, - AllowFileLookup: true, - BaseURL: baseURL, + //AllowRemoteLookup: true, + //AllowFileLookup: true, + BaseURL: baseURL, }) resolver := NewResolver(idx) diff --git a/index/rolodex.go b/index/rolodex.go index f208ab1..28100b0 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -54,6 +54,9 @@ type Rolodex struct { localFS map[string]fs.FS remoteFS map[string]fs.FS indexed bool + built bool + resolved bool + circChecked bool indexConfig *SpecIndexConfig indexingDuration time.Duration indexes []*SpecIndex @@ -324,11 +327,12 @@ func (r *Rolodex) IndexTheRolodex() error { } // now that we have indexed all the files, we can build the index. - for _, idx := range indexBuildQueue { - idx.BuildIndex() - - } r.indexes = indexBuildQueue + if !r.indexConfig.AvoidBuildIndex { + for _, idx := range indexBuildQueue { + idx.BuildIndex() + } + } // indexed and built every supporting file, we can build the root index (our entry point) index := NewSpecIndexWithConfig(r.indexConfig.SpecInfo.RootNode, r.indexConfig) @@ -339,10 +343,16 @@ func (r *Rolodex) IndexTheRolodex() error { if r.indexConfig.IgnorePolymorphicCircularReferences { resolver.IgnorePolymorphicCircularReferences() } - index.resolver = resolver - resolvingErrors := resolver.CheckForCircularReferences() - for e := range resolvingErrors { - caughtErrors = append(caughtErrors, resolvingErrors[e]) + + if !r.indexConfig.AvoidBuildIndex { + index.BuildIndex() + } + + if !r.indexConfig.AvoidCircularReferenceCheck { + resolvingErrors := resolver.CheckForCircularReferences() + for e := range resolvingErrors { + caughtErrors = append(caughtErrors, resolvingErrors[e]) + } } r.rootIndex = index @@ -353,6 +363,29 @@ func (r *Rolodex) IndexTheRolodex() error { } +func (r *Rolodex) CheckForCircularReferences() { + if r.rootIndex != nil && r.rootIndex.resolver != nil { + resolvingErrors := r.rootIndex.resolver.CheckForCircularReferences() + for e := range resolvingErrors { + r.caughtErrors = append(r.caughtErrors, resolvingErrors[e]) + } + } +} + +func (r *Rolodex) BuildIndexes() { + if r.built { + return + } + for _, idx := range r.indexes { + idx.BuildIndex() + } + if r.rootIndex != nil { + r.rootIndex.BuildIndex() + } + r.built = true + return +} + func (r *Rolodex) Open(location string) (RolodexFile, error) { var errorStack []error @@ -360,6 +393,10 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { var localFile *LocalFile //var remoteFile *RemoteFile + if r == nil || r.localFS == nil && r.remoteFS == nil { + panic("WHAT NO....") + } + for k, v := range r.localFS { // check if this is a URL or an abs/rel reference. diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 9729c43..bc6f756 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -124,10 +124,13 @@ func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { if absBaseErr != nil { return nil, absBaseErr } - walkErr := fs.WalkDir(dirFS, baseDir, func(p string, d fs.DirEntry, err error) error { + walkErr := fs.WalkDir(dirFS, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } - // we don't care about directories. - if d.IsDir() { + // we don't care about directories, or errors, just read everything we can. + if d == nil || d.IsDir() { return nil }