From e6626fe22edd43890fc37debdba9699f54505844 Mon Sep 17 00:00:00 2001 From: Dave Shanley Date: Sat, 18 Feb 2023 13:57:20 -0500 Subject: [PATCH] Support for relative links now in place #73 Tests all passing, runs super fast, pulls in every single DigitalOcean spec and parses it. There may be some issues deeper down in the models, but for now high level tests all pass. --- datamodel/low/extraction_functions.go | 5 +- datamodel/low/v3/path_item.go | 4 +- document_test.go | 2 +- go.mod | 1 + go.sum | 3 + index/extract_refs.go | 36 +++---- index/find_component.go | 107 ++++++++++++--------- index/index_model.go | 130 +++++++++++++++----------- index/index_model_test.go | 25 +++++ index/search_index.go | 25 ++++- index/search_index_test.go | 49 ++++++++++ index/spec_index.go | 26 ++++-- index/spec_index_test.go | 111 +++++++++++++++------- index/utility_methods.go | 15 ++- resolver/resolver.go | 100 +++++++++++++------- resolver/resolver_test.go | 50 ++++++++-- 16 files changed, 471 insertions(+), 218 deletions(-) create mode 100644 index/index_model_test.go create mode 100644 index/search_index_test.go diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index 2ad2484..4997df4 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -39,7 +39,6 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re idx.GetAllHeaders, idx.GetAllCallbacks, idx.GetAllLinks, - idx.GetAllExternalDocuments, idx.GetAllExamples, idx.GetAllRequestBodies, idx.GetAllResponses, @@ -51,6 +50,7 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re // the reference being supplied. If there is a match found, the reference *yaml.Node is returned. func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) { if rf, _, rv := utils.IsNodeRefValue(root); rf { + // run through everything and return as soon as we find a match. // this operates as fast as possible as ever collections := generateIndexCollection(idx) @@ -89,11 +89,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) { } } + // perform a search for the reference in the index foundRefs := idx.SearchIndexForReference(rv) if len(foundRefs) > 0 { return foundRefs[0].Node, nil } + // let's try something else to find our references. + // cant be found? last resort is to try a path lookup _, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv) if friendly != "" { diff --git a/datamodel/low/v3/path_item.go b/datamodel/low/v3/path_item.go index 84c17ea..4c796e8 100644 --- a/datamodel/low/v3/path_item.go +++ b/datamodel/low/v3/path_item.go @@ -194,7 +194,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error { var op Operation - wg.Add(1) + if ok, _, _ := utils.IsNodeRefValue(pathNode); ok { r, err := low.LocateRefNode(pathNode, idx) @@ -210,7 +210,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error { pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column) } } - + wg.Add(1) go low.BuildModelAsync(pathNode, &op, &wg, &errors) opRef := low.NodeReference[*Operation]{ diff --git a/document_test.go b/document_test.go index 955a22b..8af52ec 100644 --- a/document_test.go +++ b/document_test.go @@ -113,7 +113,7 @@ paths: assert.NoError(t, err) v3Doc, docErr := doc.BuildV3Model() - assert.Len(t, docErr, 1) + assert.Len(t, docErr, 2) assert.Nil(t, v3Doc) } diff --git a/go.mod b/go.mod index 6addddc..ae7eecf 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/stretchr/testify v1.8.0 github.com/vmware-labs/yaml-jsonpath v0.3.2 + golang.org/x/sync v0.1.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index cf27655..8ce168b 100644 --- a/go.sum +++ b/go.sum @@ -75,7 +75,10 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/index/extract_refs.go b/index/extract_refs.go index 07ea366..bb79e6c 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -325,9 +325,8 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc var found []*Reference //run this async because when things get recursive, it can take a while - //c := make(chan bool) - //tm := make(chan bool) - + c := make(chan bool) + locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) { located := index.FindComponent(ref.Definition, ref.Node) if located != nil { @@ -351,7 +350,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc } index.refErrors = append(index.refErrors, indexError) } - //c <- true + c <- true } var refsToCheck []*Reference @@ -372,32 +371,19 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc refsToCheck = append(refsToCheck, ref) } mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck)) - //go func() { - // time.Sleep(20 * time.Second) - // tm <- true - //}() for r := range refsToCheck { // expand our index of all mapped refs - // go locate(refsToCheck[r], r, mappedRefsInSequence) - locate(refsToCheck[r], r, mappedRefsInSequence) + go locate(refsToCheck[r], r, mappedRefsInSequence) } - //completedRefs := 0 - //for completedRefs < len(refsToCheck) { - // select { - // case <-c: - // completedRefs++ - // if completedRefs >= len(refsToCheck) { - // fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck)) - // break - // } else { - // //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck)) - // } - // //case <-tm: - // // panic("OH NO") - // } - //} + completedRefs := 0 + for completedRefs < len(refsToCheck) { + select { + case <-c: + completedRefs++ + } + } for m := range mappedRefsInSequence { if mappedRefsInSequence[m] != nil { index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m]) diff --git a/index/find_component.go b/index/find_component.go index 5da0e47..e904abe 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -9,9 +9,10 @@ import ( "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" "gopkg.in/yaml.v3" "io/ioutil" + "net/http" "net/url" "strings" - "sync" + "time" ) // FindComponent will locate a component by its reference, returns nil if nothing is found. @@ -26,7 +27,7 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re if index.config.AllowRemoteLookup { return index.lookupRemoteReference(id) } else { - return nil, nil, fmt.Errorf("remote lookups are not premitted, " + + return nil, nil, fmt.Errorf("remote lookups are not permitted, " + "please set AllowRemoteLookup to true in the configuration") } } @@ -82,11 +83,22 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re return nil } -var n sync.Mutex +var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second} -var openCalls = 0 -var closedCalls = 0 -var totalCalls = 0 +func getRemoteDoc(u string, d chan []byte, e chan error) { + resp, err := httpClient.Get(u) + if err != nil { + e <- err + close(e) + close(d) + return + } + var body []byte + body, _ = ioutil.ReadAll(resp.Body) + d <- body + close(e) + close(d) +} func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { // split string to remove file reference @@ -99,30 +111,43 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod if alreadySeen { parsedRemoteDocument = foundDocument } else { - resp, err := index.httpClient.Get(uri[0]) - totalCalls++ - fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls) - if err != nil { - return nil, nil, err - } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } + d := make(chan bool) + var body []byte + var err error - var remoteDoc yaml.Node - err = yaml.Unmarshal(body, &remoteDoc) + go func(uri string) { + bc := make(chan []byte) + ec := make(chan error) + go getRemoteDoc(uri, bc, ec) + select { + case v := <-bc: + body = v + break + case er := <-ec: + err = er + break + } + var remoteDoc yaml.Node + er := yaml.Unmarshal(body, &remoteDoc) + if er != nil { + err = er + d <- true + return + } + parsedRemoteDocument = &remoteDoc + if index.config != nil { + index.config.seenRemoteSources.Store(uri, &remoteDoc) + } + d <- true + }(uri[0]) + + // wait for double go fun. + <-d if err != nil { + // no bueno. return nil, nil, err } - parsedRemoteDocument = &remoteDoc - //n.Lock() - //index.seenRemoteSources[uri[0]] = &remoteDoc - if index.config != nil { - index.config.seenRemoteSources[uri[0]] = &remoteDoc - } - //n.Unlock() } // lookup item from reference by using a path query. @@ -163,9 +188,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, // try and read the file off the local file system, if it fails // check for a baseURL and then ask our remote lookup function to go try and get it. - // index.fileLock.Lock() body, err := ioutil.ReadFile(file) - // index.fileLock.Unlock() if err != nil { @@ -173,26 +196,12 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, if index.config != nil && index.config.BaseURL != nil { u := index.config.BaseURL - remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) a, b, e := index.lookupRemoteReference(remoteRef) if e != nil { // give up, we can't find the file, not locally, not remotely. It's toast. return nil, nil, e } - - //// everything looks good, lets just make sure we also add a key to the raw reference name. - //if _, ok := index.seenRemoteSources[file]; !ok { - // if b != nil { - // //index.seenRemoteSources[ref] = b - // } else { - // panic("oh now") - // } - // - //} else { - // panic("oh no") - //} - return a, b, nil } else { @@ -207,7 +216,9 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, return nil, nil, err } parsedRemoteDocument = &remoteDoc - index.seenLocalSources[file] = &remoteDoc + if index.seenLocalSources != nil { + index.seenLocalSources[file] = &remoteDoc + } } // lookup item from reference by using a path query. @@ -279,10 +290,14 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string, // cool, cool, lets index this spec also. This is a recursive action and will keep going // until all remote references have been found. - // TODO: start here tomorrow, need to pass in the config to the new spec index. - // set the base URL to the path. + var j *url.URL + if index.config.BaseURL != nil { + j = index.config.BaseURL + } else { + j, _ = url.Parse(uri[0]) + } - path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false) + path := GenerateCleanSpecConfigBaseURL(j, uri[0], false) newUrl, e := url.Parse(path) if e == nil { @@ -296,10 +311,12 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string, var newIndex *SpecIndex newIndex = NewSpecIndexWithConfig(newRoot, newConfig) + index.refLock.Lock() index.externalSpecIndex[uri[0]] = newIndex newIndex.relativePath = path newIndex.parentIndex = index index.AddChild(newIndex) + index.refLock.Unlock() externalSpecIndex = newIndex } else { diff --git a/index/index_model.go b/index/index_model.go index b226942..ac5ac4f 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -4,6 +4,7 @@ package index import ( + "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" "net/http" "net/url" @@ -72,10 +73,30 @@ type SpecIndexConfig struct { RootIndex *SpecIndex // if this is a sub-index, the root knows about everything below. ParentIndex *SpecIndex // Who owns this index? - seenRemoteSources map[string]*yaml.Node + seenRemoteSources *syncmap.Map remoteLock *sync.Mutex } +// CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and +// AllowFileLookup set to true. This is the default behaviour of the index in previous versions of libopenapi. (pre 0.6.0) +func CreateOpenAPIIndexConfig() *SpecIndexConfig { + return &SpecIndexConfig{ + AllowRemoteLookup: true, + AllowFileLookup: true, + seenRemoteSources: &syncmap.Map{}, + } +} + +// CreateClosedAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and +// AllowFileLookup set to false. This is the default behaviour of the index in versions 0.6.0+ +func CreateClosedAPIIndexConfig() *SpecIndexConfig { + return &SpecIndexConfig{ + AllowRemoteLookup: false, + AllowFileLookup: false, + seenRemoteSources: &syncmap.Map{}, + } +} + // SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules, // everything is pre-walked if you need it. @@ -134,63 +155,59 @@ type SpecIndex struct { componentParamCount int // number of params defined in components componentsInlineParamUniqueCount int // number of inline params with unique names componentsInlineParamDuplicateCount int // number of inline params with duplicate names - schemaCount int // number of schemas - refCount int // total ref count - root *yaml.Node // the root document - pathsNode *yaml.Node // paths node - tagsNode *yaml.Node // tags node - componentsNode *yaml.Node // components node - parametersNode *yaml.Node // components/parameters node - allParametersNode map[string]*Reference // all parameters node - allParameters map[string]*Reference // all parameters (components/defs) - schemasNode *yaml.Node // components/schemas node - allInlineSchemaDefinitions []*Reference // all schemas found in document outside of components (openapi) or definitions (swagger). - allInlineSchemaObjectDefinitions []*Reference // all schemas that are objects found in document outside of components (openapi) or definitions (swagger). - allComponentSchemaDefinitions map[string]*Reference // all schemas found in components (openapi) or definitions (swagger). - securitySchemesNode *yaml.Node // components/securitySchemes node - allSecuritySchemes map[string]*Reference // all security schemes / definitions. - requestBodiesNode *yaml.Node // components/requestBodies node - allRequestBodies map[string]*Reference // all request bodies - responsesNode *yaml.Node // components/responses node - allResponses map[string]*Reference // all responses - headersNode *yaml.Node // components/headers node - allHeaders map[string]*Reference // all headers - examplesNode *yaml.Node // components/examples node - allExamples map[string]*Reference // all components examples - linksNode *yaml.Node // components/links node - allLinks map[string]*Reference // all links - callbacksNode *yaml.Node // components/callbacks node - allCallbacks map[string]*Reference // all components examples - externalDocumentsNode *yaml.Node // external documents node - allExternalDocuments map[string]*Reference // all external documents - externalSpecIndex map[string]*SpecIndex // create a primary index of all external specs and componentIds - refErrors []error // errors when indexing references - operationParamErrors []error // errors when indexing parameters - allDescriptions []*DescriptionReference // every single description found in the spec. - allSummaries []*DescriptionReference // every single summary found in the spec. - allEnums []*EnumReference // every single enum found in the spec. - allObjectsWithProperties []*ObjectReference // every single object with properties found in the spec. - enumCount int - descriptionCount int - summaryCount int - seenRemoteSources map[string]*yaml.Node - seenLocalSources map[string]*yaml.Node - remoteLock sync.RWMutex - httpLock sync.RWMutex - fileLock sync.Mutex - refLock sync.Mutex - circularReferences []*CircularReferenceResult // only available when the resolver has been used. - allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. - relativePath string // relative path of the spec file. - config *SpecIndexConfig // configuration for the index - httpClient *http.Client - componentIndexChan chan bool - polyComponentIndexChan chan bool + schemaCount int // number of schemas + refCount int // total ref count + root *yaml.Node // the root document + pathsNode *yaml.Node // paths node + tagsNode *yaml.Node // tags node + componentsNode *yaml.Node // components node + parametersNode *yaml.Node // components/parameters node + allParametersNode map[string]*Reference // all parameters node + allParameters map[string]*Reference // all parameters (components/defs) + schemasNode *yaml.Node // components/schemas node + allInlineSchemaDefinitions []*Reference // all schemas found in document outside of components (openapi) or definitions (swagger). + allInlineSchemaObjectDefinitions []*Reference // all schemas that are objects found in document outside of components (openapi) or definitions (swagger). + allComponentSchemaDefinitions map[string]*Reference // all schemas found in components (openapi) or definitions (swagger). + securitySchemesNode *yaml.Node // components/securitySchemes node + allSecuritySchemes map[string]*Reference // all security schemes / definitions. + requestBodiesNode *yaml.Node // components/requestBodies node + allRequestBodies map[string]*Reference // all request bodies + responsesNode *yaml.Node // components/responses node + allResponses map[string]*Reference // all responses + headersNode *yaml.Node // components/headers node + allHeaders map[string]*Reference // all headers + examplesNode *yaml.Node // components/examples node + allExamples map[string]*Reference // all components examples + linksNode *yaml.Node // components/links node + allLinks map[string]*Reference // all links + callbacksNode *yaml.Node // components/callbacks node + allCallbacks map[string]*Reference // all components examples + externalDocumentsNode *yaml.Node // external documents node + allExternalDocuments map[string]*Reference // all external documents + externalSpecIndex map[string]*SpecIndex // create a primary index of all external specs and componentIds + refErrors []error // errors when indexing references + operationParamErrors []error // errors when indexing parameters + allDescriptions []*DescriptionReference // every single description found in the spec. + allSummaries []*DescriptionReference // every single summary found in the spec. + allEnums []*EnumReference // every single enum found in the spec. + allObjectsWithProperties []*ObjectReference // every single object with properties found in the spec. + enumCount int + descriptionCount int + summaryCount int + seenRemoteSources map[string]*yaml.Node + seenLocalSources map[string]*yaml.Node + refLock sync.Mutex + circularReferences []*CircularReferenceResult // only available when the resolver has been used. + allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. + relativePath string // relative path of the spec file. + config *SpecIndexConfig // configuration for the index + httpClient *http.Client + componentIndexChan chan bool + polyComponentIndexChan chan bool // when things get complex (looking at you digital ocean) then we need to know // what we have seen across indexes, so we need to be able to travel back up to the root // cto avoid re-downloading sources. - rootIndex *SpecIndex parentIndex *SpecIndex children []*SpecIndex } @@ -199,6 +216,11 @@ func (index *SpecIndex) AddChild(child *SpecIndex) { index.children = append(index.children, child) } +// GetChildren returns the children of this index. +func (index *SpecIndex) GetChildren() []*SpecIndex { + return index.children +} + // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the // URI based document. Decides if the reference is local, remote or in a file. type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error) diff --git a/index/index_model_test.go b/index/index_model_test.go new file mode 100644 index 0000000..1f6daab --- /dev/null +++ b/index/index_model_test.go @@ -0,0 +1,25 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSpecIndex_Children(t *testing.T) { + idx1 := new(SpecIndex) + idx2 := new(SpecIndex) + idx3 := new(SpecIndex) + idx4 := new(SpecIndex) + idx5 := new(SpecIndex) + idx1.AddChild(idx2) + idx1.AddChild(idx3) + idx3.AddChild(idx4) + idx4.AddChild(idx5) + assert.Equal(t, 2, len(idx1.GetChildren())) + assert.Equal(t, 1, len(idx3.GetChildren())) + assert.Equal(t, 1, len(idx4.GetChildren())) + assert.Equal(t, 0, len(idx5.GetChildren())) +} diff --git a/index/search_index.go b/index/search_index.go index 4580cc8..8f5b5ce 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -3,16 +3,37 @@ package index -func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { +import "gopkg.in/yaml.v3" +// SearchIndexForReference searches the index for a reference, first looking through the mapped references +// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes +// extracted when parsing the OpenAPI Spec. +func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { if r, ok := index.allMappedRefs[ref]; ok { + if r.Node.Kind == yaml.DocumentNode { + // the reference is an entire document, so we need to dig down a level and rewire the reference. + r.Node = r.Node.Content[0] + } return []*Reference{r} } + if r, ok := index.externalSpecIndex[ref]; ok { + return []*Reference{ + { + Node: r.root.Content[0], + Name: ref, + Definition: ref, + }, + } + } for c := range index.children { - found := index.children[c].SearchIndexForReference(ref) + found := goFindMeSomething(index.children[c], ref) if found != nil { return found } } return nil } + +func goFindMeSomething(i *SpecIndex, ref string) []*Reference { + return i.SearchIndexForReference(ref) +} diff --git a/index/search_index_test.go b/index/search_index_test.go new file mode 100644 index 0000000..77a95ba --- /dev/null +++ b/index/search_index_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "io/ioutil" + "net/url" + "testing" +) + +func TestSpecIndex_SearchIndexForReference(t *testing.T) { + petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json") + var rootNode yaml.Node + _ = yaml.Unmarshal(petstore, &rootNode) + + c := CreateOpenAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, c) + + ref := idx.SearchIndexForReference("#/components/schemas/Pet") + assert.NotNil(t, ref) +} + +func TestSpecIndex_SearchIndexForReferene_ExternalSpecs(t *testing.T) { + + // load up an index with lots of references + petstore, _ := ioutil.ReadFile("../test_specs/digitalocean.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(petstore, &rootNode) + + c := CreateOpenAPIIndexConfig() + c.BaseURL, _ = url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") + idx := NewSpecIndexWithConfig(&rootNode, c) + + ref := idx.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml") + assert.NotNil(t, ref) + assert.Equal(t, "operationId", ref[0].Node.Content[0].Value) + + ref = idx.SearchIndexForReference("examples/ruby/domains_create.yml") + assert.NotNil(t, ref) + assert.Equal(t, "lang", ref[0].Node.Content[0].Value) + + ref = idx.SearchIndexForReference("../../shared/responses/server_error.yml") + assert.NotNil(t, ref) + assert.Equal(t, "description", ref[0].Node.Content[0].Value) + +} diff --git a/index/spec_index.go b/index/spec_index.go index 950a2e7..9657842 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -16,6 +16,7 @@ import ( "fmt" "github.com/pb33f/libopenapi/utils" "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" + "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" "strings" "sync" @@ -27,7 +28,7 @@ import ( func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { index := new(SpecIndex) if config != nil && config.seenRemoteSources == nil { - config.seenRemoteSources = make(map[string]*yaml.Node) + config.seenRemoteSources = &syncmap.Map{} } config.remoteLock = &sync.Mutex{} index.config = config @@ -46,11 +47,7 @@ func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecI // - https://github.com/pb33f/libopenapi/issues/73 func NewSpecIndex(rootNode *yaml.Node) *SpecIndex { index := new(SpecIndex) - index.config = &SpecIndexConfig{ - AllowRemoteLookup: true, - AllowFileLookup: true, - seenRemoteSources: make(map[string]*yaml.Node), - } + index.config = CreateOpenAPIIndexConfig() boostrapIndexCollections(rootNode, index) return createNewIndex(rootNode, index) } @@ -108,7 +105,12 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex { index.GetInlineDuplicateParamCount() index.GetAllDescriptionsCount() index.GetTotalTagsCount() - index.seenRemoteSources = index.config.seenRemoteSources + + // do a copy! + index.config.seenRemoteSources.Range(func(k, v any) bool { + index.seenRemoteSources[k.(string)] = v.(*yaml.Node) + return true + }) return index } @@ -1138,9 +1140,15 @@ func (index *SpecIndex) GetAllSummariesCount() int { return len(index.allSummaries) } +// CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it, +// to avoid making duplicate remote calls for document data. func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) { - if _, ok := index.config.seenRemoteSources[url]; ok { - return true, index.config.seenRemoteSources[url] + if index.config == nil || index.config.seenRemoteSources == nil { + return false, nil + } + j, _ := index.config.seenRemoteSources.Load(url) + if j != nil { + return true, j.(*yaml.Node) } return false, nil } diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 82cd5bc..0bb9cae 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -128,7 +128,6 @@ func TestSpecIndex_BaseURLError(t *testing.T) { }) assert.Len(t, index.GetAllExternalIndexes(), 0) - assert.Len(t, index.GetReferenceIndexErrors(), 582) } func TestSpecIndex_k8s(t *testing.T) { @@ -199,9 +198,9 @@ func TestSpecIndex_XSOAR(t *testing.T) { } func TestSpecIndex_PetstoreV3(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/petstorev3.json") + petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json") var rootNode yaml.Node - yaml.Unmarshal(asana, &rootNode) + yaml.Unmarshal(petstore, &rootNode) index := NewSpecIndex(&rootNode) @@ -368,21 +367,24 @@ func TestSpecIndex_NoRoot(t *testing.T) { } func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { - spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") - var rootNode yaml.Node - yaml.Unmarshal(spec, &rootNode) + spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") + var rootNode yaml.Node + yaml.Unmarshal(spec, &rootNode) - index := NewSpecIndex(&rootNode) + index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ + AllowRemoteLookup: true, + AllowFileLookup: true, + }) - assert.Len(t, index.allRefs, 5) - assert.Len(t, index.allMappedRefs, 5) - assert.Equal(t, 5, index.GetPathCount()) - assert.Equal(t, 5, index.GetOperationCount()) - assert.Equal(t, 1, index.GetComponentSchemaCount()) - assert.Equal(t, 2, index.GetGlobalTagsCount()) - assert.Equal(t, 3, index.GetTotalTagsCount()) - assert.Equal(t, 2, index.GetOperationTagsCount()) - assert.Equal(t, 0, index.GetGlobalLinksCount()) + assert.Len(t, index.allRefs, 5) + assert.Len(t, index.allMappedRefs, 5) + assert.Equal(t, 5, index.GetPathCount()) + assert.Equal(t, 5, index.GetOperationCount()) + assert.Equal(t, 1, index.GetComponentSchemaCount()) + assert.Equal(t, 2, index.GetGlobalTagsCount()) + assert.Equal(t, 3, index.GetTotalTagsCount()) + assert.Equal(t, 2, index.GetOperationTagsCount()) + assert.Equal(t, 0, index.GetGlobalLinksCount()) assert.Equal(t, 0, index.GetComponentParameterCount()) assert.Equal(t, 2, index.GetOperationsParameterCount()) assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) @@ -589,43 +591,80 @@ func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing index := new(SpecIndex) index.seenRemoteSources = make(map[string]*yaml.Node) index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") - assert.NoError(t, err) - assert.Nil(t, a) + a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") + assert.Error(t, err) + assert.Nil(t, a) assert.Nil(t, b) } // Discovered in issue https://github.com/pb33f/libopenapi/issues/37 func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") - assert.NoError(t, err) - assert.Nil(t, a) - assert.Nil(t, b) + index := new(SpecIndex) + index.seenRemoteSources = make(map[string]*yaml.Node) + index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} + a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") + assert.NoError(t, err) + assert.NotNil(t, a) + assert.NotNil(t, b) } // Discovered in issue https://github.com/daveshanley/vacuum/issues/225 func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { index := new(SpecIndex) - _ = ioutil.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") + _ = ioutil.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) + defer os.Remove("coffee-time.yaml") - index.seenRemoteSources = make(map[string]*yaml.Node) - a, b, err := index.lookupFileReference("coffee-time.yaml") - assert.NoError(t, err) - assert.NotNil(t, a) - assert.NotNil(t, b) + index.seenRemoteSources = make(map[string]*yaml.Node) + a, b, err := index.lookupFileReference("coffee-time.yaml") + assert.NoError(t, err) + assert.NotNil(t, a) + assert.NotNil(t, b) +} + +func TestSpecIndex_CheckBadURLRef(t *testing.T) { + + yml := `openapi: 3.1.0 +paths: + /cakes: + post: + parameters: + - $ref: 'httpsss://badurl'` + + var rootNode yaml.Node + yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndex(&rootNode) + + assert.Len(t, index.refErrors, 2) +} + +func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { + + yml := `openapi: 3.1.0 +paths: + /cakes: + post: + parameters: + - $ref: 'httpsss://badurl'` + + var rootNode yaml.Node + yaml.Unmarshal([]byte(yml), &rootNode) + + c := CreateClosedAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, c) + + assert.Len(t, idx.refErrors, 2) + assert.Equal(t, "remote lookups are not permitted, "+ + "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error()) } func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) { - _ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") + _ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) + defer os.Remove("coffee-time.yaml") - yml := `openapi: 3.0.3 + yml := `openapi: 3.0.3 paths: /cakes: post: diff --git a/index/utility_methods.go b/index/utility_methods.go index 9d45ff3..b22ba7c 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -401,9 +401,20 @@ func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bo } cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/")) } else { - cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/")) + if !strings.HasPrefix(dir, "http") { + if len(pathSegs) > 1 || len(dirSegs) > 1 { + cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/")) + } + } else { + cleanedPath = strings.Join(dirSegs, "/") + } + } + var p string + if baseURL.Scheme != "" && !strings.HasPrefix(dir, "http") { + p = fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath) + } else { + p = cleanedPath } - p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath) if strings.HasSuffix(p, "/") { p = p[:len(p)-1] } diff --git a/resolver/resolver.go b/resolver/resolver.go index 4741ae4..0e426ec 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -38,6 +38,10 @@ type Resolver struct { resolvedRoot *yaml.Node resolvingErrors []*ResolvingError circularReferences []*index.CircularReferenceResult + referencesVisited int + indexesVisited int + journeysTaken int + relativesSeen int } // NewResolver will create a new resolver from a *index.SpecIndex @@ -73,7 +77,6 @@ func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularRefere res = append(res, resolver.circularReferences[i]) } } - return res } @@ -93,40 +96,33 @@ func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularRef return res } +// GetJourneysTaken returns the number of journeys taken by the resolver +func (resolver *Resolver) GetJourneysTaken() int { + return resolver.journeysTaken +} + +// GetReferenceVisited returns the number of references visited by the resolver +func (resolver *Resolver) GetReferenceVisited() int { + return resolver.referencesVisited +} + +// GetIndexesVisited returns the number of indexes visited by the resolver +func (resolver *Resolver) GetIndexesVisited() int { + return resolver.indexesVisited +} + +// +func (resolver *Resolver) GetRelativesSeen() int { + return resolver.relativesSeen +} + // Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved. // this data can get big, it results in a massive duplication of data. This is a destructive method and will permanently // re-organize the node tree. Make sure you have copied your original tree before running this (if you want to preserve // original data) func (resolver *Resolver) Resolve() []*ResolvingError { - mapped := resolver.specIndex.GetMappedReferencesSequenced() - mappedIndex := resolver.specIndex.GetMappedReferences() - for _, ref := range mapped { - seenReferences := make(map[string]bool) - var journey []*index.Reference - if ref != nil && ref.Reference != nil { - ref.Reference.Node.Content = resolver.VisitReference(ref.Reference, seenReferences, journey, true) - } - } - - schemas := resolver.specIndex.GetAllComponentSchemas() - for s, schemaRef := range schemas { - if mappedIndex[s] == nil { - seenReferences := make(map[string]bool) - var journey []*index.Reference - schemaRef.Node.Content = resolver.VisitReference(schemaRef, seenReferences, journey, true) - } - } - - // map everything - for _, sequenced := range resolver.specIndex.GetAllSequencedReferences() { - locatedDef := mappedIndex[sequenced.Definition] - if locatedDef != nil { - if !locatedDef.Circular && locatedDef.Seen { - sequenced.Node.Content = locatedDef.Node.Content - } - } - } + visitIndex(resolver, resolver.specIndex) for _, circRef := range resolver.circularReferences { // If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one @@ -181,12 +177,47 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { return resolver.resolvingErrors } -// VisitReference will visit a reference as part of a journey and will return resolved nodes. -func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node { - if ref == nil { - panic("what?") +func visitIndex(res *Resolver, idx *index.SpecIndex) { + mapped := idx.GetMappedReferencesSequenced() + mappedIndex := idx.GetMappedReferences() + res.indexesVisited++ + + for _, ref := range mapped { + seenReferences := make(map[string]bool) + var journey []*index.Reference + res.journeysTaken++ + if ref != nil && ref.Reference != nil { + ref.Reference.Node.Content = res.VisitReference(ref.Reference, seenReferences, journey, true) + } } + schemas := idx.GetAllComponentSchemas() + for s, schemaRef := range schemas { + if mappedIndex[s] == nil { + seenReferences := make(map[string]bool) + var journey []*index.Reference + res.journeysTaken++ + schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) + } + } + + // map everything + for _, sequenced := range idx.GetAllSequencedReferences() { + locatedDef := mappedIndex[sequenced.Definition] + if locatedDef != nil { + if !locatedDef.Circular && locatedDef.Seen { + sequenced.Node.Content = locatedDef.Node.Content + } + } + } + for _, c := range idx.GetChildren() { + visitIndex(res, c) + } +} + +// VisitReference will visit a reference as part of a journey and will return resolved nodes. +func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node { + resolver.referencesVisited++ if ref.Resolved || ref.Seen { return ref.Node.Content } @@ -308,7 +339,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node, ref := resolver.specIndex.SearchIndexForReference(value) if ref == nil { - // TODO handle error, missing ref, can't resolve. _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) err := &ResolvingError{ ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value), @@ -414,6 +444,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node, } } } - + resolver.relativesSeen += len(found) return found } diff --git a/resolver/resolver_test.go b/resolver/resolver_test.go index 1de0d6e..84b7b9f 100644 --- a/resolver/resolver_test.go +++ b/resolver/resolver_test.go @@ -170,19 +170,56 @@ components: assert.Len(t, circ, 0) } -// TODO: Test for remote and file references with the option switched on and off -func TestResolver_ResolveComponents_MixedRef(t *testing.T) { - mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") +func TestResolver_ResolveComponents_Missing(t *testing.T) { + yml := `paths: + /hey: + get: + responses: + "200": + $ref: '#/components/schemas/crackers' +components: + schemas: + cheese: + description: cheese + properties: + thang: + $ref: '#/components/schemas/crackers' + crackers: + description: crackers + properties: + butter: + $ref: 'go home, I am drunk'` + var rootNode yaml.Node - yaml.Unmarshal(mixedref, &rootNode) + yaml.Unmarshal([]byte(yml), &rootNode) index := index.NewSpecIndex(&rootNode) resolver := NewResolver(index) assert.NotNil(t, resolver) + err := resolver.Resolve() + assert.Len(t, err, 1) + assert.Equal(t, "cannot resolve reference `go home, I am drunk`, it's missing: $go home, I am drunk [18:11]", err[0].Error()) +} + +func TestResolver_ResolveComponents_MixedRef(t *testing.T) { + mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") + var rootNode yaml.Node + yaml.Unmarshal(mixedref, &rootNode) + + b := index.CreateOpenAPIIndexConfig() + idx := index.NewSpecIndexWithConfig(&rootNode, b) + + resolver := NewResolver(idx) + assert.NotNil(t, resolver) + circ := resolver.Resolve() - assert.Len(t, circ, 10) + assert.Len(t, circ, 0) + assert.Equal(t, 5, resolver.GetIndexesVisited()) + assert.Equal(t, 209, resolver.GetRelativesSeen()) + assert.Equal(t, 35, resolver.GetJourneysTaken()) + assert.Equal(t, 62, resolver.GetReferenceVisited()) } func TestResolver_ResolveComponents_k8s(t *testing.T) { @@ -211,7 +248,8 @@ func ExampleNewResolver() { _ = yaml.Unmarshal(stripeBytes, &rootNode) // create a new spec index (resolver depends on it) - index := index.NewSpecIndex(&rootNode) + indexConfig := index.CreateClosedAPIIndexConfig() + index := index.NewSpecIndexWithConfig(&rootNode, indexConfig) // create a new resolver using the index. resolver := NewResolver(index)