diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index 2ad2484..4997df4 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -39,7 +39,6 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re idx.GetAllHeaders, idx.GetAllCallbacks, idx.GetAllLinks, - idx.GetAllExternalDocuments, idx.GetAllExamples, idx.GetAllRequestBodies, idx.GetAllResponses, @@ -51,6 +50,7 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re // the reference being supplied. If there is a match found, the reference *yaml.Node is returned. func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) { if rf, _, rv := utils.IsNodeRefValue(root); rf { + // run through everything and return as soon as we find a match. // this operates as fast as possible as ever collections := generateIndexCollection(idx) @@ -89,11 +89,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) { } } + // perform a search for the reference in the index foundRefs := idx.SearchIndexForReference(rv) if len(foundRefs) > 0 { return foundRefs[0].Node, nil } + // let's try something else to find our references. + // cant be found? last resort is to try a path lookup _, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv) if friendly != "" { diff --git a/datamodel/low/v3/path_item.go b/datamodel/low/v3/path_item.go index 84c17ea..4c796e8 100644 --- a/datamodel/low/v3/path_item.go +++ b/datamodel/low/v3/path_item.go @@ -194,7 +194,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error { var op Operation - wg.Add(1) + if ok, _, _ := utils.IsNodeRefValue(pathNode); ok { r, err := low.LocateRefNode(pathNode, idx) @@ -210,7 +210,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error { pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column) } } - + wg.Add(1) go low.BuildModelAsync(pathNode, &op, &wg, &errors) opRef := low.NodeReference[*Operation]{ diff --git a/document_test.go b/document_test.go index 955a22b..8af52ec 100644 --- a/document_test.go +++ b/document_test.go @@ -113,7 +113,7 @@ paths: assert.NoError(t, err) v3Doc, docErr := doc.BuildV3Model() - assert.Len(t, docErr, 1) + assert.Len(t, docErr, 2) assert.Nil(t, v3Doc) } diff --git a/go.mod b/go.mod index 6addddc..ae7eecf 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/stretchr/testify v1.8.0 github.com/vmware-labs/yaml-jsonpath v0.3.2 + golang.org/x/sync v0.1.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index cf27655..8ce168b 100644 --- a/go.sum +++ b/go.sum @@ -75,7 +75,10 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/index/extract_refs.go b/index/extract_refs.go index 07ea366..bb79e6c 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -325,9 +325,8 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc var found []*Reference //run this async because when things get recursive, it can take a while - //c := make(chan bool) - //tm := make(chan bool) - + c := make(chan bool) + locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) { located := index.FindComponent(ref.Definition, ref.Node) if located != nil { @@ -351,7 +350,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc } index.refErrors = append(index.refErrors, indexError) } - //c <- true + c <- true } var refsToCheck []*Reference @@ -372,32 +371,19 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc refsToCheck = append(refsToCheck, ref) } mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck)) - //go func() { - // time.Sleep(20 * time.Second) - // tm <- true - //}() for r := range refsToCheck { // expand our index of all mapped refs - // go locate(refsToCheck[r], r, mappedRefsInSequence) - locate(refsToCheck[r], r, mappedRefsInSequence) + go locate(refsToCheck[r], r, mappedRefsInSequence) } - //completedRefs := 0 - //for completedRefs < len(refsToCheck) { - // select { - // case <-c: - // completedRefs++ - // if completedRefs >= len(refsToCheck) { - // fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck)) - // break - // } else { - // //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck)) - // } - // //case <-tm: - // // panic("OH NO") - // } - //} + completedRefs := 0 + for completedRefs < len(refsToCheck) { + select { + case <-c: + completedRefs++ + } + } for m := range mappedRefsInSequence { if mappedRefsInSequence[m] != nil { index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m]) diff --git a/index/find_component.go b/index/find_component.go index 5da0e47..e904abe 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -9,9 +9,10 @@ import ( "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" "gopkg.in/yaml.v3" "io/ioutil" + "net/http" "net/url" "strings" - "sync" + "time" ) // FindComponent will locate a component by its reference, returns nil if nothing is found. @@ -26,7 +27,7 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re if index.config.AllowRemoteLookup { return index.lookupRemoteReference(id) } else { - return nil, nil, fmt.Errorf("remote lookups are not premitted, " + + return nil, nil, fmt.Errorf("remote lookups are not permitted, " + "please set AllowRemoteLookup to true in the configuration") } } @@ -82,11 +83,22 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re return nil } -var n sync.Mutex +var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second} -var openCalls = 0 -var closedCalls = 0 -var totalCalls = 0 +func getRemoteDoc(u string, d chan []byte, e chan error) { + resp, err := httpClient.Get(u) + if err != nil { + e <- err + close(e) + close(d) + return + } + var body []byte + body, _ = ioutil.ReadAll(resp.Body) + d <- body + close(e) + close(d) +} func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { // split string to remove file reference @@ -99,30 +111,43 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod if alreadySeen { parsedRemoteDocument = foundDocument } else { - resp, err := index.httpClient.Get(uri[0]) - totalCalls++ - fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls) - if err != nil { - return nil, nil, err - } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } + d := make(chan bool) + var body []byte + var err error - var remoteDoc yaml.Node - err = yaml.Unmarshal(body, &remoteDoc) + go func(uri string) { + bc := make(chan []byte) + ec := make(chan error) + go getRemoteDoc(uri, bc, ec) + select { + case v := <-bc: + body = v + break + case er := <-ec: + err = er + break + } + var remoteDoc yaml.Node + er := yaml.Unmarshal(body, &remoteDoc) + if er != nil { + err = er + d <- true + return + } + parsedRemoteDocument = &remoteDoc + if index.config != nil { + index.config.seenRemoteSources.Store(uri, &remoteDoc) + } + d <- true + }(uri[0]) + + // wait for double go fun. + <-d if err != nil { + // no bueno. return nil, nil, err } - parsedRemoteDocument = &remoteDoc - //n.Lock() - //index.seenRemoteSources[uri[0]] = &remoteDoc - if index.config != nil { - index.config.seenRemoteSources[uri[0]] = &remoteDoc - } - //n.Unlock() } // lookup item from reference by using a path query. @@ -163,9 +188,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, // try and read the file off the local file system, if it fails // check for a baseURL and then ask our remote lookup function to go try and get it. - // index.fileLock.Lock() body, err := ioutil.ReadFile(file) - // index.fileLock.Unlock() if err != nil { @@ -173,26 +196,12 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, if index.config != nil && index.config.BaseURL != nil { u := index.config.BaseURL - remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) a, b, e := index.lookupRemoteReference(remoteRef) if e != nil { // give up, we can't find the file, not locally, not remotely. It's toast. return nil, nil, e } - - //// everything looks good, lets just make sure we also add a key to the raw reference name. - //if _, ok := index.seenRemoteSources[file]; !ok { - // if b != nil { - // //index.seenRemoteSources[ref] = b - // } else { - // panic("oh now") - // } - // - //} else { - // panic("oh no") - //} - return a, b, nil } else { @@ -207,7 +216,9 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, return nil, nil, err } parsedRemoteDocument = &remoteDoc - index.seenLocalSources[file] = &remoteDoc + if index.seenLocalSources != nil { + index.seenLocalSources[file] = &remoteDoc + } } // lookup item from reference by using a path query. @@ -279,10 +290,14 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string, // cool, cool, lets index this spec also. This is a recursive action and will keep going // until all remote references have been found. - // TODO: start here tomorrow, need to pass in the config to the new spec index. - // set the base URL to the path. + var j *url.URL + if index.config.BaseURL != nil { + j = index.config.BaseURL + } else { + j, _ = url.Parse(uri[0]) + } - path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false) + path := GenerateCleanSpecConfigBaseURL(j, uri[0], false) newUrl, e := url.Parse(path) if e == nil { @@ -296,10 +311,12 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string, var newIndex *SpecIndex newIndex = NewSpecIndexWithConfig(newRoot, newConfig) + index.refLock.Lock() index.externalSpecIndex[uri[0]] = newIndex newIndex.relativePath = path newIndex.parentIndex = index index.AddChild(newIndex) + index.refLock.Unlock() externalSpecIndex = newIndex } else { diff --git a/index/index_model.go b/index/index_model.go index b226942..ac5ac4f 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -4,6 +4,7 @@ package index import ( + "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" "net/http" "net/url" @@ -72,10 +73,30 @@ type SpecIndexConfig struct { RootIndex *SpecIndex // if this is a sub-index, the root knows about everything below. ParentIndex *SpecIndex // Who owns this index? - seenRemoteSources map[string]*yaml.Node + seenRemoteSources *syncmap.Map remoteLock *sync.Mutex } +// CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and +// AllowFileLookup set to true. This is the default behaviour of the index in previous versions of libopenapi. (pre 0.6.0) +func CreateOpenAPIIndexConfig() *SpecIndexConfig { + return &SpecIndexConfig{ + AllowRemoteLookup: true, + AllowFileLookup: true, + seenRemoteSources: &syncmap.Map{}, + } +} + +// CreateClosedAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and +// AllowFileLookup set to false. This is the default behaviour of the index in versions 0.6.0+ +func CreateClosedAPIIndexConfig() *SpecIndexConfig { + return &SpecIndexConfig{ + AllowRemoteLookup: false, + AllowFileLookup: false, + seenRemoteSources: &syncmap.Map{}, + } +} + // SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules, // everything is pre-walked if you need it. @@ -134,63 +155,59 @@ type SpecIndex struct { componentParamCount int // number of params defined in components componentsInlineParamUniqueCount int // number of inline params with unique names componentsInlineParamDuplicateCount int // number of inline params with duplicate names - schemaCount int // number of schemas - refCount int // total ref count - root *yaml.Node // the root document - pathsNode *yaml.Node // paths node - tagsNode *yaml.Node // tags node - componentsNode *yaml.Node // components node - parametersNode *yaml.Node // components/parameters node - allParametersNode map[string]*Reference // all parameters node - allParameters map[string]*Reference // all parameters (components/defs) - schemasNode *yaml.Node // components/schemas node - allInlineSchemaDefinitions []*Reference // all schemas found in document outside of components (openapi) or definitions (swagger). - allInlineSchemaObjectDefinitions []*Reference // all schemas that are objects found in document outside of components (openapi) or definitions (swagger). - allComponentSchemaDefinitions map[string]*Reference // all schemas found in components (openapi) or definitions (swagger). - securitySchemesNode *yaml.Node // components/securitySchemes node - allSecuritySchemes map[string]*Reference // all security schemes / definitions. - requestBodiesNode *yaml.Node // components/requestBodies node - allRequestBodies map[string]*Reference // all request bodies - responsesNode *yaml.Node // components/responses node - allResponses map[string]*Reference // all responses - headersNode *yaml.Node // components/headers node - allHeaders map[string]*Reference // all headers - examplesNode *yaml.Node // components/examples node - allExamples map[string]*Reference // all components examples - linksNode *yaml.Node // components/links node - allLinks map[string]*Reference // all links - callbacksNode *yaml.Node // components/callbacks node - allCallbacks map[string]*Reference // all components examples - externalDocumentsNode *yaml.Node // external documents node - allExternalDocuments map[string]*Reference // all external documents - externalSpecIndex map[string]*SpecIndex // create a primary index of all external specs and componentIds - refErrors []error // errors when indexing references - operationParamErrors []error // errors when indexing parameters - allDescriptions []*DescriptionReference // every single description found in the spec. - allSummaries []*DescriptionReference // every single summary found in the spec. - allEnums []*EnumReference // every single enum found in the spec. - allObjectsWithProperties []*ObjectReference // every single object with properties found in the spec. - enumCount int - descriptionCount int - summaryCount int - seenRemoteSources map[string]*yaml.Node - seenLocalSources map[string]*yaml.Node - remoteLock sync.RWMutex - httpLock sync.RWMutex - fileLock sync.Mutex - refLock sync.Mutex - circularReferences []*CircularReferenceResult // only available when the resolver has been used. - allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. - relativePath string // relative path of the spec file. - config *SpecIndexConfig // configuration for the index - httpClient *http.Client - componentIndexChan chan bool - polyComponentIndexChan chan bool + schemaCount int // number of schemas + refCount int // total ref count + root *yaml.Node // the root document + pathsNode *yaml.Node // paths node + tagsNode *yaml.Node // tags node + componentsNode *yaml.Node // components node + parametersNode *yaml.Node // components/parameters node + allParametersNode map[string]*Reference // all parameters node + allParameters map[string]*Reference // all parameters (components/defs) + schemasNode *yaml.Node // components/schemas node + allInlineSchemaDefinitions []*Reference // all schemas found in document outside of components (openapi) or definitions (swagger). + allInlineSchemaObjectDefinitions []*Reference // all schemas that are objects found in document outside of components (openapi) or definitions (swagger). + allComponentSchemaDefinitions map[string]*Reference // all schemas found in components (openapi) or definitions (swagger). + securitySchemesNode *yaml.Node // components/securitySchemes node + allSecuritySchemes map[string]*Reference // all security schemes / definitions. + requestBodiesNode *yaml.Node // components/requestBodies node + allRequestBodies map[string]*Reference // all request bodies + responsesNode *yaml.Node // components/responses node + allResponses map[string]*Reference // all responses + headersNode *yaml.Node // components/headers node + allHeaders map[string]*Reference // all headers + examplesNode *yaml.Node // components/examples node + allExamples map[string]*Reference // all components examples + linksNode *yaml.Node // components/links node + allLinks map[string]*Reference // all links + callbacksNode *yaml.Node // components/callbacks node + allCallbacks map[string]*Reference // all components examples + externalDocumentsNode *yaml.Node // external documents node + allExternalDocuments map[string]*Reference // all external documents + externalSpecIndex map[string]*SpecIndex // create a primary index of all external specs and componentIds + refErrors []error // errors when indexing references + operationParamErrors []error // errors when indexing parameters + allDescriptions []*DescriptionReference // every single description found in the spec. + allSummaries []*DescriptionReference // every single summary found in the spec. + allEnums []*EnumReference // every single enum found in the spec. + allObjectsWithProperties []*ObjectReference // every single object with properties found in the spec. + enumCount int + descriptionCount int + summaryCount int + seenRemoteSources map[string]*yaml.Node + seenLocalSources map[string]*yaml.Node + refLock sync.Mutex + circularReferences []*CircularReferenceResult // only available when the resolver has been used. + allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. + relativePath string // relative path of the spec file. + config *SpecIndexConfig // configuration for the index + httpClient *http.Client + componentIndexChan chan bool + polyComponentIndexChan chan bool // when things get complex (looking at you digital ocean) then we need to know // what we have seen across indexes, so we need to be able to travel back up to the root // cto avoid re-downloading sources. - rootIndex *SpecIndex parentIndex *SpecIndex children []*SpecIndex } @@ -199,6 +216,11 @@ func (index *SpecIndex) AddChild(child *SpecIndex) { index.children = append(index.children, child) } +// GetChildren returns the children of this index. +func (index *SpecIndex) GetChildren() []*SpecIndex { + return index.children +} + // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the // URI based document. Decides if the reference is local, remote or in a file. type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error) diff --git a/index/index_model_test.go b/index/index_model_test.go new file mode 100644 index 0000000..1f6daab --- /dev/null +++ b/index/index_model_test.go @@ -0,0 +1,25 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestSpecIndex_Children(t *testing.T) { + idx1 := new(SpecIndex) + idx2 := new(SpecIndex) + idx3 := new(SpecIndex) + idx4 := new(SpecIndex) + idx5 := new(SpecIndex) + idx1.AddChild(idx2) + idx1.AddChild(idx3) + idx3.AddChild(idx4) + idx4.AddChild(idx5) + assert.Equal(t, 2, len(idx1.GetChildren())) + assert.Equal(t, 1, len(idx3.GetChildren())) + assert.Equal(t, 1, len(idx4.GetChildren())) + assert.Equal(t, 0, len(idx5.GetChildren())) +} diff --git a/index/search_index.go b/index/search_index.go index 4580cc8..8f5b5ce 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -3,16 +3,37 @@ package index -func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { +import "gopkg.in/yaml.v3" +// SearchIndexForReference searches the index for a reference, first looking through the mapped references +// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes +// extracted when parsing the OpenAPI Spec. +func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { if r, ok := index.allMappedRefs[ref]; ok { + if r.Node.Kind == yaml.DocumentNode { + // the reference is an entire document, so we need to dig down a level and rewire the reference. + r.Node = r.Node.Content[0] + } return []*Reference{r} } + if r, ok := index.externalSpecIndex[ref]; ok { + return []*Reference{ + { + Node: r.root.Content[0], + Name: ref, + Definition: ref, + }, + } + } for c := range index.children { - found := index.children[c].SearchIndexForReference(ref) + found := goFindMeSomething(index.children[c], ref) if found != nil { return found } } return nil } + +func goFindMeSomething(i *SpecIndex, ref string) []*Reference { + return i.SearchIndexForReference(ref) +} diff --git a/index/search_index_test.go b/index/search_index_test.go new file mode 100644 index 0000000..77a95ba --- /dev/null +++ b/index/search_index_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "io/ioutil" + "net/url" + "testing" +) + +func TestSpecIndex_SearchIndexForReference(t *testing.T) { + petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json") + var rootNode yaml.Node + _ = yaml.Unmarshal(petstore, &rootNode) + + c := CreateOpenAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, c) + + ref := idx.SearchIndexForReference("#/components/schemas/Pet") + assert.NotNil(t, ref) +} + +func TestSpecIndex_SearchIndexForReferene_ExternalSpecs(t *testing.T) { + + // load up an index with lots of references + petstore, _ := ioutil.ReadFile("../test_specs/digitalocean.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(petstore, &rootNode) + + c := CreateOpenAPIIndexConfig() + c.BaseURL, _ = url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") + idx := NewSpecIndexWithConfig(&rootNode, c) + + ref := idx.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml") + assert.NotNil(t, ref) + assert.Equal(t, "operationId", ref[0].Node.Content[0].Value) + + ref = idx.SearchIndexForReference("examples/ruby/domains_create.yml") + assert.NotNil(t, ref) + assert.Equal(t, "lang", ref[0].Node.Content[0].Value) + + ref = idx.SearchIndexForReference("../../shared/responses/server_error.yml") + assert.NotNil(t, ref) + assert.Equal(t, "description", ref[0].Node.Content[0].Value) + +} diff --git a/index/spec_index.go b/index/spec_index.go index 950a2e7..9657842 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -16,6 +16,7 @@ import ( "fmt" "github.com/pb33f/libopenapi/utils" "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" + "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" "strings" "sync" @@ -27,7 +28,7 @@ import ( func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { index := new(SpecIndex) if config != nil && config.seenRemoteSources == nil { - config.seenRemoteSources = make(map[string]*yaml.Node) + config.seenRemoteSources = &syncmap.Map{} } config.remoteLock = &sync.Mutex{} index.config = config @@ -46,11 +47,7 @@ func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecI // - https://github.com/pb33f/libopenapi/issues/73 func NewSpecIndex(rootNode *yaml.Node) *SpecIndex { index := new(SpecIndex) - index.config = &SpecIndexConfig{ - AllowRemoteLookup: true, - AllowFileLookup: true, - seenRemoteSources: make(map[string]*yaml.Node), - } + index.config = CreateOpenAPIIndexConfig() boostrapIndexCollections(rootNode, index) return createNewIndex(rootNode, index) } @@ -108,7 +105,12 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex { index.GetInlineDuplicateParamCount() index.GetAllDescriptionsCount() index.GetTotalTagsCount() - index.seenRemoteSources = index.config.seenRemoteSources + + // do a copy! + index.config.seenRemoteSources.Range(func(k, v any) bool { + index.seenRemoteSources[k.(string)] = v.(*yaml.Node) + return true + }) return index } @@ -1138,9 +1140,15 @@ func (index *SpecIndex) GetAllSummariesCount() int { return len(index.allSummaries) } +// CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it, +// to avoid making duplicate remote calls for document data. func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) { - if _, ok := index.config.seenRemoteSources[url]; ok { - return true, index.config.seenRemoteSources[url] + if index.config == nil || index.config.seenRemoteSources == nil { + return false, nil + } + j, _ := index.config.seenRemoteSources.Load(url) + if j != nil { + return true, j.(*yaml.Node) } return false, nil } diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 82cd5bc..0bb9cae 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -128,7 +128,6 @@ func TestSpecIndex_BaseURLError(t *testing.T) { }) assert.Len(t, index.GetAllExternalIndexes(), 0) - assert.Len(t, index.GetReferenceIndexErrors(), 582) } func TestSpecIndex_k8s(t *testing.T) { @@ -199,9 +198,9 @@ func TestSpecIndex_XSOAR(t *testing.T) { } func TestSpecIndex_PetstoreV3(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/petstorev3.json") + petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json") var rootNode yaml.Node - yaml.Unmarshal(asana, &rootNode) + yaml.Unmarshal(petstore, &rootNode) index := NewSpecIndex(&rootNode) @@ -368,21 +367,24 @@ func TestSpecIndex_NoRoot(t *testing.T) { } func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { - spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") - var rootNode yaml.Node - yaml.Unmarshal(spec, &rootNode) + spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") + var rootNode yaml.Node + yaml.Unmarshal(spec, &rootNode) - index := NewSpecIndex(&rootNode) + index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ + AllowRemoteLookup: true, + AllowFileLookup: true, + }) - assert.Len(t, index.allRefs, 5) - assert.Len(t, index.allMappedRefs, 5) - assert.Equal(t, 5, index.GetPathCount()) - assert.Equal(t, 5, index.GetOperationCount()) - assert.Equal(t, 1, index.GetComponentSchemaCount()) - assert.Equal(t, 2, index.GetGlobalTagsCount()) - assert.Equal(t, 3, index.GetTotalTagsCount()) - assert.Equal(t, 2, index.GetOperationTagsCount()) - assert.Equal(t, 0, index.GetGlobalLinksCount()) + assert.Len(t, index.allRefs, 5) + assert.Len(t, index.allMappedRefs, 5) + assert.Equal(t, 5, index.GetPathCount()) + assert.Equal(t, 5, index.GetOperationCount()) + assert.Equal(t, 1, index.GetComponentSchemaCount()) + assert.Equal(t, 2, index.GetGlobalTagsCount()) + assert.Equal(t, 3, index.GetTotalTagsCount()) + assert.Equal(t, 2, index.GetOperationTagsCount()) + assert.Equal(t, 0, index.GetGlobalLinksCount()) assert.Equal(t, 0, index.GetComponentParameterCount()) assert.Equal(t, 2, index.GetOperationsParameterCount()) assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) @@ -589,43 +591,80 @@ func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing index := new(SpecIndex) index.seenRemoteSources = make(map[string]*yaml.Node) index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") - assert.NoError(t, err) - assert.Nil(t, a) + a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") + assert.Error(t, err) + assert.Nil(t, a) assert.Nil(t, b) } // Discovered in issue https://github.com/pb33f/libopenapi/issues/37 func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") - assert.NoError(t, err) - assert.Nil(t, a) - assert.Nil(t, b) + index := new(SpecIndex) + index.seenRemoteSources = make(map[string]*yaml.Node) + index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} + a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") + assert.NoError(t, err) + assert.NotNil(t, a) + assert.NotNil(t, b) } // Discovered in issue https://github.com/daveshanley/vacuum/issues/225 func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { index := new(SpecIndex) - _ = ioutil.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") + _ = ioutil.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) + defer os.Remove("coffee-time.yaml") - index.seenRemoteSources = make(map[string]*yaml.Node) - a, b, err := index.lookupFileReference("coffee-time.yaml") - assert.NoError(t, err) - assert.NotNil(t, a) - assert.NotNil(t, b) + index.seenRemoteSources = make(map[string]*yaml.Node) + a, b, err := index.lookupFileReference("coffee-time.yaml") + assert.NoError(t, err) + assert.NotNil(t, a) + assert.NotNil(t, b) +} + +func TestSpecIndex_CheckBadURLRef(t *testing.T) { + + yml := `openapi: 3.1.0 +paths: + /cakes: + post: + parameters: + - $ref: 'httpsss://badurl'` + + var rootNode yaml.Node + yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndex(&rootNode) + + assert.Len(t, index.refErrors, 2) +} + +func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { + + yml := `openapi: 3.1.0 +paths: + /cakes: + post: + parameters: + - $ref: 'httpsss://badurl'` + + var rootNode yaml.Node + yaml.Unmarshal([]byte(yml), &rootNode) + + c := CreateClosedAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, c) + + assert.Len(t, idx.refErrors, 2) + assert.Equal(t, "remote lookups are not permitted, "+ + "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error()) } func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) { - _ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") + _ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) + defer os.Remove("coffee-time.yaml") - yml := `openapi: 3.0.3 + yml := `openapi: 3.0.3 paths: /cakes: post: diff --git a/index/utility_methods.go b/index/utility_methods.go index 9d45ff3..b22ba7c 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -401,9 +401,20 @@ func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bo } cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/")) } else { - cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/")) + if !strings.HasPrefix(dir, "http") { + if len(pathSegs) > 1 || len(dirSegs) > 1 { + cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/")) + } + } else { + cleanedPath = strings.Join(dirSegs, "/") + } + } + var p string + if baseURL.Scheme != "" && !strings.HasPrefix(dir, "http") { + p = fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath) + } else { + p = cleanedPath } - p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath) if strings.HasSuffix(p, "/") { p = p[:len(p)-1] } diff --git a/resolver/resolver.go b/resolver/resolver.go index 4741ae4..0e426ec 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -38,6 +38,10 @@ type Resolver struct { resolvedRoot *yaml.Node resolvingErrors []*ResolvingError circularReferences []*index.CircularReferenceResult + referencesVisited int + indexesVisited int + journeysTaken int + relativesSeen int } // NewResolver will create a new resolver from a *index.SpecIndex @@ -73,7 +77,6 @@ func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularRefere res = append(res, resolver.circularReferences[i]) } } - return res } @@ -93,40 +96,33 @@ func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularRef return res } +// GetJourneysTaken returns the number of journeys taken by the resolver +func (resolver *Resolver) GetJourneysTaken() int { + return resolver.journeysTaken +} + +// GetReferenceVisited returns the number of references visited by the resolver +func (resolver *Resolver) GetReferenceVisited() int { + return resolver.referencesVisited +} + +// GetIndexesVisited returns the number of indexes visited by the resolver +func (resolver *Resolver) GetIndexesVisited() int { + return resolver.indexesVisited +} + +// +func (resolver *Resolver) GetRelativesSeen() int { + return resolver.relativesSeen +} + // Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved. // this data can get big, it results in a massive duplication of data. This is a destructive method and will permanently // re-organize the node tree. Make sure you have copied your original tree before running this (if you want to preserve // original data) func (resolver *Resolver) Resolve() []*ResolvingError { - mapped := resolver.specIndex.GetMappedReferencesSequenced() - mappedIndex := resolver.specIndex.GetMappedReferences() - for _, ref := range mapped { - seenReferences := make(map[string]bool) - var journey []*index.Reference - if ref != nil && ref.Reference != nil { - ref.Reference.Node.Content = resolver.VisitReference(ref.Reference, seenReferences, journey, true) - } - } - - schemas := resolver.specIndex.GetAllComponentSchemas() - for s, schemaRef := range schemas { - if mappedIndex[s] == nil { - seenReferences := make(map[string]bool) - var journey []*index.Reference - schemaRef.Node.Content = resolver.VisitReference(schemaRef, seenReferences, journey, true) - } - } - - // map everything - for _, sequenced := range resolver.specIndex.GetAllSequencedReferences() { - locatedDef := mappedIndex[sequenced.Definition] - if locatedDef != nil { - if !locatedDef.Circular && locatedDef.Seen { - sequenced.Node.Content = locatedDef.Node.Content - } - } - } + visitIndex(resolver, resolver.specIndex) for _, circRef := range resolver.circularReferences { // If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one @@ -181,12 +177,47 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { return resolver.resolvingErrors } -// VisitReference will visit a reference as part of a journey and will return resolved nodes. -func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node { - if ref == nil { - panic("what?") +func visitIndex(res *Resolver, idx *index.SpecIndex) { + mapped := idx.GetMappedReferencesSequenced() + mappedIndex := idx.GetMappedReferences() + res.indexesVisited++ + + for _, ref := range mapped { + seenReferences := make(map[string]bool) + var journey []*index.Reference + res.journeysTaken++ + if ref != nil && ref.Reference != nil { + ref.Reference.Node.Content = res.VisitReference(ref.Reference, seenReferences, journey, true) + } } + schemas := idx.GetAllComponentSchemas() + for s, schemaRef := range schemas { + if mappedIndex[s] == nil { + seenReferences := make(map[string]bool) + var journey []*index.Reference + res.journeysTaken++ + schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) + } + } + + // map everything + for _, sequenced := range idx.GetAllSequencedReferences() { + locatedDef := mappedIndex[sequenced.Definition] + if locatedDef != nil { + if !locatedDef.Circular && locatedDef.Seen { + sequenced.Node.Content = locatedDef.Node.Content + } + } + } + for _, c := range idx.GetChildren() { + visitIndex(res, c) + } +} + +// VisitReference will visit a reference as part of a journey and will return resolved nodes. +func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node { + resolver.referencesVisited++ if ref.Resolved || ref.Seen { return ref.Node.Content } @@ -308,7 +339,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node, ref := resolver.specIndex.SearchIndexForReference(value) if ref == nil { - // TODO handle error, missing ref, can't resolve. _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) err := &ResolvingError{ ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value), @@ -414,6 +444,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node, } } } - + resolver.relativesSeen += len(found) return found } diff --git a/resolver/resolver_test.go b/resolver/resolver_test.go index 1de0d6e..84b7b9f 100644 --- a/resolver/resolver_test.go +++ b/resolver/resolver_test.go @@ -170,19 +170,56 @@ components: assert.Len(t, circ, 0) } -// TODO: Test for remote and file references with the option switched on and off -func TestResolver_ResolveComponents_MixedRef(t *testing.T) { - mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") +func TestResolver_ResolveComponents_Missing(t *testing.T) { + yml := `paths: + /hey: + get: + responses: + "200": + $ref: '#/components/schemas/crackers' +components: + schemas: + cheese: + description: cheese + properties: + thang: + $ref: '#/components/schemas/crackers' + crackers: + description: crackers + properties: + butter: + $ref: 'go home, I am drunk'` + var rootNode yaml.Node - yaml.Unmarshal(mixedref, &rootNode) + yaml.Unmarshal([]byte(yml), &rootNode) index := index.NewSpecIndex(&rootNode) resolver := NewResolver(index) assert.NotNil(t, resolver) + err := resolver.Resolve() + assert.Len(t, err, 1) + assert.Equal(t, "cannot resolve reference `go home, I am drunk`, it's missing: $go home, I am drunk [18:11]", err[0].Error()) +} + +func TestResolver_ResolveComponents_MixedRef(t *testing.T) { + mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") + var rootNode yaml.Node + yaml.Unmarshal(mixedref, &rootNode) + + b := index.CreateOpenAPIIndexConfig() + idx := index.NewSpecIndexWithConfig(&rootNode, b) + + resolver := NewResolver(idx) + assert.NotNil(t, resolver) + circ := resolver.Resolve() - assert.Len(t, circ, 10) + assert.Len(t, circ, 0) + assert.Equal(t, 5, resolver.GetIndexesVisited()) + assert.Equal(t, 209, resolver.GetRelativesSeen()) + assert.Equal(t, 35, resolver.GetJourneysTaken()) + assert.Equal(t, 62, resolver.GetReferenceVisited()) } func TestResolver_ResolveComponents_k8s(t *testing.T) { @@ -211,7 +248,8 @@ func ExampleNewResolver() { _ = yaml.Unmarshal(stripeBytes, &rootNode) // create a new spec index (resolver depends on it) - index := index.NewSpecIndex(&rootNode) + indexConfig := index.CreateClosedAPIIndexConfig() + index := index.NewSpecIndexWithConfig(&rootNode, indexConfig) // create a new resolver using the index. resolver := NewResolver(index)