From d7b9ded8bb18f24f62495afc04d652f2c4186fff Mon Sep 17 00:00:00 2001 From: Tristan Cartledge Date: Thu, 15 Dec 2022 15:22:43 +0000 Subject: [PATCH] feat: add path to references and parameters --- index/spec_index.go | 71 +++++++++++---------------- index/spec_index_test.go | 101 ++++++++++++++++++++++++++------------- 2 files changed, 94 insertions(+), 78 deletions(-) diff --git a/index/spec_index.go b/index/spec_index.go index bc8d001..ae8575e 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -15,13 +15,14 @@ package index import ( "errors" "fmt" - "github.com/pb33f/libopenapi/utils" - "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" - "gopkg.in/yaml.v3" "io/ioutil" "net/http" "strings" "sync" + + "github.com/pb33f/libopenapi/utils" + "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" + "gopkg.in/yaml.v3" ) // Constants used to determine if resolving is local, file based or remote file based. @@ -192,7 +193,6 @@ func runIndexFunction(funcs []func() int, wg *sync.WaitGroup) { // other than a raw index of every node for every content type in the specification. This process runs as fast as // possible so dependencies looking through the tree, don't need to walk the entire thing over, and over. func NewSpecIndex(rootNode *yaml.Node) *SpecIndex { - index := new(SpecIndex) index.root = rootNode index.allRefs = make(map[string]*Reference) @@ -593,7 +593,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, segs := strings.Split(value, "/") name := segs[len(segs)-1] - //name := strings.ReplaceAll(segs[len(segs)-1], "~1", "/") + // name := strings.ReplaceAll(segs[len(segs)-1], "~1", "/") ref := &Reference{ Definition: value, Name: name, @@ -972,7 +972,7 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { return index.globalCallbacksCount } - //index.pathRefsLock.Lock() + // index.pathRefsLock.Lock() for path, p := range index.pathRefs { for _, m := range p { @@ -981,7 +981,6 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { res, _ := callbacks.Find(m.Node) if len(res) > 0 { - for _, callback := range res[0].Content { if utils.IsNodeMap(callback) { @@ -1005,7 +1004,7 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { } } } - //index.pathRefsLock.Unlock() + // index.pathRefsLock.Unlock() return index.globalCallbacksCount } @@ -1019,7 +1018,7 @@ func (index *SpecIndex) GetGlobalLinksCount() int { return index.globalLinksCount } - //index.pathRefsLock.Lock() + // index.pathRefsLock.Lock() for path, p := range index.pathRefs { for _, m := range p { @@ -1028,7 +1027,6 @@ func (index *SpecIndex) GetGlobalLinksCount() int { res, _ := links.Find(m.Node) if len(res) > 0 { - for _, link := range res[0].Content { if utils.IsNodeMap(link) { @@ -1050,7 +1048,7 @@ func (index *SpecIndex) GetGlobalLinksCount() int { } } } - //index.pathRefsLock.Unlock() + // index.pathRefsLock.Unlock() return index.globalLinksCount } @@ -1383,7 +1381,6 @@ func (index *SpecIndex) GetOperationsParameterCount() int { // method level params. if isHttpMethod(prop.Value) { - for z, httpMethodProp := range pathPropertyNode.Content[y+1].Content { if z%2 == 0 { if httpMethodProp.Value == "parameters" { @@ -1478,8 +1475,8 @@ func (index *SpecIndex) GetOperationsParameterCount() int { } } - //now build main index of all params by combining comp refs with inline params from operations. - //use the namespace path:::param for inline params to identify them as inline. + // now build main index of all params by combining comp refs with inline params from operations. + // use the namespace path:::param for inline params to identify them as inline. for path, params := range index.paramOpRefs { for mName, mValue := range params { for pName, pValue := range mValue { @@ -1493,7 +1490,6 @@ func (index *SpecIndex) GetOperationsParameterCount() int { index.operationParamCount = len(index.paramCompRefs) + len(index.paramInlineDuplicates) return index.operationParamCount - } // GetInlineDuplicateParamCount returns the number of inline duplicate parameters (operation params) @@ -1777,15 +1773,12 @@ func (index *SpecIndex) extractComponentSecuritySchemes(securitySchemesNode *yam } func (index *SpecIndex) performExternalLookup(uri []string, componentId string, - lookupFunction ExternalLookupFunction, parent *yaml.Node) *Reference { - + lookupFunction ExternalLookupFunction, parent *yaml.Node, +) *Reference { if len(uri) > 0 { externalSpecIndex := index.externalSpecIndex[uri[0]] - var foundNode *yaml.Node if externalSpecIndex == nil { - - n, newRoot, err := lookupFunction(componentId) - + _, newRoot, err := lookupFunction(componentId) if err != nil { indexError := &IndexingError{ Error: err, @@ -1796,31 +1789,23 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string, return nil } - if n != nil { - foundNode = n - } - // cool, cool, lets index this spec also. This is a recursive action and will keep going // until all remote references have been found. newIndex := NewSpecIndex(newRoot) index.externalSpecIndex[uri[0]] = newIndex - - } else { - - foundRef := externalSpecIndex.FindComponentInRoot(uri[1]) - if foundRef != nil { - foundNode = foundRef.Node - } + externalSpecIndex = newIndex } - if foundNode != nil { + foundRef := externalSpecIndex.FindComponentInRoot(uri[1]) + if foundRef != nil { nameSegs := strings.Split(uri[1], "/") ref := &Reference{ Definition: componentId, Name: nameSegs[len(nameSegs)-1], - Node: foundNode, + Node: foundRef.Node, IsRemote: true, RemoteLocation: componentId, + Path: foundRef.Path, } return ref } @@ -1842,6 +1827,7 @@ func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference { Definition: componentId, Name: name, Node: res[0], + Path: friendlySearch, } return ref } @@ -1865,7 +1851,6 @@ func (index *SpecIndex) countUniqueInlineDuplicates() int { func (index *SpecIndex) scanOperationParams(params []*yaml.Node, pathItemNode *yaml.Node, method string) { for i, param := range params { - // param is ref if len(param.Content) > 0 && param.Content[0].Value == "$ref" { @@ -1886,16 +1871,15 @@ func (index *SpecIndex) scanOperationParams(params []*yaml.Node, pathItemNode *y } else { - //param is inline. + // param is inline. _, vn := utils.FindKeyNode("name", param.Content) + path := fmt.Sprintf("$.paths.%s.%s.parameters[%d]", pathItemNode.Value, method, i) + if method == "top" { + path = fmt.Sprintf("$.paths.%s.parameters[%d]", pathItemNode.Value, i) + } + if vn == nil { - - path := fmt.Sprintf("$.paths.%s.%s.parameters[%d]", pathItemNode.Value, method, i) - if method == "top" { - path = fmt.Sprintf("$.paths.%s.parameters[%d]", pathItemNode.Value, i) - } - index.operationParamErrors = append(index.operationParamErrors, &IndexingError{ Error: fmt.Errorf("the '%s' operation parameter at path '%s', index %d has no 'name' value", method, pathItemNode.Value, i), @@ -1909,6 +1893,7 @@ func (index *SpecIndex) scanOperationParams(params []*yaml.Node, pathItemNode *y Definition: vn.Value, Name: vn.Value, Node: param, + Path: path, } if index.paramOpRefs[pathItemNode.Value] == nil { index.paramOpRefs[pathItemNode.Value] = make(map[string]map[string]*Reference) @@ -1962,7 +1947,6 @@ func isHttpMethod(val string) bool { } func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { - // split string to remove file reference uri := strings.Split(ref, "#") @@ -2014,7 +1998,6 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod } func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, error) { - // split string to remove file reference uri := strings.Split(ref, "#") diff --git a/index/spec_index_test.go b/index/spec_index_test.go index c9431de..df37149 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -5,15 +5,15 @@ package index import ( "fmt" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" "io/ioutil" "os" "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestSpecIndex_ExtractRefsStripe(t *testing.T) { - stripe, _ := ioutil.ReadFile("../test_specs/stripe.yaml") var rootNode yaml.Node yaml.Unmarshal(stripe, &rootNode) @@ -61,7 +61,6 @@ func TestSpecIndex_ExtractRefsStripe(t *testing.T) { } func TestSpecIndex_Asana(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/asana.yaml") var rootNode yaml.Node yaml.Unmarshal(asana, &rootNode) @@ -84,7 +83,6 @@ func TestSpecIndex_Asana(t *testing.T) { } func TestSpecIndex_k8s(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/k8s.json") var rootNode yaml.Node yaml.Unmarshal(asana, &rootNode) @@ -107,11 +105,9 @@ func TestSpecIndex_k8s(t *testing.T) { assert.Equal(t, 10, index.componentsInlineParamUniqueCount) assert.Equal(t, 58, index.GetTotalTagsCount()) assert.Equal(t, 2524, index.GetRawReferenceCount()) - } func TestSpecIndex_PetstoreV2(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/petstorev2.json") var rootNode yaml.Node yaml.Unmarshal(asana, &rootNode) @@ -136,7 +132,6 @@ func TestSpecIndex_PetstoreV2(t *testing.T) { } func TestSpecIndex_XSOAR(t *testing.T) { - xsoar, _ := ioutil.ReadFile("../test_specs/xsoar.json") var rootNode yaml.Node yaml.Unmarshal(xsoar, &rootNode) @@ -155,7 +150,6 @@ func TestSpecIndex_XSOAR(t *testing.T) { } func TestSpecIndex_PetstoreV3(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/petstorev3.json") var rootNode yaml.Node yaml.Unmarshal(asana, &rootNode) @@ -179,13 +173,11 @@ func TestSpecIndex_PetstoreV3(t *testing.T) { assert.Equal(t, 19, index.GetAllSummariesCount()) assert.Len(t, index.GetAllDescriptions(), 90) assert.Len(t, index.GetAllSummaries(), 19) - } var mappedRefs = 15 func TestSpecIndex_BurgerShop(t *testing.T) { - burgershop, _ := ioutil.ReadFile("../test_specs/burgershop.openapi.yaml") var rootNode yaml.Node yaml.Unmarshal(burgershop, &rootNode) @@ -254,11 +246,9 @@ func TestSpecIndex_BurgerShop(t *testing.T) { assert.Equal(t, 5, len(index.GetAllPaths())) assert.Equal(t, 5, len(index.GetOperationTags())) assert.Equal(t, 3, len(index.GetAllParametersFromOperations())) - } func TestSpecIndex_BurgerShop_AllTheComponents(t *testing.T) { - burgershop, _ := ioutil.ReadFile("../test_specs/all-the-components.yaml") var rootNode yaml.Node yaml.Unmarshal(burgershop, &rootNode) @@ -272,11 +262,9 @@ func TestSpecIndex_BurgerShop_AllTheComponents(t *testing.T) { assert.Equal(t, 1, len(index.GetAllResponses())) assert.Equal(t, 2, len(index.GetAllRootServers())) assert.Equal(t, 2, len(index.GetAllOperationsServers())) - } func TestSpecIndex_SwaggerResponses(t *testing.T) { - yml := `swagger: 2.0 responses: niceResponse: @@ -288,11 +276,9 @@ responses: index := NewSpecIndex(&rootNode) assert.Equal(t, 1, len(index.GetAllResponses())) - } func TestSpecIndex_NoNameParam(t *testing.T) { - yml := `paths: /users/{id}: parameters: @@ -311,11 +297,9 @@ func TestSpecIndex_NoNameParam(t *testing.T) { index := NewSpecIndex(&rootNode) assert.Equal(t, 2, len(index.GetOperationParametersIndexErrors())) - } func TestSpecIndex_NoRoot(t *testing.T) { - index := NewSpecIndex(nil) refs := index.ExtractRefs(nil, nil, nil, 0, false, "") docs := index.ExtractExternalDocuments(nil) @@ -331,11 +315,9 @@ func TestSpecIndex_NoRoot(t *testing.T) { assert.Equal(t, -1, index.GetComponentParameterCount()) assert.Equal(t, -1, index.GetComponentSchemaCount()) assert.Equal(t, -1, index.GetGlobalLinksCount()) - } func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { - spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") var rootNode yaml.Node yaml.Unmarshal(spec, &rootNode) @@ -355,11 +337,9 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { assert.Equal(t, 2, index.GetOperationsParameterCount()) assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) assert.Equal(t, 1, index.GetInlineUniqueParamCount()) - } func TestSpecIndex_TestEmptyBrokenReferences(t *testing.T) { - asana, _ := ioutil.ReadFile("../test_specs/badref-burgershop.openapi.yaml") var rootNode yaml.Node yaml.Unmarshal(asana, &rootNode) @@ -380,7 +360,6 @@ func TestSpecIndex_TestEmptyBrokenReferences(t *testing.T) { } func TestTagsNoDescription(t *testing.T) { - yml := `tags: - name: one - name: two @@ -391,7 +370,6 @@ func TestTagsNoDescription(t *testing.T) { index := NewSpecIndex(&rootNode) assert.Equal(t, 3, index.GetGlobalTagsCount()) - } func TestGlobalCallbacksNoIndexTest(t *testing.T) { @@ -400,7 +378,6 @@ func TestGlobalCallbacksNoIndexTest(t *testing.T) { } func TestMultipleCallbacksPerOperationVerb(t *testing.T) { - yml := `components: callbacks: callbackA: @@ -516,7 +493,6 @@ func TestSpecIndex_FindComponenth(t *testing.T) { index := NewSpecIndex(&rootNode) assert.Nil(t, index.FindComponent("I-do-not-exist", nil)) - } func TestSpecIndex_performExternalLookup(t *testing.T) { @@ -611,8 +587,7 @@ func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) { } func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) { - - _ = ioutil.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0664) + _ = ioutil.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664) defer os.Remove("chickers.yaml") index := new(SpecIndex) @@ -621,8 +596,7 @@ func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) { } func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) { - - _ = ioutil.WriteFile("embie.yaml", []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy"), 0664) + _ = ioutil.WriteFile("embie.yaml", []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy"), 0o664) defer os.Remove("embie.yaml") index := new(SpecIndex) @@ -634,8 +608,7 @@ func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) { } func TestSpecIndex_lookupFileReference(t *testing.T) { - - _ = ioutil.WriteFile("fox.yaml", []byte("good:\n - puppy: dog\n - puppy: forever-more"), 0664) + _ = ioutil.WriteFile("fox.yaml", []byte("good:\n - puppy: dog\n - puppy: forever-more"), 0o664) defer os.Remove("fox.yaml") index := new(SpecIndex) @@ -644,12 +617,72 @@ func TestSpecIndex_lookupFileReference(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, doc) assert.NotNil(t, k) +} +func TestSpecIndex_parameterReferencesHavePaths(t *testing.T) { + _ = ioutil.WriteFile("paramour.yaml", []byte(`components: + parameters: + param3: + name: param3 + in: query + schema: + type: string`), 0o664) + defer os.Remove("paramour.yaml") + + yml := `paths: + /: + parameters: + - $ref: '#/components/parameters/param1' + - $ref: 'paramour.yaml#/components/parameters/param3' + get: + parameters: + - $ref: '#/components/parameters/param2' + - name: test + in: query + schema: + type: string +components: + parameters: + param1: + name: param1 + in: query + schema: + type: string + param2: + name: param2 + in: query + schema: + type: string` + + var rootNode yaml.Node + yaml.Unmarshal([]byte(yml), &rootNode) + + index := NewSpecIndex(&rootNode) + + params := index.GetAllParametersFromOperations() + + if assert.Contains(t, params, "/") { + if assert.Contains(t, params["/"], "top") { + if assert.Contains(t, params["/"]["top"], "#/components/parameters/param1") { + assert.Equal(t, "$.components.parameters['param1']", params["/"]["top"]["#/components/parameters/param1"].Path) + } + if assert.Contains(t, params["/"]["top"], "paramour.yaml#/components/parameters/param3") { + assert.Equal(t, "$.components.parameters['param3']", params["/"]["top"]["paramour.yaml#/components/parameters/param3"].Path) + } + } + if assert.Contains(t, params["/"], "get") { + if assert.Contains(t, params["/"]["get"], "#/components/parameters/param2") { + assert.Equal(t, "$.components.parameters['param2']", params["/"]["get"]["#/components/parameters/param2"].Path) + } + if assert.Contains(t, params["/"]["get"], "test") { + assert.Equal(t, "$.paths./.get.parameters[1]", params["/"]["get"]["test"].Path) + } + } + } } // Example of how to load in an OpenAPI Specification and index it. func ExampleNewSpecIndex() { - // define a rootNode to hold our raw spec AST. var rootNode yaml.Node