LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-06 12:37:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support for relative links now in place #73

Browse Source 
Tests all passing, runs super fast, pulls in every single DigitalOcean spec and parses it. There may be some issues deeper down in the models, but for now high level tests all pass.
This commit is contained in:
Dave Shanley
2023-02-18 13:57:20 -05:00
parent e19b7d2bf1
commit e6626fe22e
16 changed files with 471 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
datamodel/low/extraction_functions.go
View File

@@ -39,7 +39,6 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re
idx.GetAllHeaders,
idx.GetAllCallbacks,
idx.GetAllLinks,
idx.GetAllExternalDocuments,
idx.GetAllExamples,
idx.GetAllRequestBodies,
idx.GetAllResponses,
@@ -51,6 +50,7 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re
// the reference being supplied. If there is a match found, the reference *yaml.Node is returned.
func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
if rf, _, rv := utils.IsNodeRefValue(root); rf {
// run through everything and return as soon as we find a match.
// this operates as fast as possible as ever
collections := generateIndexCollection(idx)
@@ -89,11 +89,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
}
}
// perform a search for the reference in the index
foundRefs := idx.SearchIndexForReference(rv)
if len(foundRefs) > 0 {
return foundRefs[0].Node, nil
}
// let's try something else to find our references.
// cant be found? last resort is to try a path lookup
_, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv)
if friendly != "" {

4
datamodel/low/v3/path_item.go
View File

@@ -194,7 +194,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
var op Operation
wg.Add(1)
if ok, _, _ := utils.IsNodeRefValue(pathNode); ok {
r, err := low.LocateRefNode(pathNode, idx)
@@ -210,7 +210,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column)
}
}
wg.Add(1)
go low.BuildModelAsync(pathNode, &op, &wg, &errors)
opRef := low.NodeReference[*Operation]{

2
document_test.go
View File

@@ -113,7 +113,7 @@ paths:
assert.NoError(t, err)
v3Doc, docErr := doc.BuildV3Model()
assert.Len(t, docErr, 1)
assert.Len(t, docErr, 2)
assert.Nil(t, v3Doc)
}

1
go.mod
View File

@@ -5,6 +5,7 @@ go 1.18
require (
github.com/stretchr/testify v1.8.0
github.com/vmware-labs/yaml-jsonpath v0.3.2
golang.org/x/sync v0.1.0
gopkg.in/yaml.v3 v3.0.1
)

3
go.sum
View File

@@ -75,7 +75,10 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

34
index/extract_refs.go
View File

@@ -325,8 +325,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
var found []*Reference
//run this async because when things get recursive, it can take a while
//c := make(chan bool)
//tm := make(chan bool)
c := make(chan bool)
locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
located := index.FindComponent(ref.Definition, ref.Node)
@@ -351,7 +350,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
}
index.refErrors = append(index.refErrors, indexError)
}
//c <- true
c <- true
}
var refsToCheck []*Reference
@@ -372,32 +371,19 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
refsToCheck = append(refsToCheck, ref)
}
mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
//go func() {
// time.Sleep(20 * time.Second)
// tm <- true
//}()
for r := range refsToCheck {
// expand our index of all mapped refs
// go locate(refsToCheck[r], r, mappedRefsInSequence)
locate(refsToCheck[r], r, mappedRefsInSequence)
go locate(refsToCheck[r], r, mappedRefsInSequence)
}
//completedRefs := 0
//for completedRefs < len(refsToCheck) {
// select {
// case <-c:
// completedRefs++
// if completedRefs >= len(refsToCheck) {
// fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck))
// break
// } else {
// //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck))
// }
// //case <-tm:
// // panic("OH NO")
// }
//}
completedRefs := 0
for completedRefs < len(refsToCheck) {
select {
case <-c:
completedRefs++
}
}
for m := range mappedRefsInSequence {
if mappedRefsInSequence[m] != nil {
index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])

101
index/find_component.go
View File

@@ -9,9 +9,10 @@ import (
"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
"gopkg.in/yaml.v3"
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync"
"time"
)
// FindComponent will locate a component by its reference, returns nil if nothing is found.
@@ -26,7 +27,7 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
if index.config.AllowRemoteLookup {
return index.lookupRemoteReference(id)
} else {
return nil, nil, fmt.Errorf("remote lookups are not premitted, " +
return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
"please set AllowRemoteLookup to true in the configuration")
}
}
@@ -82,11 +83,22 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
return nil
}
var n sync.Mutex
var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second}
var openCalls = 0
var closedCalls = 0
var totalCalls = 0
func getRemoteDoc(u string, d chan []byte, e chan error) {
resp, err := httpClient.Get(u)
if err != nil {
e <- err
close(e)
close(d)
return
}
var body []byte
body, _ = ioutil.ReadAll(resp.Body)
d <- body
close(e)
close(d)
}
func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
// split string to remove file reference
@@ -99,30 +111,43 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
if alreadySeen {
parsedRemoteDocument = foundDocument
} else {
resp, err := index.httpClient.Get(uri[0])
totalCalls++
fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls)
if err != nil {
return nil, nil, err
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, nil, err
}
d := make(chan bool)
var body []byte
var err error
go func(uri string) {
bc := make(chan []byte)
ec := make(chan error)
go getRemoteDoc(uri, bc, ec)
select {
case v := <-bc:
body = v
break
case er := <-ec:
err = er
break
}
var remoteDoc yaml.Node
err = yaml.Unmarshal(body, &remoteDoc)
if err != nil {
return nil, nil, err
er := yaml.Unmarshal(body, &remoteDoc)
if er != nil {
err = er
d <- true
return
}
parsedRemoteDocument = &remoteDoc
//n.Lock()
//index.seenRemoteSources[uri[0]] = &remoteDoc
if index.config != nil {
index.config.seenRemoteSources[uri[0]] = &remoteDoc
index.config.seenRemoteSources.Store(uri, &remoteDoc)
}
d <- true
}(uri[0])
// wait for double go fun.
<-d
if err != nil {
// no bueno.
return nil, nil, err
}
//n.Unlock()
}
// lookup item from reference by using a path query.
@@ -163,9 +188,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
// try and read the file off the local file system, if it fails
// check for a baseURL and then ask our remote lookup function to go try and get it.
// index.fileLock.Lock()
body, err := ioutil.ReadFile(file)
// index.fileLock.Unlock()
if err != nil {
@@ -173,26 +196,12 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
if index.config != nil && index.config.BaseURL != nil {
u := index.config.BaseURL
remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
a, b, e := index.lookupRemoteReference(remoteRef)
if e != nil {
// give up, we can't find the file, not locally, not remotely. It's toast.
return nil, nil, e
}
//// everything looks good, lets just make sure we also add a key to the raw reference name.
//if _, ok := index.seenRemoteSources[file]; !ok {
// if b != nil {
// //index.seenRemoteSources[ref] = b
// } else {
// panic("oh now")
// }
//
//} else {
// panic("oh no")
//}
return a, b, nil
} else {
@@ -207,8 +216,10 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
return nil, nil, err
}
parsedRemoteDocument = &remoteDoc
if index.seenLocalSources != nil {
index.seenLocalSources[file] = &remoteDoc
}
}
// lookup item from reference by using a path query.
var query string
@@ -279,10 +290,14 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
// cool, cool, lets index this spec also. This is a recursive action and will keep going
// until all remote references have been found.
// TODO: start here tomorrow, need to pass in the config to the new spec index.
// set the base URL to the path.
var j *url.URL
if index.config.BaseURL != nil {
j = index.config.BaseURL
} else {
j, _ = url.Parse(uri[0])
}
path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false)
path := GenerateCleanSpecConfigBaseURL(j, uri[0], false)
newUrl, e := url.Parse(path)
if e == nil {
@@ -296,10 +311,12 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
var newIndex *SpecIndex
newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
index.refLock.Lock()
index.externalSpecIndex[uri[0]] = newIndex
newIndex.relativePath = path
newIndex.parentIndex = index
index.AddChild(newIndex)
index.refLock.Unlock()
externalSpecIndex = newIndex
} else {

32
index/index_model.go
View File

@@ -4,6 +4,7 @@
package index
import (
"golang.org/x/sync/syncmap"
"gopkg.in/yaml.v3"
"net/http"
"net/url"
@@ -72,10 +73,30 @@ type SpecIndexConfig struct {
RootIndex *SpecIndex // if this is a sub-index, the root knows about everything below.
ParentIndex *SpecIndex // Who owns this index?
seenRemoteSources map[string]*yaml.Node
seenRemoteSources *syncmap.Map
remoteLock *sync.Mutex
}
// CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
// AllowFileLookup set to true. This is the default behaviour of the index in previous versions of libopenapi. (pre 0.6.0)
func CreateOpenAPIIndexConfig() *SpecIndexConfig {
return &SpecIndexConfig{
AllowRemoteLookup: true,
AllowFileLookup: true,
seenRemoteSources: &syncmap.Map{},
}
}
// CreateClosedAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
// AllowFileLookup set to false. This is the default behaviour of the index in versions 0.6.0+
func CreateClosedAPIIndexConfig() *SpecIndexConfig {
return &SpecIndexConfig{
AllowRemoteLookup: false,
AllowFileLookup: false,
seenRemoteSources: &syncmap.Map{},
}
}
// SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and
// quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules,
// everything is pre-walked if you need it.
@@ -175,9 +196,6 @@ type SpecIndex struct {
summaryCount int
seenRemoteSources map[string]*yaml.Node
seenLocalSources map[string]*yaml.Node
remoteLock sync.RWMutex
httpLock sync.RWMutex
fileLock sync.Mutex
refLock sync.Mutex
circularReferences []*CircularReferenceResult // only available when the resolver has been used.
allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false.
@@ -190,7 +208,6 @@ type SpecIndex struct {
// when things get complex (looking at you digital ocean) then we need to know
// what we have seen across indexes, so we need to be able to travel back up to the root
// cto avoid re-downloading sources.
rootIndex *SpecIndex
parentIndex *SpecIndex
children []*SpecIndex
}
@@ -199,6 +216,11 @@ func (index *SpecIndex) AddChild(child *SpecIndex) {
index.children = append(index.children, child)
}
// GetChildren returns the children of this index.
func (index *SpecIndex) GetChildren() []*SpecIndex {
return index.children
}
// ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the
// URI based document. Decides if the reference is local, remote or in a file.
type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error)

25
index/index_model_test.go Normal file
View File

@@ -0,0 +1,25 @@
// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT
package index
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestSpecIndex_Children(t *testing.T) {
idx1 := new(SpecIndex)
idx2 := new(SpecIndex)
idx3 := new(SpecIndex)
idx4 := new(SpecIndex)
idx5 := new(SpecIndex)
idx1.AddChild(idx2)
idx1.AddChild(idx3)
idx3.AddChild(idx4)
idx4.AddChild(idx5)
assert.Equal(t, 2, len(idx1.GetChildren()))
assert.Equal(t, 1, len(idx3.GetChildren()))
assert.Equal(t, 1, len(idx4.GetChildren()))
assert.Equal(t, 0, len(idx5.GetChildren()))
}

25
index/search_index.go
View File

@@ -3,16 +3,37 @@
package index
func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
import "gopkg.in/yaml.v3"
// SearchIndexForReference searches the index for a reference, first looking through the mapped references
// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes
// extracted when parsing the OpenAPI Spec.
func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
if r, ok := index.allMappedRefs[ref]; ok {
if r.Node.Kind == yaml.DocumentNode {
// the reference is an entire document, so we need to dig down a level and rewire the reference.
r.Node = r.Node.Content[0]
}
return []*Reference{r}
}
if r, ok := index.externalSpecIndex[ref]; ok {
return []*Reference{
{
Node: r.root.Content[0],
Name: ref,
Definition: ref,
},
}
}
for c := range index.children {
found := index.children[c].SearchIndexForReference(ref)
found := goFindMeSomething(index.children[c], ref)
if found != nil {
return found
}
}
return nil
}
func goFindMeSomething(i *SpecIndex, ref string) []*Reference {
return i.SearchIndexForReference(ref)
}

49
index/search_index_test.go Normal file
View File

@@ -0,0 +1,49 @@
// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT
package index
import (
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
"io/ioutil"
"net/url"
"testing"
)
func TestSpecIndex_SearchIndexForReference(t *testing.T) {
petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
var rootNode yaml.Node
_ = yaml.Unmarshal(petstore, &rootNode)
c := CreateOpenAPIIndexConfig()
idx := NewSpecIndexWithConfig(&rootNode, c)
ref := idx.SearchIndexForReference("#/components/schemas/Pet")
assert.NotNil(t, ref)
}
func TestSpecIndex_SearchIndexForReferene_ExternalSpecs(t *testing.T) {
// load up an index with lots of references
petstore, _ := ioutil.ReadFile("../test_specs/digitalocean.yaml")
var rootNode yaml.Node
_ = yaml.Unmarshal(petstore, &rootNode)
c := CreateOpenAPIIndexConfig()
c.BaseURL, _ = url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification")
idx := NewSpecIndexWithConfig(&rootNode, c)
ref := idx.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml")
assert.NotNil(t, ref)
assert.Equal(t, "operationId", ref[0].Node.Content[0].Value)
ref = idx.SearchIndexForReference("examples/ruby/domains_create.yml")
assert.NotNil(t, ref)
assert.Equal(t, "lang", ref[0].Node.Content[0].Value)
ref = idx.SearchIndexForReference("../../shared/responses/server_error.yml")
assert.NotNil(t, ref)
assert.Equal(t, "description", ref[0].Node.Content[0].Value)
}

26
index/spec_index.go
View File

@@ -16,6 +16,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/utils"
"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
"golang.org/x/sync/syncmap"
"gopkg.in/yaml.v3"
"strings"
"sync"
@@ -27,7 +28,7 @@ import (
func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex {
index := new(SpecIndex)
if config != nil && config.seenRemoteSources == nil {
config.seenRemoteSources = make(map[string]*yaml.Node)
config.seenRemoteSources = &syncmap.Map{}
}
config.remoteLock = &sync.Mutex{}
index.config = config
@@ -46,11 +47,7 @@ func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecI
// - https://github.com/pb33f/libopenapi/issues/73
func NewSpecIndex(rootNode *yaml.Node) *SpecIndex {
index := new(SpecIndex)
index.config = &SpecIndexConfig{
AllowRemoteLookup: true,
AllowFileLookup: true,
seenRemoteSources: make(map[string]*yaml.Node),
}
index.config = CreateOpenAPIIndexConfig()
boostrapIndexCollections(rootNode, index)
return createNewIndex(rootNode, index)
}
@@ -108,7 +105,12 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex {
index.GetInlineDuplicateParamCount()
index.GetAllDescriptionsCount()
index.GetTotalTagsCount()
index.seenRemoteSources = index.config.seenRemoteSources
// do a copy!
index.config.seenRemoteSources.Range(func(k, v any) bool {
index.seenRemoteSources[k.(string)] = v.(*yaml.Node)
return true
})
return index
}
@@ -1138,9 +1140,15 @@ func (index *SpecIndex) GetAllSummariesCount() int {
return len(index.allSummaries)
}
// CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it,
// to avoid making duplicate remote calls for document data.
func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) {
if _, ok := index.config.seenRemoteSources[url]; ok {
return true, index.config.seenRemoteSources[url]
if index.config == nil || index.config.seenRemoteSources == nil {
return false, nil
}
j, _ := index.config.seenRemoteSources.Load(url)
if j != nil {
return true, j.(*yaml.Node)
}
return false, nil
}

53
index/spec_index_test.go
View File

@@ -128,7 +128,6 @@ func TestSpecIndex_BaseURLError(t *testing.T) {
})
assert.Len(t, index.GetAllExternalIndexes(), 0)
assert.Len(t, index.GetReferenceIndexErrors(), 582)
}
func TestSpecIndex_k8s(t *testing.T) {
@@ -199,9 +198,9 @@ func TestSpecIndex_XSOAR(t *testing.T) {
}
func TestSpecIndex_PetstoreV3(t *testing.T) {
asana, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
var rootNode yaml.Node
yaml.Unmarshal(asana, &rootNode)
yaml.Unmarshal(petstore, &rootNode)
index := NewSpecIndex(&rootNode)
@@ -372,7 +371,10 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
var rootNode yaml.Node
yaml.Unmarshal(spec, &rootNode)
index := NewSpecIndex(&rootNode)
index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
AllowRemoteLookup: true,
AllowFileLookup: true,
})
assert.Len(t, index.allRefs, 5)
assert.Len(t, index.allMappedRefs, 5)
@@ -590,7 +592,7 @@ func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing
index.seenRemoteSources = make(map[string]*yaml.Node)
index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
assert.NoError(t, err)
assert.Error(t, err)
assert.Nil(t, a)
assert.Nil(t, b)
}
@@ -602,8 +604,8 @@ func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) {
index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
assert.NoError(t, err)
assert.Nil(t, a)
assert.Nil(t, b)
assert.NotNil(t, a)
assert.NotNil(t, b)
}
// Discovered in issue https://github.com/daveshanley/vacuum/issues/225
@@ -620,6 +622,43 @@ func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
assert.NotNil(t, b)
}
func TestSpecIndex_CheckBadURLRef(t *testing.T) {
yml := `openapi: 3.1.0
paths:
/cakes:
post:
parameters:
- $ref: 'httpsss://badurl'`
var rootNode yaml.Node
yaml.Unmarshal([]byte(yml), &rootNode)
index := NewSpecIndex(&rootNode)
assert.Len(t, index.refErrors, 2)
}
func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) {
yml := `openapi: 3.1.0
paths:
/cakes:
post:
parameters:
- $ref: 'httpsss://badurl'`
var rootNode yaml.Node
yaml.Unmarshal([]byte(yml), &rootNode)
c := CreateClosedAPIIndexConfig()
idx := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, idx.refErrors, 2)
assert.Equal(t, "remote lookups are not permitted, "+
"please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error())
}
func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) {
_ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664)

13
index/utility_methods.go
View File

@@ -401,9 +401,20 @@ func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bo
}
cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/"))
} else {
if !strings.HasPrefix(dir, "http") {
if len(pathSegs) > 1 || len(dirSegs) > 1 {
cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/"))
}
p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
} else {
cleanedPath = strings.Join(dirSegs, "/")
}
}
var p string
if baseURL.Scheme != "" && !strings.HasPrefix(dir, "http") {
p = fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
} else {
p = cleanedPath
}
if strings.HasSuffix(p, "/") {
p = p[:len(p)-1]
}

100
resolver/resolver.go
View File

@@ -38,6 +38,10 @@ type Resolver struct {
resolvedRoot *yaml.Node
resolvingErrors []*ResolvingError
circularReferences []*index.CircularReferenceResult
referencesVisited int
indexesVisited int
journeysTaken int
relativesSeen int
}
// NewResolver will create a new resolver from a *index.SpecIndex
@@ -73,7 +77,6 @@ func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularRefere
res = append(res, resolver.circularReferences[i])
}
}
return res
}
@@ -93,40 +96,33 @@ func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularRef
return res
}
// GetJourneysTaken returns the number of journeys taken by the resolver
func (resolver *Resolver) GetJourneysTaken() int {
return resolver.journeysTaken
}
// GetReferenceVisited returns the number of references visited by the resolver
func (resolver *Resolver) GetReferenceVisited() int {
return resolver.referencesVisited
}
// GetIndexesVisited returns the number of indexes visited by the resolver
func (resolver *Resolver) GetIndexesVisited() int {
return resolver.indexesVisited
}
//
func (resolver *Resolver) GetRelativesSeen() int {
return resolver.relativesSeen
}
// Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved.
// this data can get big, it results in a massive duplication of data. This is a destructive method and will permanently
// re-organize the node tree. Make sure you have copied your original tree before running this (if you want to preserve
// original data)
func (resolver *Resolver) Resolve() []*ResolvingError {
mapped := resolver.specIndex.GetMappedReferencesSequenced()
mappedIndex := resolver.specIndex.GetMappedReferences()
for _, ref := range mapped {
seenReferences := make(map[string]bool)
var journey []*index.Reference
if ref != nil && ref.Reference != nil {
ref.Reference.Node.Content = resolver.VisitReference(ref.Reference, seenReferences, journey, true)
}
}
schemas := resolver.specIndex.GetAllComponentSchemas()
for s, schemaRef := range schemas {
if mappedIndex[s] == nil {
seenReferences := make(map[string]bool)
var journey []*index.Reference
schemaRef.Node.Content = resolver.VisitReference(schemaRef, seenReferences, journey, true)
}
}
// map everything
for _, sequenced := range resolver.specIndex.GetAllSequencedReferences() {
locatedDef := mappedIndex[sequenced.Definition]
if locatedDef != nil {
if !locatedDef.Circular && locatedDef.Seen {
sequenced.Node.Content = locatedDef.Node.Content
}
}
}
visitIndex(resolver, resolver.specIndex)
for _, circRef := range resolver.circularReferences {
// If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one
@@ -181,12 +177,47 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError {
return resolver.resolvingErrors
}
// VisitReference will visit a reference as part of a journey and will return resolved nodes.
func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
if ref == nil {
panic("what?")
func visitIndex(res *Resolver, idx *index.SpecIndex) {
mapped := idx.GetMappedReferencesSequenced()
mappedIndex := idx.GetMappedReferences()
res.indexesVisited++
for _, ref := range mapped {
seenReferences := make(map[string]bool)
var journey []*index.Reference
res.journeysTaken++
if ref != nil && ref.Reference != nil {
ref.Reference.Node.Content = res.VisitReference(ref.Reference, seenReferences, journey, true)
}
}
schemas := idx.GetAllComponentSchemas()
for s, schemaRef := range schemas {
if mappedIndex[s] == nil {
seenReferences := make(map[string]bool)
var journey []*index.Reference
res.journeysTaken++
schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true)
}
}
// map everything
for _, sequenced := range idx.GetAllSequencedReferences() {
locatedDef := mappedIndex[sequenced.Definition]
if locatedDef != nil {
if !locatedDef.Circular && locatedDef.Seen {
sequenced.Node.Content = locatedDef.Node.Content
}
}
}
for _, c := range idx.GetChildren() {
visitIndex(res, c)
}
}
// VisitReference will visit a reference as part of a journey and will return resolved nodes.
func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
resolver.referencesVisited++
if ref.Resolved || ref.Seen {
return ref.Node.Content
}
@@ -308,7 +339,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
ref := resolver.specIndex.SearchIndexForReference(value)
if ref == nil {
// TODO handle error, missing ref, can't resolve.
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
err := &ResolvingError{
ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value),
@@ -414,6 +444,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
}
}
}
resolver.relativesSeen += len(found)
return found
}

50
resolver/resolver_test.go
View File

@@ -170,19 +170,56 @@ components:
assert.Len(t, circ, 0)
}
// TODO: Test for remote and file references with the option switched on and off
func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
func TestResolver_ResolveComponents_Missing(t *testing.T) {
yml := `paths:
/hey:
get:
responses:
"200":
$ref: '#/components/schemas/crackers'
components:
schemas:
cheese:
description: cheese
properties:
thang:
$ref: '#/components/schemas/crackers'
crackers:
description: crackers
properties:
butter:
$ref: 'go home, I am drunk'`
var rootNode yaml.Node
yaml.Unmarshal(mixedref, &rootNode)
yaml.Unmarshal([]byte(yml), &rootNode)
index := index.NewSpecIndex(&rootNode)
resolver := NewResolver(index)
assert.NotNil(t, resolver)
err := resolver.Resolve()
assert.Len(t, err, 1)
assert.Equal(t, "cannot resolve reference `go home, I am drunk`, it's missing: $go home, I am drunk [18:11]", err[0].Error())
}
func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
var rootNode yaml.Node
yaml.Unmarshal(mixedref, &rootNode)
b := index.CreateOpenAPIIndexConfig()
idx := index.NewSpecIndexWithConfig(&rootNode, b)
resolver := NewResolver(idx)
assert.NotNil(t, resolver)
circ := resolver.Resolve()
assert.Len(t, circ, 10)
assert.Len(t, circ, 0)
assert.Equal(t, 5, resolver.GetIndexesVisited())
assert.Equal(t, 209, resolver.GetRelativesSeen())
assert.Equal(t, 35, resolver.GetJourneysTaken())
assert.Equal(t, 62, resolver.GetReferenceVisited())
}
func TestResolver_ResolveComponents_k8s(t *testing.T) {
@@ -211,7 +248,8 @@ func ExampleNewResolver() {
_ = yaml.Unmarshal(stripeBytes, &rootNode)
// create a new spec index (resolver depends on it)
index := index.NewSpecIndex(&rootNode)
indexConfig := index.CreateClosedAPIIndexConfig()
index := index.NewSpecIndexWithConfig(&rootNode, indexConfig)
// create a new resolver using the index.
resolver := NewResolver(index)