LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-07 04:20:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

resolver tests all operational

Browse Source 
time to start some cleanup.

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-10-18 12:01:06 -04:00
parent 51971762a9
commit 48c83ddb30
9 changed files with 960 additions and 855 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
datamodel/low/extraction_functions.go
View File

@@ -90,9 +90,9 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
}
// perform a search for the reference in the index
foundRefs := idx.SearchIndexForReference(rv)
if len(foundRefs) > 0 {
return utils.NodeAlias(foundRefs[0].Node), nil
foundRef := idx.SearchIndexForReference(rv)
if foundRef != nil {
return utils.NodeAlias(foundRef.Node), nil
}
// let's try something else to find our references.

34
index/extract_refs.go
View File

@@ -467,14 +467,14 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
}
}
}
if len(seenPath) > 0 {
seenPath = seenPath[:len(seenPath)-1]
}
//if len(seenPath) > 0 {
// seenPath = seenPath[:len(seenPath)-1]
//}
}
if len(seenPath) > 0 {
seenPath = seenPath[:len(seenPath)-1]
}
//if len(seenPath) > 0 {
// seenPath = seenPath[:len(seenPath)-1]
//}
index.refCount = len(index.allRefs)
@@ -487,7 +487,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
var found []*Reference
// run this async because when things get recursive, it can take a while
//c := make(chan bool)
c := make(chan bool)
locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
located := index.FindComponent(ref.FullDefinition, ref.Node)
@@ -532,7 +532,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
index.refErrors = append(index.refErrors, indexError)
index.errorLock.Unlock()
}
//c <- true
c <- true
}
var refsToCheck []*Reference
@@ -556,17 +556,17 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
for r := range refsToCheck {
// expand our index of all mapped refs
//go locate(refsToCheck[r], r, mappedRefsInSequence)
locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
go locate(refsToCheck[r], r, mappedRefsInSequence)
//locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
}
//completedRefs := 0
//for completedRefs < len(refsToCheck) {
// select {
// case <-c:
// completedRefs++
// }
//}
completedRefs := 0
for completedRefs < len(refsToCheck) {
select {
case <-c:
completedRefs++
}
}
for m := range mappedRefsInSequence {
if mappedRefsInSequence[m] != nil {
index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])

42
index/resolver.go
View File

@@ -7,6 +7,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"strings"
)
// ResolvingError represents an issue the resolver had trying to stitch the tree together.
@@ -215,7 +216,6 @@ func visitIndex(res *Resolver, idx *SpecIndex) {
mapped := idx.GetMappedReferencesSequenced()
mappedIndex := idx.GetMappedReferences()
res.indexesVisited++
for _, ref := range mapped {
seenReferences := make(map[string]bool)
var journey []*Reference
@@ -244,9 +244,6 @@ func visitIndex(res *Resolver, idx *SpecIndex) {
}
}
}
//for _, c := range idx.GetChildren() {
// visitIndex(res, c)
//}
}
// VisitReference will visit a reference as part of a journey and will return resolved nodes.
@@ -269,9 +266,9 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
if j.Definition == r.Definition {
var foundDup *Reference
foundRefs := resolver.specIndex.SearchIndexForReferenceByReference(r)
if len(foundRefs) > 0 {
foundDup = foundRefs[0]
foundRef := resolver.specIndex.SearchIndexForReferenceByReference(r)
if foundRef != nil {
foundDup = foundRef
}
var circRef *CircularReferenceResult
@@ -295,10 +292,8 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
}
if resolver.IgnoreArray && isArray {
fmt.Printf("Ignored: %s\n", circRef.GenerateJourneyPath())
resolver.ignoredArrayReferences = append(resolver.ignoredArrayReferences, circRef)
} else {
fmt.Printf("Not Ignored: %s\n", circRef.GenerateJourneyPath())
resolver.circularReferences = append(resolver.circularReferences, circRef)
}
@@ -311,9 +306,9 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
if !skip {
var original *Reference
foundRefs := resolver.specIndex.SearchIndexForReferenceByReference(r)
if len(foundRefs) > 0 {
original = foundRefs[0]
foundRef := resolver.specIndex.SearchIndexForReferenceByReference(r)
if foundRef != nil {
original = foundRef
}
resolved := resolver.VisitReference(original, seen, journey, resolve)
if resolve && !original.Circular {
@@ -408,10 +403,23 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
}
value := node.Content[i+1].Value
var locatedRef []*Reference
var locatedRef *Reference
var fullDef string
exp := strings.Split(ref.FullDefinition, "#/")
if len(exp) == 2 {
if exp[0] != "" {
fullDef = fmt.Sprintf("%s%s", exp[0], value)
} else {
fullDef = value
}
} else {
fullDef = value
}
searchRef := &Reference{
Definition: value,
FullDefinition: ref.FullDefinition,
FullDefinition: fullDef,
RemoteLocation: ref.RemoteLocation,
IsRemote: true,
}
@@ -451,10 +459,8 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
}
}
locatedRef[0].ParentNodeSchemaType = schemaType
found = append(found, locatedRef[0])
locatedRef.ParentNodeSchemaType = schemaType
found = append(found, locatedRef)
foundRelatives[value] = true
}

97
index/resolver_test.go
View File

@@ -5,9 +5,9 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/utils"
"net/http"
"net/url"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
@@ -24,27 +24,15 @@ func Benchmark_ResolveDocumentStripe(b *testing.B) {
var rootNode yaml.Node
_ = yaml.Unmarshal(resolveFile, &rootNode)
fileFS, err := NewLocalFS(baseDir, os.DirFS(filepath.Dir(baseDir)))
for n := 0; n < b.N; n++ {
if err != nil {
b.Fatal(err)
}
cf := CreateOpenAPIIndexConfig()
cf.AvoidBuildIndex = true
rolo := NewRolodex(cf)
rolo.SetRootNode(&rootNode)
cf.Rolodex = rolo
// TODO: pick up here.
rolo.AddLocalFS(baseDir, fileFS)
indexedErr := rolo.IndexTheRolodex()
assert.Error(b, indexedErr)
assert.Len(b, utils.UnwrapErrors(indexedErr), 3)
}
}
@@ -407,20 +395,16 @@ func TestResolver_ResolveComponents_Stripe(t *testing.T) {
resolveFile, _ := os.ReadFile(baseDir)
var stripeRoot yaml.Node
_ = yaml.Unmarshal(resolveFile, &stripeRoot)
info, _ := datamodel.ExtractSpecInfoWithDocumentCheck(resolveFile, true)
fileFS, err := NewLocalFS(baseDir, os.DirFS(filepath.Dir(baseDir)))
if err != nil {
t.Fatal(err)
}
cf := CreateOpenAPIIndexConfig()
//cf.AvoidBuildIndex = true
cf.SpecInfo = info
rolo := NewRolodex(cf)
cf.Rolodex = rolo
rolo.AddLocalFS(baseDir, fileFS)
rolo := NewRolodex(cf)
rolo.SetRootNode(&stripeRoot)
indexedErr := rolo.IndexTheRolodex()
@@ -545,20 +529,67 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
var rootNode yaml.Node
_ = yaml.Unmarshal(mixedref, &rootNode)
b := CreateOpenAPIIndexConfig()
idx := NewSpecIndexWithConfig(&rootNode, b)
// create a test server.
server := test_buildMixedRefServer()
defer server.Close()
resolver := NewResolver(idx)
assert.NotNil(t, resolver)
// create a new config that allows local and remote to be mixed up.
cf := CreateOpenAPIIndexConfig()
cf.AvoidBuildIndex = true
cf.AllowRemoteLookup = true
cf.AvoidCircularReferenceCheck = true
cf.BasePath = "../test_specs"
circ := resolver.Resolve()
assert.Len(t, circ, 0)
assert.Equal(t, 5, resolver.GetIndexesVisited())
// setting this baseURL will override the base
cf.BaseURL, _ = url.Parse(server.URL)
// create a new rolodex
rolo := NewRolodex(cf)
// set the rolodex root node to the root node of the spec.
rolo.SetRootNode(&rootNode)
// create a new remote fs and set the config for indexing.
remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
remoteFS.SetIndexConfig(cf)
// set our remote handler func
c := http.Client{}
remoteFS.RemoteHandlerFunc = c.Get
// configure the local filesystem.
fsCfg := LocalFSConfig{
BaseDirectory: cf.BasePath,
FileFilters: []string{"burgershop.openapi.yaml"},
DirFS: os.DirFS(cf.BasePath),
}
// create a new local filesystem.
fileFS, err := NewLocalFSWithConfig(&fsCfg)
assert.NoError(t, err)
// add file systems to the rolodex
rolo.AddLocalFS(cf.BasePath, fileFS)
rolo.AddRemoteFS(server.URL, remoteFS)
// index the rolodex.
indexedErr := rolo.IndexTheRolodex()
assert.NoError(t, indexedErr)
rolo.Resolve()
index := rolo.GetRootIndex
resolver := index().GetResolver()
assert.Len(t, resolver.GetCircularErrors(), 0)
assert.Equal(t, 3, resolver.GetIndexesVisited())
// in v0.8.2 a new check was added when indexing, to prevent re-indexing the same file multiple times.
assert.Equal(t, 191, resolver.GetRelativesSeen())
assert.Equal(t, 35, resolver.GetJourneysTaken())
assert.Equal(t, 62, resolver.GetReferenceVisited())
assert.Equal(t, 6, resolver.GetRelativesSeen())
assert.Equal(t, 5, resolver.GetJourneysTaken())
assert.Equal(t, 7, resolver.GetReferenceVisited())
}
func TestResolver_ResolveComponents_k8s(t *testing.T) {

6
index/rolodex.go
View File

@@ -356,7 +356,7 @@ func (r *Rolodex) IndexTheRolodex() error {
// now that we have indexed all the files, we can build the index.
r.indexes = indexBuildQueue
if !r.indexConfig.AvoidBuildIndex {
//if !r.indexConfig.AvoidBuildIndex {
for _, idx := range indexBuildQueue {
idx.BuildIndex()
if r.indexConfig.AvoidCircularReferenceCheck {
@@ -367,7 +367,7 @@ func (r *Rolodex) IndexTheRolodex() error {
caughtErrors = append(caughtErrors, errs[e])
}
}
}
//}
// indexed and built every supporting file, we can build the root index (our entry point)
@@ -453,7 +453,7 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) {
var remoteFile *RemoteFile
if r == nil || r.localFS == nil && r.remoteFS == nil {
panic("WHAT NO....")
return nil, fmt.Errorf("rolodex has no file systems configured, cannot open '%s'", location)
}
fileLookup := location

26
index/search_index.go
View File

@@ -9,7 +9,7 @@ import (
"strings"
)
func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) []*Reference {
func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *Reference {
ref := fullRef.FullDefinition
@@ -34,28 +34,24 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) [
}
if r, ok := index.allMappedRefs[ref]; ok {
return []*Reference{r}
}
if r, ok := index.allRefs[ref]; ok {
return []*Reference{r}
return r
}
// check the rolodex for the reference.
if roloLookup != "" {
rFile, err := index.rolodex.Open(roloLookup)
if err != nil {
return nil
}
// extract the index from the rolodex file.
idx := rFile.GetIndex()
index.resolver.indexesVisited++
if idx != nil {
// check mapped refs.
if r, ok := idx.allMappedRefs[ref]; ok {
return []*Reference{r}
}
if r, ok := index.allRefs[ref]; ok {
return []*Reference{r}
return r
}
// build a collection of all the inline schemas and search them
@@ -66,7 +62,7 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) [
d = append(d, idx.allInlineSchemaObjectDefinitions...)
for _, s := range d {
if s.Definition == ref {
return []*Reference{s}
return s
}
}
}
@@ -79,10 +75,6 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) [
// SearchIndexForReference searches the index for a reference, first looking through the mapped references
// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes
// extracted when parsing the OpenAPI Spec.
func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref})
}
func (index *SpecIndex) SearchIndexForReferenceWithParent(ref string, reference *Reference) []*Reference {
func (index *SpecIndex) SearchIndexForReference(ref string) *Reference {
return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref})
}

102
index/spec_index_test.go
View File

@@ -95,8 +95,8 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification")
index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
BaseURL: baseURL,
//AllowRemoteLookup: true,
//AllowFileLookup: true,
AllowRemoteLookup: true,
AllowFileLookup: true,
})
assert.Len(t, index.GetAllExternalIndexes(), 291)
@@ -127,15 +127,15 @@ func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) {
ref := index.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml")
assert.NotNil(t, ref)
assert.Equal(t, "operationId", ref[0].Node.Content[0].Value)
assert.Equal(t, "operationId", ref.Node.Content[0].Value)
ref = index.SearchIndexForReference("examples/ruby/domains_create.yml")
assert.NotNil(t, ref)
assert.Equal(t, "lang", ref[0].Node.Content[0].Value)
assert.Equal(t, "lang", ref.Node.Content[0].Value)
ref = index.SearchIndexForReference("../../shared/responses/server_error.yml")
assert.NotNil(t, ref)
assert.Equal(t, "description", ref[0].Node.Content[0].Value)
assert.Equal(t, "description", ref.Node.Content[0].Value)
ref = index.SearchIndexForReference("../models/options.yml")
assert.NotNil(t, ref)
@@ -463,7 +463,7 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
cf.BasePath = "../test_specs"
// setting this baseURL will override the base
//cf.BaseURL, _ = url.Parse(server.URL)
cf.BaseURL, _ = url.Parse(server.URL)
cFile := "../test_specs/mixedref-burgershop.openapi.yaml"
yml, _ := os.ReadFile(cFile)
@@ -477,8 +477,7 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
rolo.SetRootNode(&rootNode)
// create a new remote fs and set the config for indexing.
//remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
remoteFS, _ := NewRemoteFS()
remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
remoteFS.SetIndexConfig(cf)
// set our remote handler func
@@ -529,9 +528,9 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
}
func TestSpecIndex_TestEmptyBrokenReferences(t *testing.T) {
asana, _ := os.ReadFile("../test_specs/badref-burgershop.openapi.yaml")
badref, _ := os.ReadFile("../test_specs/badref-burgershop.openapi.yaml")
var rootNode yaml.Node
_ = yaml.Unmarshal(asana, &rootNode)
_ = yaml.Unmarshal(badref, &rootNode)
index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig())
assert.Equal(t, 5, index.GetPathCount())
@@ -545,7 +544,7 @@ func TestSpecIndex_TestEmptyBrokenReferences(t *testing.T) {
assert.Equal(t, 2, index.GetOperationsParameterCount())
assert.Equal(t, 1, index.GetInlineDuplicateParamCount())
assert.Equal(t, 1, index.GetInlineUniqueParamCount())
assert.Len(t, index.refErrors, 7)
assert.Len(t, index.refErrors, 5)
}
func TestTagsNoDescription(t *testing.T) {
@@ -852,18 +851,53 @@ func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) {
}
func TestSpecIndex_lookupFileReference(t *testing.T) {
_ = os.WriteFile("fox.yaml", []byte("good:\n - puppy: dog\n - puppy: forever-more"), 0o664)
pup := []byte("good:\n - puppy: dog\n - puppy: forever-more")
var myPuppy yaml.Node
_ = yaml.Unmarshal(pup, &myPuppy)
_ = os.WriteFile("fox.yaml", pup, 0o664)
defer os.Remove("fox.yaml")
index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
index.seenRemoteSources = make(map[string]*yaml.Node)
k, doc, err := index.lookupFileReference("fox.yaml#/good")
// create a new config that allows local and remote to be mixed up.
cf := CreateOpenAPIIndexConfig()
cf.AvoidBuildIndex = true
cf.AllowRemoteLookup = true
cf.AvoidCircularReferenceCheck = true
cf.BasePath = "."
// create a new rolodex
rolo := NewRolodex(cf)
// set the rolodex root node to the root node of the spec.
rolo.SetRootNode(&myPuppy)
// configure the local filesystem.
fsCfg := LocalFSConfig{
BaseDirectory: cf.BasePath,
FileFilters: []string{"fox.yaml"},
DirFS: os.DirFS(cf.BasePath),
}
// create a new local filesystem.
fileFS, err := NewLocalFSWithConfig(&fsCfg)
assert.NoError(t, err)
assert.NotNil(t, doc)
assert.NotNil(t, k)
rolo.AddLocalFS(cf.BasePath, fileFS)
rErr := rolo.IndexTheRolodex()
assert.NoError(t, rErr)
fox, fErr := rolo.Open("fox.yaml")
assert.NoError(t, fErr)
assert.Equal(t, "fox.yaml", fox.Name())
assert.Equal(t, "good:\n - puppy: dog\n - puppy: forever-more", string(fox.GetContent()))
}
func TestSpecIndex_parameterReferencesHavePaths(t *testing.T) {
_ = os.WriteFile("paramour.yaml", []byte(`components:
parameters:
param3:
@@ -873,6 +907,13 @@ func TestSpecIndex_parameterReferencesHavePaths(t *testing.T) {
type: string`), 0o664)
defer os.Remove("paramour.yaml")
// create a new config that allows local and remote to be mixed up.
cf := CreateOpenAPIIndexConfig()
cf.AvoidBuildIndex = true
cf.AllowRemoteLookup = true
cf.AvoidCircularReferenceCheck = true
cf.BasePath = "."
yml := `paths:
/:
parameters:
@@ -903,7 +944,32 @@ components:
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &rootNode)
index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig())
// create a new rolodex
rolo := NewRolodex(cf)
// set the rolodex root node to the root node of the spec.
rolo.SetRootNode(&rootNode)
// configure the local filesystem.
fsCfg := LocalFSConfig{
BaseDirectory: cf.BasePath,
FileFilters: []string{"paramour.yaml"},
DirFS: os.DirFS(cf.BasePath),
}
// create a new local filesystem.
fileFS, err := NewLocalFSWithConfig(&fsCfg)
assert.NoError(t, err)
// add file system
rolo.AddLocalFS(cf.BasePath, fileFS)
// index the rolodex.
indexedErr := rolo.IndexTheRolodex()
assert.NoError(t, indexedErr)
rolo.BuildIndexes()
index := rolo.GetRootIndex()
params := index.GetAllParametersFromOperations()

10
index/utility_methods.go
View File

@@ -22,7 +22,10 @@ func (index *SpecIndex) extractDefinitionsAndSchemas(schemasNode *yaml.Node, pat
}
def := fmt.Sprintf("%s%s", pathPrefix, name)
fullDef := fmt.Sprintf("%s%s", index.specAbsolutePath, def)
ref := &Reference{
FullDefinition: fullDef,
Definition: def,
Name: name,
Node: schema,
@@ -278,6 +281,13 @@ func (index *SpecIndex) scanOperationParams(params []*yaml.Node, pathItemNode *y
paramRefName := param.Content[1].Value
paramRef := index.allMappedRefs[paramRefName]
if paramRef == nil {
// could be in the rolodex
ref := index.SearchIndexForReference(paramRefName)
if ref != nil {
paramRef = ref
}
}
if index.paramOpRefs[pathItemNode.Value] == nil {
index.paramOpRefs[pathItemNode.Value] = make(map[string]map[string][]*Reference)