LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-09 12:37:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Working through fix #73

Browse Source 
Quite a bit of surgery required.
This commit is contained in:
Dave Shanley
2023-02-17 16:31:34 -05:00
parent 1df5f44e6f
commit e19b7d2bf1
12 changed files with 347 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
datamodel/low/extraction_functions.go
View File

@@ -89,6 +89,11 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
}
}
foundRefs := idx.SearchIndexForReference(rv)
if len(foundRefs) > 0 {
return foundRefs[0].Node, nil
}
// cant be found? last resort is to try a path lookup
_, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv)
if friendly != "" {

82
index/extract_references.go
View File

@@ -1,82 +0,0 @@
// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT
package index
import (
"fmt"
"github.com/pb33f/libopenapi/utils"
"strings"
)
// ExtractComponentsFromRefs returns located components from references. The returned nodes from here
// can be used for resolving as they contain the actual object properties.
func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference {
var found []*Reference
//run this async because when things get recursive, it can take a while
c := make(chan bool)
locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
located := index.FindComponent(ref.Definition, ref.Node)
if located != nil {
index.refLock.Lock()
if index.allMappedRefs[ref.Definition] == nil {
found = append(found, located)
index.allMappedRefs[ref.Definition] = located
sequence[refIndex] = &ReferenceMapped{
Reference: located,
Definition: ref.Definition,
}
}
index.refLock.Unlock()
} else {
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
indexError := &IndexingError{
Err: fmt.Errorf("component '%s' does not exist in the specification", ref.Definition),
Node: ref.Node,
Path: path,
}
index.refErrors = append(index.refErrors, indexError)
}
c <- true
}
var refsToCheck []*Reference
for _, ref := range refs {
// check reference for backslashes (hah yeah seen this too!)
if strings.Contains(ref.Definition, "\\") { // this was from blazemeter.com haha!
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
indexError := &IndexingError{
Err: fmt.Errorf("component '%s' contains a backslash '\\'. It's not valid", ref.Definition),
Node: ref.Node,
Path: path,
}
index.refErrors = append(index.refErrors, indexError)
continue
}
refsToCheck = append(refsToCheck, ref)
}
mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
for r := range refsToCheck {
// expand our index of all mapped refs
go locate(refsToCheck[r], r, mappedRefsInSequence)
}
completedRefs := 0
for completedRefs < len(refsToCheck) {
select {
case <-c:
completedRefs++
}
}
for m := range mappedRefsInSequence {
if mappedRefsInSequence[m] != nil {
index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])
}
}
return found
}

88
index/extract_refs.go
View File

@@ -318,3 +318,91 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
return found
}
// ExtractComponentsFromRefs returns located components from references. The returned nodes from here
// can be used for resolving as they contain the actual object properties.
func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference {
var found []*Reference
//run this async because when things get recursive, it can take a while
//c := make(chan bool)
//tm := make(chan bool)
locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
located := index.FindComponent(ref.Definition, ref.Node)
if located != nil {
index.refLock.Lock()
if index.allMappedRefs[ref.Definition] == nil {
found = append(found, located)
index.allMappedRefs[ref.Definition] = located
sequence[refIndex] = &ReferenceMapped{
Reference: located,
Definition: ref.Definition,
}
}
index.refLock.Unlock()
} else {
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
indexError := &IndexingError{
Err: fmt.Errorf("component '%s' does not exist in the specification", ref.Definition),
Node: ref.Node,
Path: path,
}
index.refErrors = append(index.refErrors, indexError)
}
//c <- true
}
var refsToCheck []*Reference
for _, ref := range refs {
// check reference for backslashes (hah yeah seen this too!)
if strings.Contains(ref.Definition, "\\") { // this was from blazemeter.com haha!
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
indexError := &IndexingError{
Err: fmt.Errorf("component '%s' contains a backslash '\\'. It's not valid", ref.Definition),
Node: ref.Node,
Path: path,
}
index.refErrors = append(index.refErrors, indexError)
continue
}
refsToCheck = append(refsToCheck, ref)
}
mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
//go func() {
// time.Sleep(20 * time.Second)
// tm <- true
//}()
for r := range refsToCheck {
// expand our index of all mapped refs
// go locate(refsToCheck[r], r, mappedRefsInSequence)
locate(refsToCheck[r], r, mappedRefsInSequence)
}
//completedRefs := 0
//for completedRefs < len(refsToCheck) {
// select {
// case <-c:
// completedRefs++
// if completedRefs >= len(refsToCheck) {
// fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck))
// break
// } else {
// //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck))
// }
// //case <-tm:
// // panic("OH NO")
// }
//}
for m := range mappedRefsInSequence {
if mappedRefsInSequence[m] != nil {
index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])
}
}
return found
}

79
index/find_component.go
View File

@@ -9,7 +9,9 @@ import (
"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
"gopkg.in/yaml.v3"
"io/ioutil"
"net/url"
"strings"
"sync"
)
// FindComponent will locate a component by its reference, returns nil if nothing is found.
@@ -80,17 +82,27 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
return nil
}
var n sync.Mutex
var openCalls = 0
var closedCalls = 0
var totalCalls = 0
func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
// split string to remove file reference
uri := strings.Split(ref, "#")
// have we already seen this remote source?
var parsedRemoteDocument *yaml.Node
if index.seenRemoteSources[uri[0]] != nil {
parsedRemoteDocument = index.seenRemoteSources[uri[0]]
alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
if alreadySeen {
parsedRemoteDocument = foundDocument
} else {
index.httpLock.Lock()
resp, err := index.httpClient.Get(uri[0])
index.httpLock.Unlock()
totalCalls++
fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls)
if err != nil {
return nil, nil, err
}
@@ -105,9 +117,12 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
return nil, nil, err
}
parsedRemoteDocument = &remoteDoc
index.remoteLock.Lock()
index.seenRemoteSources[uri[0]] = &remoteDoc
index.remoteLock.Unlock()
//n.Lock()
//index.seenRemoteSources[uri[0]] = &remoteDoc
if index.config != nil {
index.config.seenRemoteSources[uri[0]] = &remoteDoc
}
//n.Unlock()
}
// lookup item from reference by using a path query.
@@ -158,17 +173,25 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
if index.config != nil && index.config.BaseURL != nil {
u := index.config.BaseURL
remoteRef := fmt.Sprintf("%s://%s%s/%s", u.Scheme, u.Host, u.Path, ref)
remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
a, b, e := index.lookupRemoteReference(remoteRef)
if e != nil {
// give up, we can't find the file, not locally, not remotely. It's toast.
return nil, nil, e
}
// everything looks good, lets just make sure we also add a key to the raw reference name.
if _, ok := index.seenRemoteSources[file]; !ok {
index.seenRemoteSources[file] = b
}
//// everything looks good, lets just make sure we also add a key to the raw reference name.
//if _, ok := index.seenRemoteSources[file]; !ok {
// if b != nil {
// //index.seenRemoteSources[ref] = b
// } else {
// panic("oh now")
// }
//
//} else {
// panic("oh no")
//}
return a, b, nil
@@ -184,7 +207,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
return nil, nil, err
}
parsedRemoteDocument = &remoteDoc
index.seenRemoteSources[file] = &remoteDoc
index.seenLocalSources[file] = &remoteDoc
}
// lookup item from reference by using a path query.
@@ -238,7 +261,9 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
lookupFunction ExternalLookupFunction, parent *yaml.Node,
) *Reference {
if len(uri) > 0 {
//index.fileLock.Lock()
externalSpecIndex := index.externalSpecIndex[uri[0]]
if externalSpecIndex == nil {
_, newRoot, err := lookupFunction(componentId)
if err != nil {
@@ -253,11 +278,33 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
// cool, cool, lets index this spec also. This is a recursive action and will keep going
// until all remote references have been found.
newIndex := NewSpecIndexWithConfig(newRoot, index.config)
index.fileLock.Lock()
// TODO: start here tomorrow, need to pass in the config to the new spec index.
// set the base URL to the path.
path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false)
newUrl, e := url.Parse(path)
if e == nil {
newConfig := &SpecIndexConfig{
BaseURL: newUrl,
AllowRemoteLookup: index.config.AllowRemoteLookup,
AllowFileLookup: index.config.AllowFileLookup,
seenRemoteSources: index.config.seenRemoteSources,
remoteLock: index.config.remoteLock,
}
var newIndex *SpecIndex
newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
index.externalSpecIndex[uri[0]] = newIndex
index.fileLock.Unlock()
newIndex.relativePath = path
newIndex.parentIndex = index
index.AddChild(newIndex)
externalSpecIndex = newIndex
} else {
return nil
}
}
foundRef := externalSpecIndex.FindComponentInRoot(uri[1])

25
index/index_model.go
View File

@@ -66,6 +66,14 @@ type SpecIndexConfig struct {
// To read more about this, you can find a discussion here: https://github.com/pb33f/libopenapi/pull/64
AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
AllowFileLookup bool // Allow file lookups for references. Defaults to false
// Indexes can be deeply nested, depending on how complex each spec is. If creating a sub-index
// this will help lookups remain efficient.
RootIndex *SpecIndex // if this is a sub-index, the root knows about everything below.
ParentIndex *SpecIndex // Who owns this index?
seenRemoteSources map[string]*yaml.Node
remoteLock *sync.Mutex
}
// SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and
@@ -166,16 +174,29 @@ type SpecIndex struct {
descriptionCount int
summaryCount int
seenRemoteSources map[string]*yaml.Node
remoteLock sync.Mutex
httpLock sync.Mutex
seenLocalSources map[string]*yaml.Node
remoteLock sync.RWMutex
httpLock sync.RWMutex
fileLock sync.Mutex
refLock sync.Mutex
circularReferences []*CircularReferenceResult // only available when the resolver has been used.
allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false.
relativePath string // relative path of the spec file.
config *SpecIndexConfig // configuration for the index
httpClient *http.Client
componentIndexChan chan bool
polyComponentIndexChan chan bool
// when things get complex (looking at you digital ocean) then we need to know
// what we have seen across indexes, so we need to be able to travel back up to the root
// cto avoid re-downloading sources.
rootIndex *SpecIndex
parentIndex *SpecIndex
children []*SpecIndex
}
func (index *SpecIndex) AddChild(child *SpecIndex) {
index.children = append(index.children, child)
}
// ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the

1
index/index_utils.go
View File

@@ -83,6 +83,7 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) {
index.polymorphicRefs = make(map[string]*Reference)
index.refsWithSiblings = make(map[string]Reference)
index.seenRemoteSources = make(map[string]*yaml.Node)
index.seenLocalSources = make(map[string]*yaml.Node)
index.opServersRefs = make(map[string]map[string][]*Reference)
index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second}
index.componentIndexChan = make(chan bool)

18
index/search_index.go Normal file
View File

@@ -0,0 +1,18 @@
// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT
package index
func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
if r, ok := index.allMappedRefs[ref]; ok {
return []*Reference{r}
}
for c := range index.children {
found := index.children[c].SearchIndexForReference(ref)
if found != nil {
return found
}
}
return nil
}

14
index/spec_index.go
View File

@@ -26,6 +26,10 @@ import (
// how the index is set up.
func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex {
index := new(SpecIndex)
if config != nil && config.seenRemoteSources == nil {
config.seenRemoteSources = make(map[string]*yaml.Node)
}
config.remoteLock = &sync.Mutex{}
index.config = config
boostrapIndexCollections(rootNode, index)
return createNewIndex(rootNode, index)
@@ -45,6 +49,7 @@ func NewSpecIndex(rootNode *yaml.Node) *SpecIndex {
index.config = &SpecIndexConfig{
AllowRemoteLookup: true,
AllowFileLookup: true,
seenRemoteSources: make(map[string]*yaml.Node),
}
boostrapIndexCollections(rootNode, index)
return createNewIndex(rootNode, index)
@@ -103,7 +108,7 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex {
index.GetInlineDuplicateParamCount()
index.GetAllDescriptionsCount()
index.GetTotalTagsCount()
index.seenRemoteSources = index.config.seenRemoteSources
return index
}
@@ -1132,3 +1137,10 @@ func (index *SpecIndex) GetAllDescriptionsCount() int {
func (index *SpecIndex) GetAllSummariesCount() int {
return len(index.allSummaries)
}
func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) {
if _, ok := index.config.seenRemoteSources[url]; ok {
return true, index.config.seenRemoteSources[url]
}
return false, nil
}

42
index/utility_methods.go
View File

@@ -7,6 +7,8 @@ import (
"fmt"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"net/url"
"strings"
"sync"
)
@@ -370,3 +372,43 @@ func runIndexFunction(funcs []func() int, wg *sync.WaitGroup) {
}(wg, cFunc)
}
}
func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bool) string {
cleanedPath := baseURL.Path // not cleaned yet!
// create a slice of path segments from existing path
pathSegs := strings.Split(cleanedPath, "/")
dirSegs := strings.Split(dir, "/")
var cleanedSegs []string
if !includeFile {
dirSegs = dirSegs[:len(dirSegs)-1]
}
// relative paths are a pain in the ass, damn you digital ocean, use a single spec, and break them
// down into services, please don't blast apart specs into a billion shards.
if strings.Contains(dir, "../") {
for s := range dirSegs {
if dirSegs[s] == ".." {
// chop off the last segment of the base path.
if len(pathSegs) > 0 {
pathSegs = pathSegs[:len(pathSegs)-1]
}
} else {
cleanedSegs = append(cleanedSegs, dirSegs[s])
}
}
cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/"))
} else {
cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/"))
}
p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
if strings.HasSuffix(p, "/") {
p = p[:len(p)-1]
}
return p
}

27
index/utility_methods_test.go Normal file
View File

@@ -0,0 +1,27 @@
// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT
package index
import (
"github.com/stretchr/testify/assert"
"net/url"
"testing"
)
func TestGenerateCleanSpecConfigBaseURL(t *testing.T) {
u, _ := url.Parse("https://pb33f.io/things/stuff")
path := "."
assert.Equal(t, "https://pb33f.io/things/stuff",
GenerateCleanSpecConfigBaseURL(u, path, false))
}
func TestGenerateCleanSpecConfigBaseURL_RelativeDeep(t *testing.T) {
u, _ := url.Parse("https://pb33f.io/things/stuff/jazz/cakes/winter/oil")
path := "../../../../foo/bar/baz/crap.yaml#thang"
assert.Equal(t, "https://pb33f.io/things/stuff/foo/bar/baz",
GenerateCleanSpecConfigBaseURL(u, path, false))
assert.Equal(t, "https://pb33f.io/things/stuff/foo/bar/baz/crap.yaml#thang",
GenerateCleanSpecConfigBaseURL(u, path, true))
}

19
resolver/resolver.go
View File

@@ -183,6 +183,10 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError {
// VisitReference will visit a reference as part of a journey and will return resolved nodes.
func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
if ref == nil {
panic("what?")
}
if ref.Resolved || ref.Seen {
return ref.Node.Content
}
@@ -198,7 +202,12 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b
skip := false
for i, j := range journey {
if j.Definition == r.Definition {
foundDup := resolver.specIndex.GetMappedReferences()[r.Definition]
var foundDup *index.Reference
foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition)
if len(foundRefs) > 0 {
foundDup = foundRefs[0]
}
var circRef *index.CircularReferenceResult
if !foundDup.Circular {
@@ -223,7 +232,11 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b
}
if !skip {
original := resolver.specIndex.GetMappedReferences()[r.Definition]
var original *index.Reference
foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition)
if len(foundRefs) > 0 {
original = foundRefs[0]
}
resolved := resolver.VisitReference(original, seen, journey, resolve)
if resolve {
r.Node.Content = resolved // this is where we perform the actual resolving.
@@ -292,7 +305,7 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
value := node.Content[i+1].Value
ref := resolver.specIndex.GetMappedReferences()[value]
ref := resolver.specIndex.SearchIndexForReference(value)
if ref == nil {
// TODO handle error, missing ref, can't resolve.