LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-06 12:37:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

a first working engine of the new design.

Browse Source 
There is a horrible amount of work to be done to clean this up, and wire in remote support. but so far, this is working as expected and is now a much cleaner design, (once everything has been cleaned up that is)

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-10-16 13:36:30 -04:00
parent 8b795c6321
commit d5f72a2a2e
12 changed files with 775 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
datamodel/high/v3/document.go
View File

@@ -89,7 +89,11 @@ type Document struct {
//
// This property is not a part of the OpenAPI schema, this is custom to libopenapi.
Index *index.SpecIndex `json:"-" yaml:"-"`
low *low.Document
// Rolodex is the low-level rolodex used when creating this document.
// This in an internal structure and not part of the OpenAPI schema.
Rolodex *index.Rolodex `json:"-" yaml:"-"`
low *low.Document
}
// NewDocument will create a new high-level Document from a low-level one.

8
datamodel/low/v2/swagger.go
View File

@@ -142,10 +142,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
// build an index
idx := index.NewSpecIndexWithConfig(info.RootNode, &index.SpecIndexConfig{
BaseURL: config.BaseURL,
RemoteURLHandler: config.RemoteURLHandler,
AllowRemoteLookup: config.AllowRemoteReferences,
AllowFileLookup: config.AllowFileReferences,
BaseURL: config.BaseURL,
RemoteURLHandler: config.RemoteURLHandler,
//AllowRemoteLookup: config.AllowRemoteReferences,
//AllowFileLookup: config.AllowFileReferences,
})
doc.Index = idx
doc.SpecInfo = info

107
datamodel/low/v3/create_document.go
View File

@@ -3,13 +3,13 @@ package v3
import (
"errors"
"os"
"path/filepath"
"sync"
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/datamodel/low/base"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/resolver"
"github.com/pb33f/libopenapi/utils"
)
@@ -17,7 +17,7 @@ import (
//
// Deprecated: Use CreateDocumentFromConfig instead. This function will be removed in a later version, it
// defaults to allowing file and remote references, and does not support relative file references.
func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) {
func CreateDocument(info *datamodel.SpecInfo) (*Document, error) {
config := datamodel.DocumentConfiguration{
AllowFileReferences: true,
AllowRemoteReferences: true,
@@ -26,61 +26,92 @@ func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) {
}
// CreateDocumentFromConfig Create a new document from the provided SpecInfo and DocumentConfiguration pointer.
func CreateDocumentFromConfig(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, []error) {
func CreateDocumentFromConfig(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, error) {
return createDocument(info, config)
}
func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, []error) {
func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, error) {
_, labelNode, versionNode := utils.FindKeyNodeFull(OpenAPILabel, info.RootNode.Content)
var version low.NodeReference[string]
if versionNode == nil {
return nil, []error{errors.New("no openapi version/tag found, cannot create document")}
return nil, errors.New("no openapi version/tag found, cannot create document")
}
version = low.NodeReference[string]{Value: versionNode.Value, KeyNode: labelNode, ValueNode: versionNode}
doc := Document{Version: version}
// get current working directory as a basePath
cwd, _ := os.Getwd()
// If basePath is provided override it
if config.BasePath != "" {
cwd = config.BasePath
}
// build an index
idx := index.NewSpecIndexWithConfig(info.RootNode, &index.SpecIndexConfig{
BaseURL: config.BaseURL,
RemoteURLHandler: config.RemoteURLHandler,
BasePath: cwd,
AllowFileLookup: config.AllowFileReferences,
AllowRemoteLookup: config.AllowRemoteReferences,
AvoidBuildIndex: config.AvoidIndexBuild,
SpecInfo: info,
})
doc.Index = idx
// TODO: configure allowFileReferences and allowRemoteReferences stuff
// create an index config and shadow the document configuration.
idxConfig := index.CreateOpenAPIIndexConfig()
idxConfig.SpecInfo = info
idxConfig.BasePath = cwd
idxConfig.IgnoreArrayCircularReferences = config.IgnoreArrayCircularReferences
idxConfig.IgnorePolymorphicCircularReferences = config.IgnorePolymorphicCircularReferences
idxConfig.AvoidCircularReferenceCheck = config.SkipCircularReferenceCheck
rolodex := index.NewRolodex(idxConfig)
doc.Rolodex = rolodex
// If basePath is provided override it
if config.BasePath != "" {
var absError error
cwd, absError = filepath.Abs(config.BasePath)
if absError != nil {
return nil, absError
}
// create a local filesystem
fileFS, err := index.NewLocalFS(cwd, os.DirFS(cwd))
if err != nil {
return nil, err
}
// add the filesystem to the rolodex
rolodex.AddLocalFS(cwd, fileFS)
}
// TODO: Remote filesystem
// index the rolodex
err := rolodex.IndexTheRolodex()
var errs []error
if err != nil {
errs = append(errs, rolodex.GetCaughtErrors()...)
}
errs = idx.GetReferenceIndexErrors()
doc.Index = rolodex.GetRootIndex()
//errs = idx.GetReferenceIndexErrors()
// create resolver and check for circular references.
resolve := resolver.NewResolver(idx)
// if configured, ignore circular references in arrays and polymorphic schemas
if config.IgnoreArrayCircularReferences {
resolve.IgnoreArrayCircularReferences()
}
if config.IgnorePolymorphicCircularReferences {
resolve.IgnorePolymorphicCircularReferences()
}
// check for circular references.
resolvingErrors := resolve.CheckForCircularReferences()
if len(resolvingErrors) > 0 {
for r := range resolvingErrors {
errs = append(errs, resolvingErrors[r])
}
}
//resolve := resolver.NewResolver(idx)
//
//// if configured, ignore circular references in arrays and polymorphic schemas
//if config.IgnoreArrayCircularReferences {
// resolve.IgnoreArrayCircularReferences()
//}
//if config.IgnorePolymorphicCircularReferences {
// resolve.IgnorePolymorphicCircularReferences()
//}
//
//if !config.AvoidIndexBuild {
// // check for circular references.
// resolvingErrors := resolve.CheckForCircularReferences()
//
// if len(resolvingErrors) > 0 {
// for r := range resolvingErrors {
// errs = append(errs, resolvingErrors[r])
// }
// }
//}
var wg sync.WaitGroup
@@ -117,10 +148,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
wg.Add(len(extractionFuncs))
for _, f := range extractionFuncs {
go runExtraction(info, &doc, idx, f, &errs, &wg)
go runExtraction(info, &doc, rolodex.GetRootIndex(), f, &errs, &wg)
}
wg.Wait()
return &doc, errs
return &doc, errors.Join(errs...)
}
func extractInfo(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error {

3
datamodel/low/v3/document.go
View File

@@ -82,6 +82,9 @@ type Document struct {
//
// This property is not a part of the OpenAPI schema, this is custom to libopenapi.
Index *index.SpecIndex
// Rolodex is a reference to the rolodex used when creating this document.
Rolodex *index.Rolodex
}
// FindSecurityRequirement will attempt to locate a security requirement string from a supplied name.

71
index/extract_refs.go
View File

@@ -6,6 +6,7 @@ package index
import (
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/pb33f/libopenapi/utils"
@@ -157,11 +158,31 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
segs := strings.Split(value, "/")
name := segs[len(segs)-1]
_, p := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
// determine absolute path to this definition
iroot := filepath.Dir(index.specAbsolutePath)
uri := strings.Split(value, "#/")
var componentName string
var fullDefinitionPath string
if len(uri) == 2 {
if uri[0] == "" {
fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1])
} else {
abs, _ := filepath.Abs(filepath.Join(iroot, uri[0]))
fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1])
}
componentName = fmt.Sprintf("#/%s", uri[1])
} else {
fullDefinitionPath = fmt.Sprintf("%s#/%s", iroot, uri[0])
componentName = fmt.Sprintf("#/%s", uri[0])
}
ref := &Reference{
Definition: value,
Name: name,
Node: node,
Path: p,
FullDefinition: fullDefinitionPath,
Definition: componentName,
Name: name,
Node: node,
Path: p,
}
// add to raw sequenced refs
@@ -183,10 +204,11 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
if len(node.Content) > 2 {
copiedNode := *node
copied := Reference{
Definition: ref.Definition,
Name: ref.Name,
Node: &copiedNode,
Path: p,
FullDefinition: fullDefinitionPath,
Definition: ref.Definition,
Name: ref.Name,
Node: &copiedNode,
Path: p,
}
// protect this data using a copy, prevent the resolver from destroying things.
index.refsWithSiblings[value] = copied
@@ -413,19 +435,22 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
var found []*Reference
// run this async because when things get recursive, it can take a while
c := make(chan bool)
//c := make(chan bool)
locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
located := index.FindComponent(ref.Definition, ref.Node)
located := index.FindComponent(ref.FullDefinition, ref.Node)
if located != nil {
index.refLock.Lock()
if index.allMappedRefs[ref.Definition] == nil {
found = append(found, located)
index.allMappedRefs[ref.Definition] = located
sequence[refIndex] = &ReferenceMapped{
Reference: located,
Definition: ref.Definition,
rm := &ReferenceMapped{
Reference: located,
Definition: ref.Definition,
FullDefinition: ref.FullDefinition,
}
sequence[refIndex] = rm
}
index.refLock.Unlock()
} else {
@@ -440,7 +465,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
index.refErrors = append(index.refErrors, indexError)
index.errorLock.Unlock()
}
c <- true
//c <- true
}
var refsToCheck []*Reference
@@ -464,17 +489,17 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
for r := range refsToCheck {
// expand our index of all mapped refs
go locate(refsToCheck[r], r, mappedRefsInSequence)
// locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
//go locate(refsToCheck[r], r, mappedRefsInSequence)
locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
}
completedRefs := 0
for completedRefs < len(refsToCheck) {
select {
case <-c:
completedRefs++
}
}
//completedRefs := 0
//for completedRefs < len(refsToCheck) {
// select {
// case <-c:
// completedRefs++
// }
//}
for m := range mappedRefsInSequence {
if mappedRefsInSequence[m] != nil {
index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])

734
index/find_component.go
View File

@@ -8,7 +8,6 @@ import (
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
@@ -26,64 +25,33 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
return nil
}
remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
if index.config.AllowRemoteLookup {
return index.lookupRemoteReference(id)
} else {
return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
"please set AllowRemoteLookup to true in the configuration")
}
}
//remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
// if index.config.AllowRemoteLookup {
// return index.lookupRemoteReference(id)
// } else {
// return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
// "please set AllowRemoteLookup to true in the configuration")
// }
//}
//
//fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
// if index.config.AllowFileLookup {
// return index.lookupFileReference(id)
// } else {
// return nil, nil, fmt.Errorf("local lookups are not permitted, " +
// "please set AllowFileLookup to true in the configuration")
// }
//}
fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
if index.config.AllowFileLookup {
return index.lookupFileReference(id)
} else {
return nil, nil, fmt.Errorf("local lookups are not permitted, " +
"please set AllowFileLookup to true in the configuration")
}
}
//witch DetermineReferenceResolveType(componentId) {
//case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case.
// return index.FindComponentInRoot(componentId)
switch DetermineReferenceResolveType(componentId) {
case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case.
return index.FindComponentInRoot(componentId)
//case HttpResolve, FileResolve:
return index.performExternalLookup(strings.Split(componentId, "#/"))
case HttpResolve:
uri := strings.Split(componentId, "#")
if len(uri) >= 2 {
return index.performExternalLookup(uri, componentId, remoteLookup, parent)
}
if len(uri) == 1 {
// if there is no reference, second segment is empty / has no name
// this means there is no component to look-up and the entire file should be pulled in.
// to stop all the other code from breaking (that is expecting a component), let's just post-pend
// a hash to the end of the componentId and ensure the uri slice is as expected.
// described in https://github.com/pb33f/libopenapi/issues/37
componentId = fmt.Sprintf("%s#", componentId)
uri = append(uri, "")
return index.performExternalLookup(uri, componentId, remoteLookup, parent)
}
case FileResolve:
uri := strings.Split(componentId, "#")
if len(uri) == 2 {
return index.performExternalLookup(uri, componentId, fileLookup, parent)
}
if len(uri) == 1 {
// if there is no reference, second segment is empty / has no name
// this means there is no component to look-up and the entire file should be pulled in.
// to stop all the other code from breaking (that is expecting a component), let's just post-pend
// a hash to the end of the componentId and ensure the uri slice is as expected.
// described in https://github.com/pb33f/libopenapi/issues/37
//
// ^^ this same issue was re-reported in file based lookups in vacuum.
// more info here: https://github.com/daveshanley/vacuum/issues/225
componentId = fmt.Sprintf("%s#", componentId)
uri = append(uri, "")
return index.performExternalLookup(uri, componentId, fileLookup, parent)
}
}
return nil
//}
//return nil
}
var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second}
@@ -107,106 +75,106 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
// split string to remove file reference
uri := strings.Split(ref, "#")
// have we already seen this remote source?
var parsedRemoteDocument *yaml.Node
alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
if alreadySeen {
parsedRemoteDocument = foundDocument
} else {
d := make(chan bool)
var body []byte
var err error
go func(uri string) {
bc := make(chan []byte)
ec := make(chan error)
var getter RemoteURLHandler = httpClient.Get
if index.config != nil && index.config.RemoteURLHandler != nil {
getter = index.config.RemoteURLHandler
}
// if we have a remote handler, use it instead of the default.
if index.config != nil && index.config.FSHandler != nil {
go func() {
remoteFS := index.config.FSHandler
remoteFile, rErr := remoteFS.Open(uri)
if rErr != nil {
e := fmt.Errorf("unable to open remote file: %s", rErr)
ec <- e
return
}
b, ioErr := io.ReadAll(remoteFile)
if ioErr != nil {
e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
ec <- e
return
}
bc <- b
}()
} else {
go getRemoteDoc(getter, uri, bc, ec)
}
select {
case v := <-bc:
body = v
break
case er := <-ec:
err = er
break
}
if len(body) > 0 {
var remoteDoc yaml.Node
er := yaml.Unmarshal(body, &remoteDoc)
if er != nil {
err = er
d <- true
return
}
parsedRemoteDocument = &remoteDoc
if index.config != nil {
index.config.seenRemoteSources.Store(uri, &remoteDoc)
}
}
d <- true
}(uri[0])
// wait for double go fun.
<-d
if err != nil {
// no bueno.
return nil, nil, err
}
}
// lookup item from reference by using a path query.
var query string
if len(uri) >= 2 {
query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
} else {
query = "$"
}
query, err := url.PathUnescape(query)
if err != nil {
return nil, nil, err
}
// remove any URL encoding
query = strings.Replace(query, "~1", "./", 1)
query = strings.ReplaceAll(query, "~1", "/")
path, err := yamlpath.NewPath(query)
if err != nil {
return nil, nil, err
}
result, _ := path.Find(parsedRemoteDocument)
if len(result) == 1 {
return result[0], parsedRemoteDocument, nil
}
//uri := strings.Split(ref, "#")
//
//// have we already seen this remote source?
//var parsedRemoteDocument *yaml.Node
//alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
//
//if alreadySeen {
// parsedRemoteDocument = foundDocument
//} else {
//
// d := make(chan bool)
// var body []byte
// var err error
//
// go func(uri string) {
// bc := make(chan []byte)
// ec := make(chan error)
// var getter = httpClient.Get
// if index.config != nil && index.config.RemoteURLHandler != nil {
// getter = index.config.RemoteURLHandler
// }
//
// // if we have a remote handler, use it instead of the default.
// if index.config != nil && index.config.FSHandler != nil {
// go func() {
// remoteFS := index.config.FSHandler
// remoteFile, rErr := remoteFS.Open(uri)
// if rErr != nil {
// e := fmt.Errorf("unable to open remote file: %s", rErr)
// ec <- e
// return
// }
// b, ioErr := io.ReadAll(remoteFile)
// if ioErr != nil {
// e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
// ec <- e
// return
// }
// bc <- b
// }()
// } else {
// go getRemoteDoc(getter, uri, bc, ec)
// }
// select {
// case v := <-bc:
// body = v
// break
// case er := <-ec:
// err = er
// break
// }
// if len(body) > 0 {
// var remoteDoc yaml.Node
// er := yaml.Unmarshal(body, &remoteDoc)
// if er != nil {
// err = er
// d <- true
// return
// }
// parsedRemoteDocument = &remoteDoc
// if index.config != nil {
// index.config.seenRemoteSources.Store(uri, &remoteDoc)
// }
// }
// d <- true
// }(uri[0])
//
// // wait for double go fun.
// <-d
// if err != nil {
// // no bueno.
// return nil, nil, err
// }
//}
//
//// lookup item from reference by using a path query.
//var query string
//if len(uri) >= 2 {
// query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
//} else {
// query = "$"
//}
//
//query, err := url.PathUnescape(query)
//if err != nil {
// return nil, nil, err
//}
//
//// remove any URL encoding
//query = strings.Replace(query, "~1", "./", 1)
//query = strings.ReplaceAll(query, "~1", "/")
//
//path, err := yamlpath.NewPath(query)
//if err != nil {
// return nil, nil, err
//}
//result, _ := path.Find(parsedRemoteDocument)
//if len(result) == 1 {
// return result[0], parsedRemoteDocument, nil
//}
return nil, nil, nil
}
@@ -214,73 +182,83 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
// split string to remove file reference
uri := strings.Split(ref, "#")
file := strings.ReplaceAll(uri[0], "file:", "")
filePath := filepath.Dir(file)
fileName := filepath.Base(file)
//filePath := filepath.Dir(file)
//fileName := filepath.Base(file)
absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file))
var parsedRemoteDocument *yaml.Node
if index.seenRemoteSources[file] != nil {
parsedRemoteDocument = index.seenRemoteSources[file]
} else {
base := index.config.BasePath
fileToRead := filepath.Join(base, filePath, fileName)
var body []byte
var err error
// if we have an FS handler, use it instead of the default behavior
if index.config != nil && index.config.FSHandler != nil {
remoteFS := index.config.FSHandler
remoteFile, rErr := remoteFS.Open(fileToRead)
if rErr != nil {
e := fmt.Errorf("unable to open file: %s", rErr)
return nil, nil, e
}
body, err = io.ReadAll(remoteFile)
if err != nil {
e := fmt.Errorf("unable to read file bytes: %s", err)
return nil, nil, e
}
} else {
// try and read the file off the local file system, if it fails
// check for a baseURL and then ask our remote lookup function to go try and get it.
body, err = os.ReadFile(fileToRead)
if err != nil {
// if we have a baseURL, then we can try and get the file from there.
if index.config != nil && index.config.BaseURL != nil {
u := index.config.BaseURL
remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
a, b, e := index.lookupRemoteReference(remoteRef)
if e != nil {
// give up, we can't find the file, not locally, not remotely. It's toast.
return nil, nil, e
}
return a, b, nil
} else {
// no baseURL? then we can't do anything, give up.
return nil, nil, err
}
}
}
var remoteDoc yaml.Node
err = yaml.Unmarshal(body, &remoteDoc)
if err != nil {
return nil, nil, err
}
parsedRemoteDocument = &remoteDoc
if index.seenLocalSources != nil {
index.sourceLock.Lock()
index.seenLocalSources[file] = &remoteDoc
index.sourceLock.Unlock()
}
// extract the document from the rolodex.
rFile, rError := index.rolodex.Open(absoluteFileLocation)
if rError != nil {
return nil, nil, rError
}
parsedDocument, err := rFile.GetContentAsYAMLNode()
if err != nil {
return nil, nil, err
}
//if index.seenRemoteSources[file] != nil {
// parsedDocument = index.seenRemoteSources[file]
//} else {
//
// base := index.config.BasePath
// fileToRead := filepath.Join(base, filePath, fileName)
// var body []byte
// var err error
//
// // if we have an FS handler, use it instead of the default behavior
// if index.config != nil && index.config.FSHandler != nil {
// remoteFS := index.config.FSHandler
// remoteFile, rErr := remoteFS.Open(fileToRead)
// if rErr != nil {
// e := fmt.Errorf("unable to open file: %s", rErr)
// return nil, nil, e
// }
// body, err = io.ReadAll(remoteFile)
// if err != nil {
// e := fmt.Errorf("unable to read file bytes: %s", err)
// return nil, nil, e
// }
//
// } else {
//
// // try and read the file off the local file system, if it fails
// // check for a baseURL and then ask our remote lookup function to go try and get it.
// body, err = os.ReadFile(fileToRead)
//
// if err != nil {
//
// // if we have a baseURL, then we can try and get the file from there.
// if index.config != nil && index.config.BaseURL != nil {
//
// u := index.config.BaseURL
// remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
// a, b, e := index.lookupRemoteReference(remoteRef)
// if e != nil {
// // give up, we can't find the file, not locally, not remotely. It's toast.
// return nil, nil, e
// }
// return a, b, nil
//
// } else {
// // no baseURL? then we can't do anything, give up.
// return nil, nil, err
// }
// }
// }
// var remoteDoc yaml.Node
// err = yaml.Unmarshal(body, &remoteDoc)
// if err != nil {
// return nil, nil, err
// }
// parsedDocument = &remoteDoc
// if index.seenLocalSources != nil {
// index.sourceLock.Lock()
// index.seenLocalSources[file] = &remoteDoc
// index.sourceLock.Unlock()
// }
//}
// lookup item from reference by using a path query.
var query string
if len(uri) >= 2 {
@@ -289,7 +267,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
query = "$"
}
query, err := url.PathUnescape(query)
query, err = url.PathUnescape(query)
if err != nil {
return nil, nil, err
}
@@ -302,154 +280,214 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
if err != nil {
return nil, nil, err
}
result, _ := path.Find(parsedRemoteDocument)
result, _ := path.Find(parsedDocument)
if len(result) == 1 {
return result[0], parsedRemoteDocument, nil
return result[0], parsedDocument, nil
}
return nil, parsedRemoteDocument, nil
return nil, parsedDocument, nil
}
func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference {
if index.root != nil {
func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference {
// check component for url encoding.
if strings.Contains(componentId, "%") {
// decode the url.
componentId, _ = url.QueryUnescape(componentId)
}
// check component for url encoding.
if strings.Contains(componentId, "%") {
// decode the url.
componentId, _ = url.QueryUnescape(componentId)
name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId)
path, err := yamlpath.NewPath(friendlySearch)
if path == nil || err != nil {
return nil // no component found
}
res, _ := path.Find(root)
if len(res) == 1 {
resNode := res[0]
if res[0].Kind == yaml.DocumentNode {
resNode = res[0].Content[0]
}
name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId)
path, err := yamlpath.NewPath(friendlySearch)
if path == nil || err != nil {
return nil // no component found
}
res, _ := path.Find(index.root)
fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId)
if len(res) == 1 {
resNode := res[0]
if res[0].Kind == yaml.DocumentNode {
resNode = res[0].Content[0]
}
ref := &Reference{
Definition: componentId,
Name: name,
Node: resNode,
Path: friendlySearch,
RequiredRefProperties: index.extractDefinitionRequiredRefProperties(res[0], map[string][]string{}),
}
// extract properties
return ref
ref := &Reference{
FullDefinition: fullDef,
Definition: componentId,
Name: name,
Node: resNode,
Path: friendlySearch,
RequiredRefProperties: extractDefinitionRequiredRefProperties(resNode, map[string][]string{}),
}
return ref
}
return nil
}
func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
lookupFunction ExternalLookupFunction, parent *yaml.Node) *Reference {
//func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference {
// if index.root != nil {
// return FindComponent(index.root, componentId, )
// }
// return nil
//}
func (index *SpecIndex) performExternalLookup(uri []string) *Reference {
if len(uri) > 0 {
index.externalLock.RLock()
externalSpecIndex := index.externalSpecIndex[uri[0]]
index.externalLock.RUnlock()
if externalSpecIndex == nil {
_, newRoot, err := lookupFunction(componentId)
if err != nil {
indexError := &IndexingError{
Err: err,
Node: parent,
Path: componentId,
}
index.errorLock.Lock()
index.refErrors = append(index.refErrors, indexError)
index.errorLock.Unlock()
return nil
}
// split string to remove file reference
file := strings.ReplaceAll(uri[0], "file:", "")
fileName := filepath.Base(file)
// cool, cool, lets index this spec also. This is a recursive action and will keep going
// until all remote references have been found.
var bp *url.URL
var bd string
if index.config.BaseURL != nil {
bp = index.config.BaseURL
}
if index.config.BasePath != "" {
bd = index.config.BasePath
}
var path, newBasePath string
var newUrl *url.URL
if bp != nil {
path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false)
newUrl, _ = url.Parse(path)
newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path)))
}
if bd != "" {
if len(uri[0]) > 0 {
// if there is no base url defined, but we can know we have been requested remotely,
// set the base url to the remote url base path.
// first check if the first param is actually a URL
io, er := url.ParseRequestURI(uri[0])
if er != nil {
newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
} else {
if newUrl == nil || newUrl.String() != io.String() {
newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path)))
}
newBasePath = filepath.Dir(filepath.Join(bd, uri[1]))
}
} else {
newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
}
}
if newUrl != nil || newBasePath != "" {
newConfig := &SpecIndexConfig{
BaseURL: newUrl,
BasePath: newBasePath,
AllowRemoteLookup: index.config.AllowRemoteLookup,
AllowFileLookup: index.config.AllowFileLookup,
ParentIndex: index,
seenRemoteSources: index.config.seenRemoteSources,
remoteLock: index.config.remoteLock,
uri: uri,
}
var newIndex *SpecIndex
seen := index.SearchAncestryForSeenURI(uri[0])
if seen == nil {
newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
index.refLock.Lock()
index.externalLock.Lock()
index.externalSpecIndex[uri[0]] = newIndex
index.externalLock.Unlock()
newIndex.relativePath = path
newIndex.parentIndex = index
index.AddChild(newIndex)
index.refLock.Unlock()
externalSpecIndex = newIndex
} else {
externalSpecIndex = seen
}
}
var absoluteFileLocation string
if filepath.IsAbs(file) {
absoluteFileLocation = file
} else {
absoluteFileLocation, _ = filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file))
}
if externalSpecIndex != nil {
foundRef := externalSpecIndex.FindComponentInRoot(uri[1])
// extract the document from the rolodex.
rFile, rError := index.rolodex.Open(absoluteFileLocation)
if rError != nil {
logger.Error("unable to open rolodex file", "file", absoluteFileLocation, "error", rError)
return nil
}
parsedDocument, err := rFile.GetContentAsYAMLNode()
if err != nil {
logger.Error("unable to parse rolodex file", "file", absoluteFileLocation, "error", err)
return nil
}
//fmt.Printf("parsedDocument: %v\n", parsedDocument)
//index.externalLock.RLock()
//externalSpecIndex := index.externalSpecIndex[uri[0]]
//index.externalLock.RUnlock()
//if externalSpecIndex == nil {
// _, newRoot, err := lookupFunction(componentId)
// if err != nil {
// indexError := &IndexingError{
// Err: err,
// Node: parent,
// Path: componentId,
// }
// index.errorLock.Lock()
// index.refErrors = append(index.refErrors, indexError)
// index.errorLock.Unlock()
// return nil
// }
//
// // cool, cool, lets index this spec also. This is a recursive action and will keep going
// // until all remote references have been found.
// var bp *url.URL
// var bd string
//
// if index.config.BaseURL != nil {
// bp = index.config.BaseURL
// }
// if index.config.BasePath != "" {
// bd = index.config.BasePath
// }
//
// var path, newBasePath string
// var newUrl *url.URL
//
// if bp != nil {
// path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false)
// newUrl, _ = url.Parse(path)
// newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path)))
// }
// if bd != "" {
// if len(uri[0]) > 0 {
// // if there is no base url defined, but we can know we have been requested remotely,
// // set the base url to the remote url base path.
// // first check if the first param is actually a URL
// io, er := url.ParseRequestURI(uri[0])
// if er != nil {
// newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
// } else {
// if newUrl == nil || newUrl.String() != io.String() {
// newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path)))
// }
// newBasePath = filepath.Dir(filepath.Join(bd, uri[1]))
// }
// } else {
// newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
// }
// }
//
// if newUrl != nil || newBasePath != "" {
// newConfig := &SpecIndexConfig{
// BaseURL: newUrl,
// BasePath: newBasePath,
// AllowRemoteLookup: index.config.AllowRemoteLookup,
// AllowFileLookup: index.config.AllowFileLookup,
// ParentIndex: index,
// seenRemoteSources: index.config.seenRemoteSources,
// remoteLock: index.config.remoteLock,
// uri: uri,
// AvoidBuildIndex: index.config.AvoidBuildIndex,
// }
//
// var newIndex *SpecIndex
// seen := index.SearchAncestryForSeenURI(uri[0])
// if seen == nil {
//
// newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
// index.refLock.Lock()
// index.externalLock.Lock()
// index.externalSpecIndex[uri[0]] = newIndex
// index.externalLock.Unlock()
// newIndex.relativePath = path
// newIndex.parentIndex = index
// index.AddChild(newIndex)
// index.refLock.Unlock()
// externalSpecIndex = newIndex
// } else {
// externalSpecIndex = seen
// }
// }
//}
wholeFile := false
query := ""
if len(uri) < 2 {
wholeFile = true
} else {
query = fmt.Sprintf("#/%s", strings.Replace(uri[1], "~1", "./", 1))
query = strings.ReplaceAll(query, "~1", "/")
}
// check if there is a component we want to suck in, or if the
// entire root needs to come in.
var foundRef *Reference
if wholeFile {
if parsedDocument.Kind == yaml.DocumentNode {
parsedDocument = parsedDocument.Content[0]
}
// TODO: remote locations
foundRef = &Reference{
Definition: fileName,
Name: fileName,
Node: parsedDocument,
IsRemote: true,
RemoteLocation: absoluteFileLocation,
Path: "$",
RequiredRefProperties: extractDefinitionRequiredRefProperties(parsedDocument, map[string][]string{}),
}
return foundRef
} else {
foundRef = FindComponent(parsedDocument, query, absoluteFileLocation)
if foundRef != nil {
nameSegs := strings.Split(uri[1], "/")
ref := &Reference{
Definition: componentId,
Name: nameSegs[len(nameSegs)-1],
Node: foundRef.Node,
IsRemote: true,
RemoteLocation: componentId,
Path: foundRef.Path,
}
return ref
foundRef.IsRemote = true
foundRef.RemoteLocation = absoluteFileLocation
return foundRef
}
}
}

69
index/find_component_test.go
View File

@@ -44,8 +44,6 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) {
c.BasePath = "../test_specs"
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Nil(t, index.uri)
assert.NotNil(t, index.children[0].uri)
assert.NotNil(t, index.children[0].children[0].uri)
assert.NotNil(t, index.SearchIndexForReference("second.yaml#/properties/property2"))
assert.NotNil(t, index.SearchIndexForReference("second.yaml"))
assert.Nil(t, index.SearchIndexForReference("fourth.yaml"))
@@ -504,3 +502,70 @@ paths:
assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_Complex_Local_File_Design(t *testing.T) {
main := `openapi: 3.1.0
paths:
/anything/circularReference:
get:
operationId: circularReferenceGet
responses:
"200":
description: OK
content:
application/json:
schema:
$ref: "components.yaml#/components/schemas/validCircularReferenceObject"
/anything/oneOfCircularReference:
get:
operationId: oneOfCircularReferenceGet
tags:
- generation
responses:
"200":
description: OK
content:
application/json:
schema:
$ref: "components.yaml#/components/schemas/oneOfCircularReferenceObject"`
components := `components:
schemas:
validCircularReferenceObject:
type: object
properties:
circular:
type: array
items:
$ref: "#/components/schemas/validCircularReferenceObject"
oneOfCircularReferenceObject:
type: object
properties:
child:
oneOf:
- $ref: "#/components/schemas/oneOfCircularReferenceObject"
- $ref: "#/components/schemas/simpleObject"
required:
- child
simpleObject:
description: "simple"
type: object
properties:
str:
type: string
description: "A string property."
example: "example" `
_ = os.WriteFile("components.yaml", []byte(components), 0644)
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(main), &rootNode)
c := CreateOpenAPIIndexConfig()
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}

124
index/index_model.go
View File

@@ -11,7 +11,6 @@ import (
"os"
"sync"
"golang.org/x/sync/syncmap"
"gopkg.in/yaml.v3"
)
@@ -25,15 +24,17 @@ const (
// Reference is a wrapper around *yaml.Node results to make things more manageable when performing
// algorithms on data models. the *yaml.Node def is just a bit too low level for tracking state.
type Reference struct {
Definition string
Name string
Node *yaml.Node
ParentNode *yaml.Node
ParentNodeSchemaType string // used to determine if the parent node is an array or not.
Resolved bool
Circular bool
Seen bool
IsRemote bool
FullDefinition string
Definition string
Name string
Node *yaml.Node
ParentNode *yaml.Node
ParentNodeSchemaType string // used to determine if the parent node is an array or not.
Resolved bool
Circular bool
Seen bool
IsRemote bool
//FileLocation string
RemoteLocation string
Path string // this won't always be available.
RequiredRefProperties map[string][]string // definition names (eg, #/definitions/One) to a list of required properties on this definition which reference that definition
@@ -41,8 +42,9 @@ type Reference struct {
// ReferenceMapped is a helper struct for mapped references put into sequence (we lose the key)
type ReferenceMapped struct {
Reference *Reference
Definition string
Reference *Reference
Definition string
FullDefinition string
}
// SpecIndexConfig is a configuration struct for the SpecIndex introduced in 0.6.0 that provides an expandable
@@ -67,6 +69,7 @@ type SpecIndexConfig struct {
// If resolving remotely, the RemoteURLHandler will be used to fetch the remote document.
// If not set, the default http client will be used.
// Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132
// deprecated: Use the Rolodex instead
RemoteURLHandler func(url string) (*http.Response, error)
// FSHandler is an entity that implements the `fs.FS` interface that will be used to fetch local or remote documents.
@@ -81,6 +84,7 @@ type SpecIndexConfig struct {
// it also overrides the RemoteURLHandler if set.
//
// Resolves[#85] https://github.com/pb33f/libopenapi/issues/85
// deprecated: Use the Rolodex instead
FSHandler fs.FS
// If resolving locally, the BasePath will be the root from which relative references will be resolved from
@@ -92,12 +96,13 @@ type SpecIndexConfig struct {
// exploits, but it's better to be safe than sorry.
//
// To read more about this, you can find a discussion here: https://github.com/pb33f/libopenapi/pull/64
AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
AllowFileLookup bool // Allow file lookups for references. Defaults to false
//AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
//AllowFileLookup bool // Allow file lookups for references. Defaults to false
// ParentIndex allows the index to be created with knowledge of a parent, before being parsed. This allows
// a breakglass to be used to prevent loops, checking the tree before recursing down.
ParentIndex *SpecIndex
// a breakglass to be used to prevent loops, checking the tree before cursing down.
// deprecated: Use the Rolodex instead, this is no longer needed, indexes are finite and do not have children.
//ParentIndex *SpecIndex
// If set to true, the index will not be built out, which means only the foundational elements will be
// parsed and added to the index. This is useful to avoid building out an index if the specification is
@@ -106,14 +111,40 @@ type SpecIndexConfig struct {
// Use the `BuildIndex()` method on the index to build it out once resolved/ready.
AvoidBuildIndex bool
AvoidCircularReferenceCheck bool
// SpecInfo is a pointer to the SpecInfo struct that contains the root node and the spec version. It's the
// struct that was used to create this index.
SpecInfo *datamodel.SpecInfo
// Rolodex is what provides all file and remote based lookups. Without the rolodex, no remote or file lookups
// can be used. Normally you won't need to worry about setting this as each root document gets a rolodex
// of its own automatically.
Rolodex *Rolodex
SpecAbsolutePath string
// IgnorePolymorphicCircularReferences will skip over checking for circular references in polymorphic schemas.
// A polymorphic schema is any schema that is composed other schemas using references via `oneOf`, `anyOf` of `allOf`.
// This is disabled by default, which means polymorphic circular references will be checked.
IgnorePolymorphicCircularReferences bool
// IgnoreArrayCircularReferences will skip over checking for circular references in arrays. Sometimes a circular
// reference is required to describe a data-shape correctly. Often those shapes are valid circles if the
// type of the schema implementing the loop is an array. An empty array would technically break the loop.
// So if libopenapi is returning circular references for this use case, then this option should be enabled.
// this is disabled by default, which means array circular references will be checked.
IgnoreArrayCircularReferences bool
// SkipCircularReferenceCheck will skip over checking for circular references. This is disabled by default, which
// means circular references will be checked. This is useful for developers building out models that should be
// indexed later on.
//SkipCircularReferenceCheck bool
// private fields
seenRemoteSources *syncmap.Map
remoteLock *sync.Mutex
uri []string
//seenRemoteSources *syncmap.Map
//remoteLock *sync.Mutex
uri []string
}
// CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
@@ -123,10 +154,10 @@ type SpecIndexConfig struct {
func CreateOpenAPIIndexConfig() *SpecIndexConfig {
cw, _ := os.Getwd()
return &SpecIndexConfig{
BasePath: cw,
AllowRemoteLookup: true,
AllowFileLookup: true,
seenRemoteSources: &syncmap.Map{},
BasePath: cw,
//AllowRemoteLookup: true,
//AllowFileLookup: true,
//seenRemoteSources: &syncmap.Map{},
}
}
@@ -137,10 +168,10 @@ func CreateOpenAPIIndexConfig() *SpecIndexConfig {
func CreateClosedAPIIndexConfig() *SpecIndexConfig {
cw, _ := os.Getwd()
return &SpecIndexConfig{
BasePath: cw,
AllowRemoteLookup: false,
AllowFileLookup: false,
seenRemoteSources: &syncmap.Map{},
BasePath: cw,
//AllowRemoteLookup: false,
//AllowFileLookup: false,
//seenRemoteSources: &syncmap.Map{},
}
}
@@ -148,6 +179,7 @@ func CreateClosedAPIIndexConfig() *SpecIndexConfig {
// quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules,
// everything is pre-walked if you need it.
type SpecIndex struct {
rolodex *Rolodex // the rolodex is used to fetch remote and file based documents.
allRefs map[string]*Reference // all (deduplicated) refs
rawSequencedRefs []*Reference // all raw references in sequence as they are scanned, not deduped.
linesWithRefs map[int]bool // lines that link to references.
@@ -183,7 +215,8 @@ type SpecIndex struct {
rootSecurity []*Reference // root security definitions.
rootSecurityNode *yaml.Node // root security node.
refsWithSiblings map[string]Reference // references with sibling elements next to them
pathRefsLock sync.Mutex // create lock for all refs maps, we want to build data as fast as we can
pathRefsLock sync.RWMutex // create lock for all refs maps, we want to build data as fast as we can
operationLock sync.Mutex // create lock for operations
externalDocumentsCount int // number of externalDocument nodes found
operationTagsCount int // number of unique tags in operations
globalTagsCount int // number of global tags defined
@@ -248,12 +281,17 @@ type SpecIndex struct {
componentIndexChan chan bool
polyComponentIndexChan chan bool
// when things get complex (looking at you digital ocean) then we need to know
// what we have seen across indexes, so we need to be able to travel back up to the root
// cto avoid re-downloading sources.
parentIndex *SpecIndex
uri []string
children []*SpecIndex
specAbsolutePath string
resolver *Resolver
//parentIndex *SpecIndex
uri []string
//children []*SpecIndex
}
// GetResolver returns the resolver for this index.
func (index *SpecIndex) GetResolver() *Resolver {
return index.resolver
}
// GetConfig returns the SpecIndexConfig for this index.
@@ -261,15 +299,15 @@ func (index *SpecIndex) GetConfig() *SpecIndexConfig {
return index.config
}
// AddChild adds a child index to this index, a child index is an index created from a remote or file reference.
func (index *SpecIndex) AddChild(child *SpecIndex) {
index.children = append(index.children, child)
}
// GetChildren returns the children of this index.
func (index *SpecIndex) GetChildren() []*SpecIndex {
return index.children
}
//// AddChild adds a child index to this index, a child index is an index created from a remote or file reference.
//func (index *SpecIndex) AddChild(child *SpecIndex) {
// index.children = append(index.children, child)
//}
//
//// GetChildren returns the children of this index.
//func (index *SpecIndex) GetChildren() []*SpecIndex {
// return index.children
//}
// ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the
// URI based document. Decides if the reference is local, remote or in a file.

30
index/index_model_test.go
View File

@@ -8,21 +8,21 @@ import (
"testing"
)
func TestSpecIndex_Children(t *testing.T) {
idx1 := new(SpecIndex)
idx2 := new(SpecIndex)
idx3 := new(SpecIndex)
idx4 := new(SpecIndex)
idx5 := new(SpecIndex)
idx1.AddChild(idx2)
idx1.AddChild(idx3)
idx3.AddChild(idx4)
idx4.AddChild(idx5)
assert.Equal(t, 2, len(idx1.GetChildren()))
assert.Equal(t, 1, len(idx3.GetChildren()))
assert.Equal(t, 1, len(idx4.GetChildren()))
assert.Equal(t, 0, len(idx5.GetChildren()))
}
//func TestSpecIndex_Children(t *testing.T) {
// idx1 := new(SpecIndex)
// idx2 := new(SpecIndex)
// idx3 := new(SpecIndex)
// idx4 := new(SpecIndex)
// idx5 := new(SpecIndex)
// idx1.AddChild(idx2)
// idx1.AddChild(idx3)
// idx3.AddChild(idx4)
// idx4.AddChild(idx5)
// assert.Equal(t, 2, len(idx1.GetChildren()))
// assert.Equal(t, 1, len(idx3.GetChildren()))
// assert.Equal(t, 1, len(idx4.GetChildren()))
// assert.Equal(t, 0, len(idx5.GetChildren()))
//}
func TestSpecIndex_GetConfig(t *testing.T) {
idx1 := new(SpecIndex)

69
index/resolver.go
View File

@@ -4,11 +4,9 @@
package index
import (
"fmt"
"strings"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"fmt"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
)
// ResolvingError represents an issue the resolver had trying to stitch the tree together.
@@ -259,7 +257,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
}
journey = append(journey, ref)
relatives := resolver.extractRelatives(ref.Node, nil, seen, journey, resolve)
relatives := resolver.extractRelatives(ref, ref.Node, nil, seen, journey, resolve)
seen = make(map[string]bool)
@@ -362,7 +360,7 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDe
return false, visitedDefinitions
}
func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.Node,
foundRelatives map[string]bool,
journey []*Reference, resolve bool) []*Reference {
@@ -400,7 +398,7 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
// }
//}
found = append(found, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...)
found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, resolve)...)
}
if i%2 == 0 && n.Value == "$ref" {
@@ -411,9 +409,9 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
value := node.Content[i+1].Value
ref := resolver.specIndex.SearchIndexForReference(value)
locatedRef := resolver.specIndex.SearchIndexForReference(value)
if ref == nil {
if locatedRef == nil {
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
err := &ResolvingError{
ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value),
@@ -434,15 +432,9 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
}
}
r := &Reference{
Definition: value,
Name: value,
Node: node,
ParentNode: parent,
ParentNodeSchemaType: schemaType,
}
locatedRef[0].ParentNodeSchemaType = schemaType
found = append(found, r)
found = append(found, locatedRef[0])
foundRelatives[value] = true
}
@@ -459,30 +451,30 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
if _, v := utils.FindKeyNodeTop("items", node.Content[i+1].Content); v != nil {
if utils.IsNodeMap(v) {
if d, _, l := utils.IsNodeRefValue(v); d {
ref := resolver.specIndex.GetMappedReferences()[l]
if ref != nil && !ref.Circular {
mappedRefs := resolver.specIndex.GetMappedReferences()[l]
if mappedRefs != nil && !mappedRefs.Circular {
circ := false
for f := range journey {
if journey[f].Definition == ref.Definition {
if journey[f].Definition == mappedRefs.Definition {
circ = true
break
}
}
if !circ {
resolver.VisitReference(ref, foundRelatives, journey, resolve)
resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve)
} else {
loop := append(journey, ref)
loop := append(journey, mappedRefs)
circRef := &CircularReferenceResult{
Journey: loop,
Start: ref,
Start: mappedRefs,
LoopIndex: i,
LoopPoint: ref,
LoopPoint: mappedRefs,
PolymorphicType: n.Value,
IsPolymorphicResult: true,
}
ref.Seen = true
ref.Circular = true
mappedRefs.Seen = true
mappedRefs.Circular = true
if resolver.IgnorePoly {
resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef)
} else {
@@ -501,36 +493,31 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
v := node.Content[i+1].Content[q]
if utils.IsNodeMap(v) {
if d, _, l := utils.IsNodeRefValue(v); d {
strangs := strings.Split(l, "/#")
if len(strangs) == 2 {
fmt.Println("wank")
}
ref := resolver.specIndex.GetMappedReferences()[l]
if ref != nil && !ref.Circular {
mappedRefs := resolver.specIndex.GetMappedReferences()[l]
if mappedRefs != nil && !mappedRefs.Circular {
circ := false
for f := range journey {
if journey[f].Definition == ref.Definition {
if journey[f].Definition == mappedRefs.Definition {
circ = true
break
}
}
if !circ {
resolver.VisitReference(ref, foundRelatives, journey, resolve)
resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve)
} else {
loop := append(journey, ref)
loop := append(journey, mappedRefs)
circRef := &CircularReferenceResult{
Journey: loop,
Start: ref,
Start: mappedRefs,
LoopIndex: i,
LoopPoint: ref,
LoopPoint: mappedRefs,
PolymorphicType: n.Value,
IsPolymorphicResult: true,
}
ref.Seen = true
ref.Circular = true
mappedRefs.Seen = true
mappedRefs.Circular = true
if resolver.IgnorePoly {
resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef)
} else {

65
index/rolodex.go
View File

@@ -51,17 +51,18 @@ type RolodexFS interface {
}
type Rolodex struct {
localFS map[string]fs.FS
remoteFS map[string]fs.FS
indexed bool
built bool
resolved bool
circChecked bool
indexConfig *SpecIndexConfig
indexingDuration time.Duration
indexes []*SpecIndex
rootIndex *SpecIndex
caughtErrors []error
localFS map[string]fs.FS
remoteFS map[string]fs.FS
indexed bool
built bool
resolved bool
circChecked bool
indexConfig *SpecIndexConfig
indexingDuration time.Duration
indexes []*SpecIndex
rootIndex *SpecIndex
caughtErrors []error
ignoredCircularReferences []*CircularReferenceResult
}
type rolodexFile struct {
@@ -209,6 +210,10 @@ func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex {
return r
}
func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult {
return r.ignoredCircularReferences
}
func (r *Rolodex) GetIndexingDuration() time.Duration {
return r.indexingDuration
}
@@ -275,10 +280,12 @@ func (r *Rolodex) IndexTheRolodex() error {
if copiedConfig.IgnorePolymorphicCircularReferences {
resolver.IgnorePolymorphicCircularReferences()
}
resolvingErrors := resolver.CheckForCircularReferences()
for e := range resolvingErrors {
caughtErrors = append(caughtErrors, resolvingErrors[e])
}
//if !copiedConfig.AvoidCircularReferenceCheck {
// resolvingErrors := resolver.CheckForCircularReferences()
// for e := range resolvingErrors {
// caughtErrors = append(caughtErrors, resolvingErrors[e])
// }
//}
if err != nil {
errChan <- err
@@ -331,6 +338,13 @@ func (r *Rolodex) IndexTheRolodex() error {
if !r.indexConfig.AvoidBuildIndex {
for _, idx := range indexBuildQueue {
idx.BuildIndex()
if r.indexConfig.AvoidCircularReferenceCheck {
continue
}
errs := idx.resolver.CheckForCircularReferences()
for e := range errs {
caughtErrors = append(caughtErrors, errs[e])
}
}
}
@@ -369,6 +383,27 @@ func (r *Rolodex) CheckForCircularReferences() {
for e := range resolvingErrors {
r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
}
if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
}
if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
}
}
}
func (r *Rolodex) Resolve() {
if r.rootIndex != nil && r.rootIndex.resolver != nil {
resolvingErrors := r.rootIndex.resolver.Resolve()
for e := range resolvingErrors {
r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
}
if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
}
if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
}
}
}

20
index/search_index.go
View File

@@ -3,21 +3,39 @@
package index
import (
"fmt"
"path/filepath"
"strings"
)
// SearchIndexForReference searches the index for a reference, first looking through the mapped references
// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes
// extracted when parsing the OpenAPI Spec.
func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
absPath := index.specAbsolutePath
var roloLookup string
uri := strings.Split(ref, "#/")
if len(uri) == 2 {
if uri[0] != "" {
roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0]))
}
ref = fmt.Sprintf("#/%s", uri[1])
}
if r, ok := index.allMappedRefs[ref]; ok {
return []*Reference{r}
}
// TODO: look in the rolodex.
panic("should not be here")
fmt.Println(roloLookup)
return nil
//if r, ok := index.allMappedRefs[ref]; ok {
// return []*Reference{r}
// return []*Reference{r}jh
//}
//for c := range index.children {
// found := goFindMeSomething(index.children[c], ref)