From d5f72a2a2e23cc3a437d6bd2ae2113a78eb99e5a Mon Sep 17 00:00:00 2001 From: quobix Date: Mon, 16 Oct 2023 13:36:30 -0400 Subject: [PATCH] a first working engine of the new design. There is a horrible amount of work to be done to clean this up, and wire in remote support. but so far, this is working as expected and is now a much cleaner design, (once everything has been cleaned up that is) Signed-off-by: quobix --- datamodel/high/v3/document.go | 6 +- datamodel/low/v2/swagger.go | 8 +- datamodel/low/v3/create_document.go | 107 ++-- datamodel/low/v3/document.go | 3 + index/extract_refs.go | 71 ++- index/find_component.go | 734 +++++++++++++++------------- index/find_component_test.go | 69 ++- index/index_model.go | 124 +++-- index/index_model_test.go | 30 +- index/resolver.go | 69 ++- index/rolodex.go | 65 ++- index/search_index.go | 20 +- 12 files changed, 775 insertions(+), 531 deletions(-) diff --git a/datamodel/high/v3/document.go b/datamodel/high/v3/document.go index e50c5e5..5df836d 100644 --- a/datamodel/high/v3/document.go +++ b/datamodel/high/v3/document.go @@ -89,7 +89,11 @@ type Document struct { // // This property is not a part of the OpenAPI schema, this is custom to libopenapi. Index *index.SpecIndex `json:"-" yaml:"-"` - low *low.Document + + // Rolodex is the low-level rolodex used when creating this document. + // This in an internal structure and not part of the OpenAPI schema. + Rolodex *index.Rolodex `json:"-" yaml:"-"` + low *low.Document } // NewDocument will create a new high-level Document from a low-level one. diff --git a/datamodel/low/v2/swagger.go b/datamodel/low/v2/swagger.go index 394977c..58fd015 100644 --- a/datamodel/low/v2/swagger.go +++ b/datamodel/low/v2/swagger.go @@ -142,10 +142,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur // build an index idx := index.NewSpecIndexWithConfig(info.RootNode, &index.SpecIndexConfig{ - BaseURL: config.BaseURL, - RemoteURLHandler: config.RemoteURLHandler, - AllowRemoteLookup: config.AllowRemoteReferences, - AllowFileLookup: config.AllowFileReferences, + BaseURL: config.BaseURL, + RemoteURLHandler: config.RemoteURLHandler, + //AllowRemoteLookup: config.AllowRemoteReferences, + //AllowFileLookup: config.AllowFileReferences, }) doc.Index = idx doc.SpecInfo = info diff --git a/datamodel/low/v3/create_document.go b/datamodel/low/v3/create_document.go index 6df45ec..14cfff6 100644 --- a/datamodel/low/v3/create_document.go +++ b/datamodel/low/v3/create_document.go @@ -3,13 +3,13 @@ package v3 import ( "errors" "os" + "path/filepath" "sync" "github.com/pb33f/libopenapi/datamodel" "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/datamodel/low/base" "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" "github.com/pb33f/libopenapi/utils" ) @@ -17,7 +17,7 @@ import ( // // Deprecated: Use CreateDocumentFromConfig instead. This function will be removed in a later version, it // defaults to allowing file and remote references, and does not support relative file references. -func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) { +func CreateDocument(info *datamodel.SpecInfo) (*Document, error) { config := datamodel.DocumentConfiguration{ AllowFileReferences: true, AllowRemoteReferences: true, @@ -26,61 +26,92 @@ func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) { } // CreateDocumentFromConfig Create a new document from the provided SpecInfo and DocumentConfiguration pointer. -func CreateDocumentFromConfig(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, []error) { +func CreateDocumentFromConfig(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, error) { return createDocument(info, config) } -func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, []error) { +func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, error) { _, labelNode, versionNode := utils.FindKeyNodeFull(OpenAPILabel, info.RootNode.Content) var version low.NodeReference[string] if versionNode == nil { - return nil, []error{errors.New("no openapi version/tag found, cannot create document")} + return nil, errors.New("no openapi version/tag found, cannot create document") } version = low.NodeReference[string]{Value: versionNode.Value, KeyNode: labelNode, ValueNode: versionNode} doc := Document{Version: version} // get current working directory as a basePath cwd, _ := os.Getwd() - - // If basePath is provided override it if config.BasePath != "" { cwd = config.BasePath } - // build an index - idx := index.NewSpecIndexWithConfig(info.RootNode, &index.SpecIndexConfig{ - BaseURL: config.BaseURL, - RemoteURLHandler: config.RemoteURLHandler, - BasePath: cwd, - AllowFileLookup: config.AllowFileReferences, - AllowRemoteLookup: config.AllowRemoteReferences, - AvoidBuildIndex: config.AvoidIndexBuild, - SpecInfo: info, - }) - doc.Index = idx + // TODO: configure allowFileReferences and allowRemoteReferences stuff + + // create an index config and shadow the document configuration. + idxConfig := index.CreateOpenAPIIndexConfig() + idxConfig.SpecInfo = info + idxConfig.BasePath = cwd + idxConfig.IgnoreArrayCircularReferences = config.IgnoreArrayCircularReferences + idxConfig.IgnorePolymorphicCircularReferences = config.IgnorePolymorphicCircularReferences + idxConfig.AvoidCircularReferenceCheck = config.SkipCircularReferenceCheck + + rolodex := index.NewRolodex(idxConfig) + doc.Rolodex = rolodex + + // If basePath is provided override it + if config.BasePath != "" { + var absError error + cwd, absError = filepath.Abs(config.BasePath) + if absError != nil { + return nil, absError + } + + // create a local filesystem + fileFS, err := index.NewLocalFS(cwd, os.DirFS(cwd)) + if err != nil { + return nil, err + } + + // add the filesystem to the rolodex + rolodex.AddLocalFS(cwd, fileFS) + + } + + // TODO: Remote filesystem + + // index the rolodex + err := rolodex.IndexTheRolodex() var errs []error + if err != nil { + errs = append(errs, rolodex.GetCaughtErrors()...) + } - errs = idx.GetReferenceIndexErrors() + doc.Index = rolodex.GetRootIndex() + + //errs = idx.GetReferenceIndexErrors() // create resolver and check for circular references. - resolve := resolver.NewResolver(idx) - // if configured, ignore circular references in arrays and polymorphic schemas - if config.IgnoreArrayCircularReferences { - resolve.IgnoreArrayCircularReferences() - } - if config.IgnorePolymorphicCircularReferences { - resolve.IgnorePolymorphicCircularReferences() - } - - // check for circular references. - resolvingErrors := resolve.CheckForCircularReferences() - - if len(resolvingErrors) > 0 { - for r := range resolvingErrors { - errs = append(errs, resolvingErrors[r]) - } - } + //resolve := resolver.NewResolver(idx) + // + //// if configured, ignore circular references in arrays and polymorphic schemas + //if config.IgnoreArrayCircularReferences { + // resolve.IgnoreArrayCircularReferences() + //} + //if config.IgnorePolymorphicCircularReferences { + // resolve.IgnorePolymorphicCircularReferences() + //} + // + //if !config.AvoidIndexBuild { + // // check for circular references. + // resolvingErrors := resolve.CheckForCircularReferences() + // + // if len(resolvingErrors) > 0 { + // for r := range resolvingErrors { + // errs = append(errs, resolvingErrors[r]) + // } + // } + //} var wg sync.WaitGroup @@ -117,10 +148,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur wg.Add(len(extractionFuncs)) for _, f := range extractionFuncs { - go runExtraction(info, &doc, idx, f, &errs, &wg) + go runExtraction(info, &doc, rolodex.GetRootIndex(), f, &errs, &wg) } wg.Wait() - return &doc, errs + return &doc, errors.Join(errs...) } func extractInfo(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { diff --git a/datamodel/low/v3/document.go b/datamodel/low/v3/document.go index cec319c..d90e302 100644 --- a/datamodel/low/v3/document.go +++ b/datamodel/low/v3/document.go @@ -82,6 +82,9 @@ type Document struct { // // This property is not a part of the OpenAPI schema, this is custom to libopenapi. Index *index.SpecIndex + + // Rolodex is a reference to the rolodex used when creating this document. + Rolodex *index.Rolodex } // FindSecurityRequirement will attempt to locate a security requirement string from a supplied name. diff --git a/index/extract_refs.go b/index/extract_refs.go index e5f6bf7..c67c503 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -6,6 +6,7 @@ package index import ( "errors" "fmt" + "path/filepath" "strings" "github.com/pb33f/libopenapi/utils" @@ -157,11 +158,31 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, segs := strings.Split(value, "/") name := segs[len(segs)-1] _, p := utils.ConvertComponentIdIntoFriendlyPathSearch(value) + + // determine absolute path to this definition + iroot := filepath.Dir(index.specAbsolutePath) + uri := strings.Split(value, "#/") + var componentName string + var fullDefinitionPath string + if len(uri) == 2 { + if uri[0] == "" { + fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1]) + } else { + abs, _ := filepath.Abs(filepath.Join(iroot, uri[0])) + fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) + } + componentName = fmt.Sprintf("#/%s", uri[1]) + } else { + fullDefinitionPath = fmt.Sprintf("%s#/%s", iroot, uri[0]) + componentName = fmt.Sprintf("#/%s", uri[0]) + } + ref := &Reference{ - Definition: value, - Name: name, - Node: node, - Path: p, + FullDefinition: fullDefinitionPath, + Definition: componentName, + Name: name, + Node: node, + Path: p, } // add to raw sequenced refs @@ -183,10 +204,11 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, if len(node.Content) > 2 { copiedNode := *node copied := Reference{ - Definition: ref.Definition, - Name: ref.Name, - Node: &copiedNode, - Path: p, + FullDefinition: fullDefinitionPath, + Definition: ref.Definition, + Name: ref.Name, + Node: &copiedNode, + Path: p, } // protect this data using a copy, prevent the resolver from destroying things. index.refsWithSiblings[value] = copied @@ -413,19 +435,22 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc var found []*Reference // run this async because when things get recursive, it can take a while - c := make(chan bool) + //c := make(chan bool) locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) { - located := index.FindComponent(ref.Definition, ref.Node) + located := index.FindComponent(ref.FullDefinition, ref.Node) if located != nil { index.refLock.Lock() if index.allMappedRefs[ref.Definition] == nil { found = append(found, located) index.allMappedRefs[ref.Definition] = located - sequence[refIndex] = &ReferenceMapped{ - Reference: located, - Definition: ref.Definition, + rm := &ReferenceMapped{ + Reference: located, + Definition: ref.Definition, + FullDefinition: ref.FullDefinition, } + + sequence[refIndex] = rm } index.refLock.Unlock() } else { @@ -440,7 +465,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc index.refErrors = append(index.refErrors, indexError) index.errorLock.Unlock() } - c <- true + //c <- true } var refsToCheck []*Reference @@ -464,17 +489,17 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc for r := range refsToCheck { // expand our index of all mapped refs - go locate(refsToCheck[r], r, mappedRefsInSequence) - // locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing. + //go locate(refsToCheck[r], r, mappedRefsInSequence) + locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing. } - completedRefs := 0 - for completedRefs < len(refsToCheck) { - select { - case <-c: - completedRefs++ - } - } + //completedRefs := 0 + //for completedRefs < len(refsToCheck) { + // select { + // case <-c: + // completedRefs++ + // } + //} for m := range mappedRefsInSequence { if mappedRefsInSequence[m] != nil { index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m]) diff --git a/index/find_component.go b/index/find_component.go index e3664cf..61c0229 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "net/url" - "os" "path/filepath" "strings" "time" @@ -26,64 +25,33 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re return nil } - remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) { - if index.config.AllowRemoteLookup { - return index.lookupRemoteReference(id) - } else { - return nil, nil, fmt.Errorf("remote lookups are not permitted, " + - "please set AllowRemoteLookup to true in the configuration") - } - } + //remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) { + // if index.config.AllowRemoteLookup { + // return index.lookupRemoteReference(id) + // } else { + // return nil, nil, fmt.Errorf("remote lookups are not permitted, " + + // "please set AllowRemoteLookup to true in the configuration") + // } + //} + // + //fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) { + // if index.config.AllowFileLookup { + // return index.lookupFileReference(id) + // } else { + // return nil, nil, fmt.Errorf("local lookups are not permitted, " + + // "please set AllowFileLookup to true in the configuration") + // } + //} - fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) { - if index.config.AllowFileLookup { - return index.lookupFileReference(id) - } else { - return nil, nil, fmt.Errorf("local lookups are not permitted, " + - "please set AllowFileLookup to true in the configuration") - } - } + //witch DetermineReferenceResolveType(componentId) { + //case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case. + // return index.FindComponentInRoot(componentId) - switch DetermineReferenceResolveType(componentId) { - case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case. - return index.FindComponentInRoot(componentId) + //case HttpResolve, FileResolve: + return index.performExternalLookup(strings.Split(componentId, "#/")) - case HttpResolve: - uri := strings.Split(componentId, "#") - if len(uri) >= 2 { - return index.performExternalLookup(uri, componentId, remoteLookup, parent) - } - if len(uri) == 1 { - // if there is no reference, second segment is empty / has no name - // this means there is no component to look-up and the entire file should be pulled in. - // to stop all the other code from breaking (that is expecting a component), let's just post-pend - // a hash to the end of the componentId and ensure the uri slice is as expected. - // described in https://github.com/pb33f/libopenapi/issues/37 - componentId = fmt.Sprintf("%s#", componentId) - uri = append(uri, "") - return index.performExternalLookup(uri, componentId, remoteLookup, parent) - } - - case FileResolve: - uri := strings.Split(componentId, "#") - if len(uri) == 2 { - return index.performExternalLookup(uri, componentId, fileLookup, parent) - } - if len(uri) == 1 { - // if there is no reference, second segment is empty / has no name - // this means there is no component to look-up and the entire file should be pulled in. - // to stop all the other code from breaking (that is expecting a component), let's just post-pend - // a hash to the end of the componentId and ensure the uri slice is as expected. - // described in https://github.com/pb33f/libopenapi/issues/37 - // - // ^^ this same issue was re-reported in file based lookups in vacuum. - // more info here: https://github.com/daveshanley/vacuum/issues/225 - componentId = fmt.Sprintf("%s#", componentId) - uri = append(uri, "") - return index.performExternalLookup(uri, componentId, fileLookup, parent) - } - } - return nil + //} + //return nil } var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second} @@ -107,106 +75,106 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) { func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { // split string to remove file reference - uri := strings.Split(ref, "#") - - // have we already seen this remote source? - var parsedRemoteDocument *yaml.Node - alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0]) - - if alreadySeen { - parsedRemoteDocument = foundDocument - } else { - - d := make(chan bool) - var body []byte - var err error - - go func(uri string) { - bc := make(chan []byte) - ec := make(chan error) - var getter RemoteURLHandler = httpClient.Get - if index.config != nil && index.config.RemoteURLHandler != nil { - getter = index.config.RemoteURLHandler - } - - // if we have a remote handler, use it instead of the default. - if index.config != nil && index.config.FSHandler != nil { - go func() { - remoteFS := index.config.FSHandler - remoteFile, rErr := remoteFS.Open(uri) - if rErr != nil { - e := fmt.Errorf("unable to open remote file: %s", rErr) - ec <- e - return - } - b, ioErr := io.ReadAll(remoteFile) - if ioErr != nil { - e := fmt.Errorf("unable to read remote file bytes: %s", ioErr) - ec <- e - return - } - bc <- b - }() - } else { - go getRemoteDoc(getter, uri, bc, ec) - } - select { - case v := <-bc: - body = v - break - case er := <-ec: - err = er - break - } - if len(body) > 0 { - var remoteDoc yaml.Node - er := yaml.Unmarshal(body, &remoteDoc) - if er != nil { - err = er - d <- true - return - } - parsedRemoteDocument = &remoteDoc - if index.config != nil { - index.config.seenRemoteSources.Store(uri, &remoteDoc) - } - } - d <- true - }(uri[0]) - - // wait for double go fun. - <-d - if err != nil { - // no bueno. - return nil, nil, err - } - } - - // lookup item from reference by using a path query. - var query string - if len(uri) >= 2 { - query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) - } else { - query = "$" - } - - query, err := url.PathUnescape(query) - if err != nil { - return nil, nil, err - } - - // remove any URL encoding - query = strings.Replace(query, "~1", "./", 1) - query = strings.ReplaceAll(query, "~1", "/") - - path, err := yamlpath.NewPath(query) - if err != nil { - return nil, nil, err - } - result, _ := path.Find(parsedRemoteDocument) - if len(result) == 1 { - return result[0], parsedRemoteDocument, nil - } + //uri := strings.Split(ref, "#") + // + //// have we already seen this remote source? + //var parsedRemoteDocument *yaml.Node + //alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0]) + // + //if alreadySeen { + // parsedRemoteDocument = foundDocument + //} else { + // + // d := make(chan bool) + // var body []byte + // var err error + // + // go func(uri string) { + // bc := make(chan []byte) + // ec := make(chan error) + // var getter = httpClient.Get + // if index.config != nil && index.config.RemoteURLHandler != nil { + // getter = index.config.RemoteURLHandler + // } + // + // // if we have a remote handler, use it instead of the default. + // if index.config != nil && index.config.FSHandler != nil { + // go func() { + // remoteFS := index.config.FSHandler + // remoteFile, rErr := remoteFS.Open(uri) + // if rErr != nil { + // e := fmt.Errorf("unable to open remote file: %s", rErr) + // ec <- e + // return + // } + // b, ioErr := io.ReadAll(remoteFile) + // if ioErr != nil { + // e := fmt.Errorf("unable to read remote file bytes: %s", ioErr) + // ec <- e + // return + // } + // bc <- b + // }() + // } else { + // go getRemoteDoc(getter, uri, bc, ec) + // } + // select { + // case v := <-bc: + // body = v + // break + // case er := <-ec: + // err = er + // break + // } + // if len(body) > 0 { + // var remoteDoc yaml.Node + // er := yaml.Unmarshal(body, &remoteDoc) + // if er != nil { + // err = er + // d <- true + // return + // } + // parsedRemoteDocument = &remoteDoc + // if index.config != nil { + // index.config.seenRemoteSources.Store(uri, &remoteDoc) + // } + // } + // d <- true + // }(uri[0]) + // + // // wait for double go fun. + // <-d + // if err != nil { + // // no bueno. + // return nil, nil, err + // } + //} + // + //// lookup item from reference by using a path query. + //var query string + //if len(uri) >= 2 { + // query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) + //} else { + // query = "$" + //} + // + //query, err := url.PathUnescape(query) + //if err != nil { + // return nil, nil, err + //} + // + //// remove any URL encoding + //query = strings.Replace(query, "~1", "./", 1) + //query = strings.ReplaceAll(query, "~1", "/") + // + //path, err := yamlpath.NewPath(query) + //if err != nil { + // return nil, nil, err + //} + //result, _ := path.Find(parsedRemoteDocument) + //if len(result) == 1 { + // return result[0], parsedRemoteDocument, nil + //} return nil, nil, nil } @@ -214,73 +182,83 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, // split string to remove file reference uri := strings.Split(ref, "#") file := strings.ReplaceAll(uri[0], "file:", "") - filePath := filepath.Dir(file) - fileName := filepath.Base(file) + //filePath := filepath.Dir(file) + //fileName := filepath.Base(file) + absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) - var parsedRemoteDocument *yaml.Node - - if index.seenRemoteSources[file] != nil { - parsedRemoteDocument = index.seenRemoteSources[file] - } else { - - base := index.config.BasePath - fileToRead := filepath.Join(base, filePath, fileName) - var body []byte - var err error - - // if we have an FS handler, use it instead of the default behavior - if index.config != nil && index.config.FSHandler != nil { - remoteFS := index.config.FSHandler - remoteFile, rErr := remoteFS.Open(fileToRead) - if rErr != nil { - e := fmt.Errorf("unable to open file: %s", rErr) - return nil, nil, e - } - body, err = io.ReadAll(remoteFile) - if err != nil { - e := fmt.Errorf("unable to read file bytes: %s", err) - return nil, nil, e - } - - } else { - - // try and read the file off the local file system, if it fails - // check for a baseURL and then ask our remote lookup function to go try and get it. - body, err = os.ReadFile(fileToRead) - - if err != nil { - - // if we have a baseURL, then we can try and get the file from there. - if index.config != nil && index.config.BaseURL != nil { - - u := index.config.BaseURL - remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) - a, b, e := index.lookupRemoteReference(remoteRef) - if e != nil { - // give up, we can't find the file, not locally, not remotely. It's toast. - return nil, nil, e - } - return a, b, nil - - } else { - // no baseURL? then we can't do anything, give up. - return nil, nil, err - } - } - } - var remoteDoc yaml.Node - err = yaml.Unmarshal(body, &remoteDoc) - if err != nil { - return nil, nil, err - } - parsedRemoteDocument = &remoteDoc - if index.seenLocalSources != nil { - index.sourceLock.Lock() - index.seenLocalSources[file] = &remoteDoc - index.sourceLock.Unlock() - } + // extract the document from the rolodex. + rFile, rError := index.rolodex.Open(absoluteFileLocation) + if rError != nil { + return nil, nil, rError } + parsedDocument, err := rFile.GetContentAsYAMLNode() + if err != nil { + return nil, nil, err + } + + //if index.seenRemoteSources[file] != nil { + // parsedDocument = index.seenRemoteSources[file] + //} else { + // + // base := index.config.BasePath + // fileToRead := filepath.Join(base, filePath, fileName) + // var body []byte + // var err error + // + // // if we have an FS handler, use it instead of the default behavior + // if index.config != nil && index.config.FSHandler != nil { + // remoteFS := index.config.FSHandler + // remoteFile, rErr := remoteFS.Open(fileToRead) + // if rErr != nil { + // e := fmt.Errorf("unable to open file: %s", rErr) + // return nil, nil, e + // } + // body, err = io.ReadAll(remoteFile) + // if err != nil { + // e := fmt.Errorf("unable to read file bytes: %s", err) + // return nil, nil, e + // } + // + // } else { + // + // // try and read the file off the local file system, if it fails + // // check for a baseURL and then ask our remote lookup function to go try and get it. + // body, err = os.ReadFile(fileToRead) + // + // if err != nil { + // + // // if we have a baseURL, then we can try and get the file from there. + // if index.config != nil && index.config.BaseURL != nil { + // + // u := index.config.BaseURL + // remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) + // a, b, e := index.lookupRemoteReference(remoteRef) + // if e != nil { + // // give up, we can't find the file, not locally, not remotely. It's toast. + // return nil, nil, e + // } + // return a, b, nil + // + // } else { + // // no baseURL? then we can't do anything, give up. + // return nil, nil, err + // } + // } + // } + // var remoteDoc yaml.Node + // err = yaml.Unmarshal(body, &remoteDoc) + // if err != nil { + // return nil, nil, err + // } + // parsedDocument = &remoteDoc + // if index.seenLocalSources != nil { + // index.sourceLock.Lock() + // index.seenLocalSources[file] = &remoteDoc + // index.sourceLock.Unlock() + // } + //} + // lookup item from reference by using a path query. var query string if len(uri) >= 2 { @@ -289,7 +267,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, query = "$" } - query, err := url.PathUnescape(query) + query, err = url.PathUnescape(query) if err != nil { return nil, nil, err } @@ -302,154 +280,214 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, if err != nil { return nil, nil, err } - result, _ := path.Find(parsedRemoteDocument) + result, _ := path.Find(parsedDocument) if len(result) == 1 { - return result[0], parsedRemoteDocument, nil + return result[0], parsedDocument, nil } - return nil, parsedRemoteDocument, nil + return nil, parsedDocument, nil } -func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference { - if index.root != nil { +func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference { + // check component for url encoding. + if strings.Contains(componentId, "%") { + // decode the url. + componentId, _ = url.QueryUnescape(componentId) + } - // check component for url encoding. - if strings.Contains(componentId, "%") { - // decode the url. - componentId, _ = url.QueryUnescape(componentId) + name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId) + path, err := yamlpath.NewPath(friendlySearch) + if path == nil || err != nil { + return nil // no component found + } + res, _ := path.Find(root) + + if len(res) == 1 { + resNode := res[0] + if res[0].Kind == yaml.DocumentNode { + resNode = res[0].Content[0] } - name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId) - path, err := yamlpath.NewPath(friendlySearch) - if path == nil || err != nil { - return nil // no component found - } - res, _ := path.Find(index.root) + fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId) - if len(res) == 1 { - resNode := res[0] - if res[0].Kind == yaml.DocumentNode { - resNode = res[0].Content[0] - } - ref := &Reference{ - Definition: componentId, - Name: name, - Node: resNode, - Path: friendlySearch, - RequiredRefProperties: index.extractDefinitionRequiredRefProperties(res[0], map[string][]string{}), - } + // extract properties - return ref + ref := &Reference{ + FullDefinition: fullDef, + Definition: componentId, + Name: name, + Node: resNode, + Path: friendlySearch, + RequiredRefProperties: extractDefinitionRequiredRefProperties(resNode, map[string][]string{}), } + + return ref } return nil } -func (index *SpecIndex) performExternalLookup(uri []string, componentId string, - lookupFunction ExternalLookupFunction, parent *yaml.Node) *Reference { +//func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference { +// if index.root != nil { +// return FindComponent(index.root, componentId, ) +// } +// return nil +//} + +func (index *SpecIndex) performExternalLookup(uri []string) *Reference { + if len(uri) > 0 { - index.externalLock.RLock() - externalSpecIndex := index.externalSpecIndex[uri[0]] - index.externalLock.RUnlock() - if externalSpecIndex == nil { - _, newRoot, err := lookupFunction(componentId) - if err != nil { - indexError := &IndexingError{ - Err: err, - Node: parent, - Path: componentId, - } - index.errorLock.Lock() - index.refErrors = append(index.refErrors, indexError) - index.errorLock.Unlock() - return nil - } + // split string to remove file reference + file := strings.ReplaceAll(uri[0], "file:", "") + fileName := filepath.Base(file) - // cool, cool, lets index this spec also. This is a recursive action and will keep going - // until all remote references have been found. - var bp *url.URL - var bd string - - if index.config.BaseURL != nil { - bp = index.config.BaseURL - } - if index.config.BasePath != "" { - bd = index.config.BasePath - } - - var path, newBasePath string - var newUrl *url.URL - - if bp != nil { - path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false) - newUrl, _ = url.Parse(path) - newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path))) - } - if bd != "" { - if len(uri[0]) > 0 { - // if there is no base url defined, but we can know we have been requested remotely, - // set the base url to the remote url base path. - // first check if the first param is actually a URL - io, er := url.ParseRequestURI(uri[0]) - if er != nil { - newBasePath = filepath.Dir(filepath.Join(bd, uri[0])) - } else { - if newUrl == nil || newUrl.String() != io.String() { - newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path))) - } - newBasePath = filepath.Dir(filepath.Join(bd, uri[1])) - } - } else { - newBasePath = filepath.Dir(filepath.Join(bd, uri[0])) - } - } - - if newUrl != nil || newBasePath != "" { - newConfig := &SpecIndexConfig{ - BaseURL: newUrl, - BasePath: newBasePath, - AllowRemoteLookup: index.config.AllowRemoteLookup, - AllowFileLookup: index.config.AllowFileLookup, - ParentIndex: index, - seenRemoteSources: index.config.seenRemoteSources, - remoteLock: index.config.remoteLock, - uri: uri, - } - - var newIndex *SpecIndex - seen := index.SearchAncestryForSeenURI(uri[0]) - if seen == nil { - - newIndex = NewSpecIndexWithConfig(newRoot, newConfig) - index.refLock.Lock() - index.externalLock.Lock() - index.externalSpecIndex[uri[0]] = newIndex - index.externalLock.Unlock() - newIndex.relativePath = path - newIndex.parentIndex = index - index.AddChild(newIndex) - index.refLock.Unlock() - externalSpecIndex = newIndex - } else { - externalSpecIndex = seen - } - } + var absoluteFileLocation string + if filepath.IsAbs(file) { + absoluteFileLocation = file + } else { + absoluteFileLocation, _ = filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) } - if externalSpecIndex != nil { - foundRef := externalSpecIndex.FindComponentInRoot(uri[1]) + // extract the document from the rolodex. + rFile, rError := index.rolodex.Open(absoluteFileLocation) + if rError != nil { + logger.Error("unable to open rolodex file", "file", absoluteFileLocation, "error", rError) + return nil + } + + parsedDocument, err := rFile.GetContentAsYAMLNode() + if err != nil { + logger.Error("unable to parse rolodex file", "file", absoluteFileLocation, "error", err) + return nil + } + + //fmt.Printf("parsedDocument: %v\n", parsedDocument) + + //index.externalLock.RLock() + //externalSpecIndex := index.externalSpecIndex[uri[0]] + //index.externalLock.RUnlock() + + //if externalSpecIndex == nil { + // _, newRoot, err := lookupFunction(componentId) + // if err != nil { + // indexError := &IndexingError{ + // Err: err, + // Node: parent, + // Path: componentId, + // } + // index.errorLock.Lock() + // index.refErrors = append(index.refErrors, indexError) + // index.errorLock.Unlock() + // return nil + // } + // + // // cool, cool, lets index this spec also. This is a recursive action and will keep going + // // until all remote references have been found. + // var bp *url.URL + // var bd string + // + // if index.config.BaseURL != nil { + // bp = index.config.BaseURL + // } + // if index.config.BasePath != "" { + // bd = index.config.BasePath + // } + // + // var path, newBasePath string + // var newUrl *url.URL + // + // if bp != nil { + // path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false) + // newUrl, _ = url.Parse(path) + // newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path))) + // } + // if bd != "" { + // if len(uri[0]) > 0 { + // // if there is no base url defined, but we can know we have been requested remotely, + // // set the base url to the remote url base path. + // // first check if the first param is actually a URL + // io, er := url.ParseRequestURI(uri[0]) + // if er != nil { + // newBasePath = filepath.Dir(filepath.Join(bd, uri[0])) + // } else { + // if newUrl == nil || newUrl.String() != io.String() { + // newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path))) + // } + // newBasePath = filepath.Dir(filepath.Join(bd, uri[1])) + // } + // } else { + // newBasePath = filepath.Dir(filepath.Join(bd, uri[0])) + // } + // } + // + // if newUrl != nil || newBasePath != "" { + // newConfig := &SpecIndexConfig{ + // BaseURL: newUrl, + // BasePath: newBasePath, + // AllowRemoteLookup: index.config.AllowRemoteLookup, + // AllowFileLookup: index.config.AllowFileLookup, + // ParentIndex: index, + // seenRemoteSources: index.config.seenRemoteSources, + // remoteLock: index.config.remoteLock, + // uri: uri, + // AvoidBuildIndex: index.config.AvoidBuildIndex, + // } + // + // var newIndex *SpecIndex + // seen := index.SearchAncestryForSeenURI(uri[0]) + // if seen == nil { + // + // newIndex = NewSpecIndexWithConfig(newRoot, newConfig) + // index.refLock.Lock() + // index.externalLock.Lock() + // index.externalSpecIndex[uri[0]] = newIndex + // index.externalLock.Unlock() + // newIndex.relativePath = path + // newIndex.parentIndex = index + // index.AddChild(newIndex) + // index.refLock.Unlock() + // externalSpecIndex = newIndex + // } else { + // externalSpecIndex = seen + // } + // } + //} + + wholeFile := false + query := "" + if len(uri) < 2 { + wholeFile = true + } else { + query = fmt.Sprintf("#/%s", strings.Replace(uri[1], "~1", "./", 1)) + query = strings.ReplaceAll(query, "~1", "/") + } + + // check if there is a component we want to suck in, or if the + // entire root needs to come in. + var foundRef *Reference + if wholeFile { + if parsedDocument.Kind == yaml.DocumentNode { + parsedDocument = parsedDocument.Content[0] + } + + // TODO: remote locations + + foundRef = &Reference{ + Definition: fileName, + Name: fileName, + Node: parsedDocument, + IsRemote: true, + RemoteLocation: absoluteFileLocation, + Path: "$", + RequiredRefProperties: extractDefinitionRequiredRefProperties(parsedDocument, map[string][]string{}), + } + return foundRef + } else { + foundRef = FindComponent(parsedDocument, query, absoluteFileLocation) if foundRef != nil { - nameSegs := strings.Split(uri[1], "/") - ref := &Reference{ - Definition: componentId, - Name: nameSegs[len(nameSegs)-1], - Node: foundRef.Node, - IsRemote: true, - RemoteLocation: componentId, - Path: foundRef.Path, - } - return ref + foundRef.IsRemote = true + foundRef.RemoteLocation = absoluteFileLocation + return foundRef } } } diff --git a/index/find_component_test.go b/index/find_component_test.go index 872abc1..07db574 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -44,8 +44,6 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) { c.BasePath = "../test_specs" index := NewSpecIndexWithConfig(&rootNode, c) assert.Nil(t, index.uri) - assert.NotNil(t, index.children[0].uri) - assert.NotNil(t, index.children[0].children[0].uri) assert.NotNil(t, index.SearchIndexForReference("second.yaml#/properties/property2")) assert.NotNil(t, index.SearchIndexForReference("second.yaml")) assert.Nil(t, index.SearchIndexForReference("fourth.yaml")) @@ -504,3 +502,70 @@ paths: assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error()) assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error()) } + +func TestSpecIndex_Complex_Local_File_Design(t *testing.T) { + + main := `openapi: 3.1.0 +paths: + /anything/circularReference: + get: + operationId: circularReferenceGet + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "components.yaml#/components/schemas/validCircularReferenceObject" + /anything/oneOfCircularReference: + get: + operationId: oneOfCircularReferenceGet + tags: + - generation + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "components.yaml#/components/schemas/oneOfCircularReferenceObject"` + + components := `components: + schemas: + validCircularReferenceObject: + type: object + properties: + circular: + type: array + items: + $ref: "#/components/schemas/validCircularReferenceObject" + oneOfCircularReferenceObject: + type: object + properties: + child: + oneOf: + - $ref: "#/components/schemas/oneOfCircularReferenceObject" + - $ref: "#/components/schemas/simpleObject" + required: + - child + simpleObject: + description: "simple" + type: object + properties: + str: + type: string + description: "A string property." + example: "example" ` + + _ = os.WriteFile("components.yaml", []byte(components), 0644) + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(main), &rootNode) + + c := CreateOpenAPIIndexConfig() + index := NewSpecIndexWithConfig(&rootNode, c) + + assert.Len(t, index.GetReferenceIndexErrors(), 2) + assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error()) + assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error()) +} diff --git a/index/index_model.go b/index/index_model.go index 7b1fd2b..50fc436 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -11,7 +11,6 @@ import ( "os" "sync" - "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" ) @@ -25,15 +24,17 @@ const ( // Reference is a wrapper around *yaml.Node results to make things more manageable when performing // algorithms on data models. the *yaml.Node def is just a bit too low level for tracking state. type Reference struct { - Definition string - Name string - Node *yaml.Node - ParentNode *yaml.Node - ParentNodeSchemaType string // used to determine if the parent node is an array or not. - Resolved bool - Circular bool - Seen bool - IsRemote bool + FullDefinition string + Definition string + Name string + Node *yaml.Node + ParentNode *yaml.Node + ParentNodeSchemaType string // used to determine if the parent node is an array or not. + Resolved bool + Circular bool + Seen bool + IsRemote bool + //FileLocation string RemoteLocation string Path string // this won't always be available. RequiredRefProperties map[string][]string // definition names (eg, #/definitions/One) to a list of required properties on this definition which reference that definition @@ -41,8 +42,9 @@ type Reference struct { // ReferenceMapped is a helper struct for mapped references put into sequence (we lose the key) type ReferenceMapped struct { - Reference *Reference - Definition string + Reference *Reference + Definition string + FullDefinition string } // SpecIndexConfig is a configuration struct for the SpecIndex introduced in 0.6.0 that provides an expandable @@ -67,6 +69,7 @@ type SpecIndexConfig struct { // If resolving remotely, the RemoteURLHandler will be used to fetch the remote document. // If not set, the default http client will be used. // Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132 + // deprecated: Use the Rolodex instead RemoteURLHandler func(url string) (*http.Response, error) // FSHandler is an entity that implements the `fs.FS` interface that will be used to fetch local or remote documents. @@ -81,6 +84,7 @@ type SpecIndexConfig struct { // it also overrides the RemoteURLHandler if set. // // Resolves[#85] https://github.com/pb33f/libopenapi/issues/85 + // deprecated: Use the Rolodex instead FSHandler fs.FS // If resolving locally, the BasePath will be the root from which relative references will be resolved from @@ -92,12 +96,13 @@ type SpecIndexConfig struct { // exploits, but it's better to be safe than sorry. // // To read more about this, you can find a discussion here: https://github.com/pb33f/libopenapi/pull/64 - AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false - AllowFileLookup bool // Allow file lookups for references. Defaults to false + //AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false + //AllowFileLookup bool // Allow file lookups for references. Defaults to false // ParentIndex allows the index to be created with knowledge of a parent, before being parsed. This allows - // a breakglass to be used to prevent loops, checking the tree before recursing down. - ParentIndex *SpecIndex + // a breakglass to be used to prevent loops, checking the tree before cursing down. + // deprecated: Use the Rolodex instead, this is no longer needed, indexes are finite and do not have children. + //ParentIndex *SpecIndex // If set to true, the index will not be built out, which means only the foundational elements will be // parsed and added to the index. This is useful to avoid building out an index if the specification is @@ -106,14 +111,40 @@ type SpecIndexConfig struct { // Use the `BuildIndex()` method on the index to build it out once resolved/ready. AvoidBuildIndex bool + AvoidCircularReferenceCheck bool + // SpecInfo is a pointer to the SpecInfo struct that contains the root node and the spec version. It's the // struct that was used to create this index. SpecInfo *datamodel.SpecInfo + // Rolodex is what provides all file and remote based lookups. Without the rolodex, no remote or file lookups + // can be used. Normally you won't need to worry about setting this as each root document gets a rolodex + // of its own automatically. + Rolodex *Rolodex + + SpecAbsolutePath string + + // IgnorePolymorphicCircularReferences will skip over checking for circular references in polymorphic schemas. + // A polymorphic schema is any schema that is composed other schemas using references via `oneOf`, `anyOf` of `allOf`. + // This is disabled by default, which means polymorphic circular references will be checked. + IgnorePolymorphicCircularReferences bool + + // IgnoreArrayCircularReferences will skip over checking for circular references in arrays. Sometimes a circular + // reference is required to describe a data-shape correctly. Often those shapes are valid circles if the + // type of the schema implementing the loop is an array. An empty array would technically break the loop. + // So if libopenapi is returning circular references for this use case, then this option should be enabled. + // this is disabled by default, which means array circular references will be checked. + IgnoreArrayCircularReferences bool + + // SkipCircularReferenceCheck will skip over checking for circular references. This is disabled by default, which + // means circular references will be checked. This is useful for developers building out models that should be + // indexed later on. + //SkipCircularReferenceCheck bool + // private fields - seenRemoteSources *syncmap.Map - remoteLock *sync.Mutex - uri []string + //seenRemoteSources *syncmap.Map + //remoteLock *sync.Mutex + uri []string } // CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and @@ -123,10 +154,10 @@ type SpecIndexConfig struct { func CreateOpenAPIIndexConfig() *SpecIndexConfig { cw, _ := os.Getwd() return &SpecIndexConfig{ - BasePath: cw, - AllowRemoteLookup: true, - AllowFileLookup: true, - seenRemoteSources: &syncmap.Map{}, + BasePath: cw, + //AllowRemoteLookup: true, + //AllowFileLookup: true, + //seenRemoteSources: &syncmap.Map{}, } } @@ -137,10 +168,10 @@ func CreateOpenAPIIndexConfig() *SpecIndexConfig { func CreateClosedAPIIndexConfig() *SpecIndexConfig { cw, _ := os.Getwd() return &SpecIndexConfig{ - BasePath: cw, - AllowRemoteLookup: false, - AllowFileLookup: false, - seenRemoteSources: &syncmap.Map{}, + BasePath: cw, + //AllowRemoteLookup: false, + //AllowFileLookup: false, + //seenRemoteSources: &syncmap.Map{}, } } @@ -148,6 +179,7 @@ func CreateClosedAPIIndexConfig() *SpecIndexConfig { // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules, // everything is pre-walked if you need it. type SpecIndex struct { + rolodex *Rolodex // the rolodex is used to fetch remote and file based documents. allRefs map[string]*Reference // all (deduplicated) refs rawSequencedRefs []*Reference // all raw references in sequence as they are scanned, not deduped. linesWithRefs map[int]bool // lines that link to references. @@ -183,7 +215,8 @@ type SpecIndex struct { rootSecurity []*Reference // root security definitions. rootSecurityNode *yaml.Node // root security node. refsWithSiblings map[string]Reference // references with sibling elements next to them - pathRefsLock sync.Mutex // create lock for all refs maps, we want to build data as fast as we can + pathRefsLock sync.RWMutex // create lock for all refs maps, we want to build data as fast as we can + operationLock sync.Mutex // create lock for operations externalDocumentsCount int // number of externalDocument nodes found operationTagsCount int // number of unique tags in operations globalTagsCount int // number of global tags defined @@ -248,12 +281,17 @@ type SpecIndex struct { componentIndexChan chan bool polyComponentIndexChan chan bool - // when things get complex (looking at you digital ocean) then we need to know - // what we have seen across indexes, so we need to be able to travel back up to the root - // cto avoid re-downloading sources. - parentIndex *SpecIndex - uri []string - children []*SpecIndex + specAbsolutePath string + resolver *Resolver + + //parentIndex *SpecIndex + uri []string + //children []*SpecIndex +} + +// GetResolver returns the resolver for this index. +func (index *SpecIndex) GetResolver() *Resolver { + return index.resolver } // GetConfig returns the SpecIndexConfig for this index. @@ -261,15 +299,15 @@ func (index *SpecIndex) GetConfig() *SpecIndexConfig { return index.config } -// AddChild adds a child index to this index, a child index is an index created from a remote or file reference. -func (index *SpecIndex) AddChild(child *SpecIndex) { - index.children = append(index.children, child) -} - -// GetChildren returns the children of this index. -func (index *SpecIndex) GetChildren() []*SpecIndex { - return index.children -} +//// AddChild adds a child index to this index, a child index is an index created from a remote or file reference. +//func (index *SpecIndex) AddChild(child *SpecIndex) { +// index.children = append(index.children, child) +//} +// +//// GetChildren returns the children of this index. +//func (index *SpecIndex) GetChildren() []*SpecIndex { +// return index.children +//} // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the // URI based document. Decides if the reference is local, remote or in a file. diff --git a/index/index_model_test.go b/index/index_model_test.go index 60642f7..c3d2dc3 100644 --- a/index/index_model_test.go +++ b/index/index_model_test.go @@ -8,21 +8,21 @@ import ( "testing" ) -func TestSpecIndex_Children(t *testing.T) { - idx1 := new(SpecIndex) - idx2 := new(SpecIndex) - idx3 := new(SpecIndex) - idx4 := new(SpecIndex) - idx5 := new(SpecIndex) - idx1.AddChild(idx2) - idx1.AddChild(idx3) - idx3.AddChild(idx4) - idx4.AddChild(idx5) - assert.Equal(t, 2, len(idx1.GetChildren())) - assert.Equal(t, 1, len(idx3.GetChildren())) - assert.Equal(t, 1, len(idx4.GetChildren())) - assert.Equal(t, 0, len(idx5.GetChildren())) -} +//func TestSpecIndex_Children(t *testing.T) { +// idx1 := new(SpecIndex) +// idx2 := new(SpecIndex) +// idx3 := new(SpecIndex) +// idx4 := new(SpecIndex) +// idx5 := new(SpecIndex) +// idx1.AddChild(idx2) +// idx1.AddChild(idx3) +// idx3.AddChild(idx4) +// idx4.AddChild(idx5) +// assert.Equal(t, 2, len(idx1.GetChildren())) +// assert.Equal(t, 1, len(idx3.GetChildren())) +// assert.Equal(t, 1, len(idx4.GetChildren())) +// assert.Equal(t, 0, len(idx5.GetChildren())) +//} func TestSpecIndex_GetConfig(t *testing.T) { idx1 := new(SpecIndex) diff --git a/index/resolver.go b/index/resolver.go index 3b14b8a..2351fa8 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -4,11 +4,9 @@ package index import ( - "fmt" - "strings" - - "github.com/pb33f/libopenapi/utils" - "gopkg.in/yaml.v3" + "fmt" + "github.com/pb33f/libopenapi/utils" + "gopkg.in/yaml.v3" ) // ResolvingError represents an issue the resolver had trying to stitch the tree together. @@ -259,7 +257,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j } journey = append(journey, ref) - relatives := resolver.extractRelatives(ref.Node, nil, seen, journey, resolve) + relatives := resolver.extractRelatives(ref, ref.Node, nil, seen, journey, resolve) seen = make(map[string]bool) @@ -362,7 +360,7 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDe return false, visitedDefinitions } -func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, +func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.Node, foundRelatives map[string]bool, journey []*Reference, resolve bool) []*Reference { @@ -400,7 +398,7 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, // } //} - found = append(found, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...) + found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, resolve)...) } if i%2 == 0 && n.Value == "$ref" { @@ -411,9 +409,9 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, value := node.Content[i+1].Value - ref := resolver.specIndex.SearchIndexForReference(value) + locatedRef := resolver.specIndex.SearchIndexForReference(value) - if ref == nil { + if locatedRef == nil { _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) err := &ResolvingError{ ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value), @@ -434,15 +432,9 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, } } - r := &Reference{ - Definition: value, - Name: value, - Node: node, - ParentNode: parent, - ParentNodeSchemaType: schemaType, - } + locatedRef[0].ParentNodeSchemaType = schemaType - found = append(found, r) + found = append(found, locatedRef[0]) foundRelatives[value] = true } @@ -459,30 +451,30 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, if _, v := utils.FindKeyNodeTop("items", node.Content[i+1].Content); v != nil { if utils.IsNodeMap(v) { if d, _, l := utils.IsNodeRefValue(v); d { - ref := resolver.specIndex.GetMappedReferences()[l] - if ref != nil && !ref.Circular { + mappedRefs := resolver.specIndex.GetMappedReferences()[l] + if mappedRefs != nil && !mappedRefs.Circular { circ := false for f := range journey { - if journey[f].Definition == ref.Definition { + if journey[f].Definition == mappedRefs.Definition { circ = true break } } if !circ { - resolver.VisitReference(ref, foundRelatives, journey, resolve) + resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve) } else { - loop := append(journey, ref) + loop := append(journey, mappedRefs) circRef := &CircularReferenceResult{ Journey: loop, - Start: ref, + Start: mappedRefs, LoopIndex: i, - LoopPoint: ref, + LoopPoint: mappedRefs, PolymorphicType: n.Value, IsPolymorphicResult: true, } - ref.Seen = true - ref.Circular = true + mappedRefs.Seen = true + mappedRefs.Circular = true if resolver.IgnorePoly { resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) } else { @@ -501,36 +493,31 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, v := node.Content[i+1].Content[q] if utils.IsNodeMap(v) { if d, _, l := utils.IsNodeRefValue(v); d { - strangs := strings.Split(l, "/#") - if len(strangs) == 2 { - fmt.Println("wank") - } - - ref := resolver.specIndex.GetMappedReferences()[l] - if ref != nil && !ref.Circular { + mappedRefs := resolver.specIndex.GetMappedReferences()[l] + if mappedRefs != nil && !mappedRefs.Circular { circ := false for f := range journey { - if journey[f].Definition == ref.Definition { + if journey[f].Definition == mappedRefs.Definition { circ = true break } } if !circ { - resolver.VisitReference(ref, foundRelatives, journey, resolve) + resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve) } else { - loop := append(journey, ref) + loop := append(journey, mappedRefs) circRef := &CircularReferenceResult{ Journey: loop, - Start: ref, + Start: mappedRefs, LoopIndex: i, - LoopPoint: ref, + LoopPoint: mappedRefs, PolymorphicType: n.Value, IsPolymorphicResult: true, } - ref.Seen = true - ref.Circular = true + mappedRefs.Seen = true + mappedRefs.Circular = true if resolver.IgnorePoly { resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) } else { diff --git a/index/rolodex.go b/index/rolodex.go index 28100b0..0b27563 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -51,17 +51,18 @@ type RolodexFS interface { } type Rolodex struct { - localFS map[string]fs.FS - remoteFS map[string]fs.FS - indexed bool - built bool - resolved bool - circChecked bool - indexConfig *SpecIndexConfig - indexingDuration time.Duration - indexes []*SpecIndex - rootIndex *SpecIndex - caughtErrors []error + localFS map[string]fs.FS + remoteFS map[string]fs.FS + indexed bool + built bool + resolved bool + circChecked bool + indexConfig *SpecIndexConfig + indexingDuration time.Duration + indexes []*SpecIndex + rootIndex *SpecIndex + caughtErrors []error + ignoredCircularReferences []*CircularReferenceResult } type rolodexFile struct { @@ -209,6 +210,10 @@ func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex { return r } +func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult { + return r.ignoredCircularReferences +} + func (r *Rolodex) GetIndexingDuration() time.Duration { return r.indexingDuration } @@ -275,10 +280,12 @@ func (r *Rolodex) IndexTheRolodex() error { if copiedConfig.IgnorePolymorphicCircularReferences { resolver.IgnorePolymorphicCircularReferences() } - resolvingErrors := resolver.CheckForCircularReferences() - for e := range resolvingErrors { - caughtErrors = append(caughtErrors, resolvingErrors[e]) - } + //if !copiedConfig.AvoidCircularReferenceCheck { + // resolvingErrors := resolver.CheckForCircularReferences() + // for e := range resolvingErrors { + // caughtErrors = append(caughtErrors, resolvingErrors[e]) + // } + //} if err != nil { errChan <- err @@ -331,6 +338,13 @@ func (r *Rolodex) IndexTheRolodex() error { if !r.indexConfig.AvoidBuildIndex { for _, idx := range indexBuildQueue { idx.BuildIndex() + if r.indexConfig.AvoidCircularReferenceCheck { + continue + } + errs := idx.resolver.CheckForCircularReferences() + for e := range errs { + caughtErrors = append(caughtErrors, errs[e]) + } } } @@ -369,6 +383,27 @@ func (r *Rolodex) CheckForCircularReferences() { for e := range resolvingErrors { r.caughtErrors = append(r.caughtErrors, resolvingErrors[e]) } + if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 { + r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...) + } + if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 { + r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...) + } + } +} + +func (r *Rolodex) Resolve() { + if r.rootIndex != nil && r.rootIndex.resolver != nil { + resolvingErrors := r.rootIndex.resolver.Resolve() + for e := range resolvingErrors { + r.caughtErrors = append(r.caughtErrors, resolvingErrors[e]) + } + if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 { + r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...) + } + if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 { + r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...) + } } } diff --git a/index/search_index.go b/index/search_index.go index 2586d08..4af6fad 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -3,21 +3,39 @@ package index +import ( + "fmt" + "path/filepath" + "strings" +) + // SearchIndexForReference searches the index for a reference, first looking through the mapped references // and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes // extracted when parsing the OpenAPI Spec. func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { + absPath := index.specAbsolutePath + var roloLookup string + uri := strings.Split(ref, "#/") + if len(uri) == 2 { + if uri[0] != "" { + roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + } + ref = fmt.Sprintf("#/%s", uri[1]) + } + if r, ok := index.allMappedRefs[ref]; ok { return []*Reference{r} } // TODO: look in the rolodex. + panic("should not be here") + fmt.Println(roloLookup) return nil //if r, ok := index.allMappedRefs[ref]; ok { - // return []*Reference{r} + // return []*Reference{r}jh //} //for c := range index.children { // found := goFindMeSomething(index.children[c], ref)