From 51971762a96c9647edf3aca1b64acd9a50af6c38 Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 18 Oct 2023 09:29:26 -0400 Subject: [PATCH] Another massive surgical strike with the rolodex and index reshuffle. Signed-off-by: quobix --- index/extract_refs.go | 52 +- index/find_component.go | 44 +- index/find_component_test.go | 2 - index/index_model.go | 6 +- index/resolver.go | 25 +- index/resolver_test.go | 2 +- index/rolodex.go | 99 +- index/rolodex_file_loader.go | 11 +- index/rolodex_ref_extractor.go | 23 +- index/rolodex_remote_loader.go | 164 ++- index/rolodex_remote_loader_test.go | 242 ++-- index/search_index.go | 50 +- index/spec_index.go | 5 +- index/spec_index_test.go | 1241 ++++++++++--------- index/utility_methods.go | 6 +- test_specs/mixedref-burgershop.openapi.yaml | 2 +- 16 files changed, 1120 insertions(+), 854 deletions(-) diff --git a/index/extract_refs.go b/index/extract_refs.go index a59c259..fe6b1e5 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -186,24 +186,47 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, segs := strings.Split(value, "/") name := segs[len(segs)-1] - _, p := utils.ConvertComponentIdIntoFriendlyPathSearch(value) + + var p string + uri := strings.Split(value, "#/") + if strings.HasPrefix(value, "http") || filepath.IsAbs(value) { + if len(uri) == 2 { + _, p = utils.ConvertComponentIdIntoFriendlyPathSearch(fmt.Sprintf("#/%s", uri[1])) + } else { + _, p = utils.ConvertComponentIdIntoFriendlyPathSearch(uri[0]) + } + } else { + if len(uri) == 2 { + _, p = utils.ConvertComponentIdIntoFriendlyPathSearch(fmt.Sprintf("#/%s", uri[1])) + } else { + _, p = utils.ConvertComponentIdIntoFriendlyPathSearch(value) + } + } // determine absolute path to this definition iroot := filepath.Dir(index.specAbsolutePath) - uri := strings.Split(value, "#/") var componentName string var fullDefinitionPath string if len(uri) == 2 { if uri[0] == "" { fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1]) } else { - abs, _ := filepath.Abs(filepath.Join(iroot, uri[0])) - fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) + + if strings.HasPrefix(uri[0], "http") { + fullDefinitionPath = value + } else { + abs, _ := filepath.Abs(filepath.Join(iroot, uri[0])) + fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) + } } componentName = fmt.Sprintf("#/%s", uri[1]) } else { - fullDefinitionPath = fmt.Sprintf("%s#/%s", iroot, uri[0]) - componentName = fmt.Sprintf("#/%s", uri[0]) + if strings.HasPrefix(uri[0], "http") { + fullDefinitionPath = value + } else { + fullDefinitionPath = fmt.Sprintf("%s#/%s", iroot, uri[0]) + componentName = fmt.Sprintf("#/%s", uri[0]) + } } ref := &Reference{ @@ -470,6 +493,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc located := index.FindComponent(ref.FullDefinition, ref.Node) if located != nil { index.refLock.Lock() + // have we already mapped this? if index.allMappedRefs[ref.Definition] == nil { found = append(found, located) index.allMappedRefs[ref.Definition] = located @@ -478,8 +502,22 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc Definition: ref.Definition, FullDefinition: ref.FullDefinition, } - sequence[refIndex] = rm + } else { + // it exists, but is it a component with the same ID? + d := index.allMappedRefs[ref.Definition] + + // if the full definition matches, we're good and can skip this. + if d.FullDefinition != ref.FullDefinition { + found = append(found, located) + index.allMappedRefs[ref.FullDefinition] = located + rm := &ReferenceMapped{ + Reference: located, + Definition: ref.Definition, + FullDefinition: ref.FullDefinition, + } + sequence[refIndex] = rm + } } index.refLock.Unlock() } else { diff --git a/index/find_component.go b/index/find_component.go index e60bd43..4e5a753 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -45,10 +45,24 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re //witch DetermineReferenceResolveType(componentId) { //case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case. - // return index.FindComponentInRoot(componentId) + //return index.FindComponentInRoot(componentId) //case HttpResolve, FileResolve: - return index.performExternalLookup(strings.Split(componentId, "#/")) + + uri := strings.Split(componentId, "#/") + if len(uri) == 2 { + if uri[0] != "" { + if index.specAbsolutePath == uri[0] { + return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[1])) + } else { + return index.lookupRolodex(uri) + } + } else { + return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[1])) + } + } else { + return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[0])) + } //} //return nil @@ -326,23 +340,26 @@ func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Refer return nil } -//func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference { -// if index.root != nil { -// return FindComponent(index.root, componentId, ) -// } -// return nil -//} +func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference { + if index.root != nil { + return FindComponent(index.root, componentId, index.specAbsolutePath) + } + return nil +} -func (index *SpecIndex) performExternalLookup(uri []string) *Reference { +func (index *SpecIndex) lookupRolodex(uri []string) *Reference { if len(uri) > 0 { // split string to remove file reference file := strings.ReplaceAll(uri[0], "file:", "") - fileName := filepath.Base(file) - var absoluteFileLocation string - if filepath.IsAbs(file) { + var absoluteFileLocation, fileName string + + // is this a local or a remote file? + + fileName = filepath.Base(file) + if filepath.IsAbs(file) || strings.HasPrefix(file, "http") { absoluteFileLocation = file } else { absoluteFileLocation, _ = filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) @@ -363,6 +380,9 @@ func (index *SpecIndex) performExternalLookup(uri []string) *Reference { return nil } + if rFile == nil { + panic("FUCK") + } parsedDocument, err = rFile.GetContentAsYAMLNode() if err != nil { logger.Error("unable to parse rolodex file", "file", absoluteFileLocation, "error", err) diff --git a/index/find_component_test.go b/index/find_component_test.go index 36ad048..de31e42 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -50,8 +50,6 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) { rolo.SetRootNode(&rootNode) cf.Rolodex = rolo - // TODO: pick up here. - fsCfg := LocalFSConfig{ BaseDirectory: cf.BasePath, FileFilters: []string{"first.yaml", "second.yaml", "third.yaml", "fourth.yaml"}, diff --git a/index/index_model.go b/index/index_model.go index 61f4978..a2a7725 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -96,8 +96,8 @@ type SpecIndexConfig struct { // exploits, but it's better to be safe than sorry. // // To read more about this, you can find a discussion here: https://github.com/pb33f/libopenapi/pull/64 - //AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false - //AllowFileLookup bool // Allow file lookups for references. Defaults to false + AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false + AllowFileLookup bool // Allow file lookups for references. Defaults to false // ParentIndex allows the index to be created with knowledge of a parent, before being parsed. This allows // a breakglass to be used to prevent loops, checking the tree before cursing down. @@ -280,6 +280,8 @@ type SpecIndex struct { specAbsolutePath string resolver *Resolver + built bool + //parentIndex *SpecIndex uri []string //children []*SpecIndex diff --git a/index/resolver.go b/index/resolver.go index e7ed7e6..f759a04 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -269,7 +269,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j if j.Definition == r.Definition { var foundDup *Reference - foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition) + foundRefs := resolver.specIndex.SearchIndexForReferenceByReference(r) if len(foundRefs) > 0 { foundDup = foundRefs[0] } @@ -311,7 +311,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j if !skip { var original *Reference - foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition) + foundRefs := resolver.specIndex.SearchIndexForReferenceByReference(r) if len(foundRefs) > 0 { original = foundRefs[0] } @@ -408,8 +408,27 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No } value := node.Content[i+1].Value + var locatedRef []*Reference + searchRef := &Reference{ + Definition: value, + FullDefinition: ref.FullDefinition, + RemoteLocation: ref.RemoteLocation, + IsRemote: true, + } - locatedRef := resolver.specIndex.SearchIndexForReference(value) + // we're searching a remote document, we need to build a full path to the reference + if ref.IsRemote { + if ref.RemoteLocation != "" { + searchRef = &Reference{ + Definition: value, + FullDefinition: fmt.Sprintf("%s%s", ref.RemoteLocation, value), + RemoteLocation: ref.RemoteLocation, + IsRemote: true, + } + } + } + + locatedRef = resolver.specIndex.SearchIndexForReferenceByReference(searchRef) if locatedRef == nil { _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) diff --git a/index/resolver_test.go b/index/resolver_test.go index b57b3a1..b576df8 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -407,7 +407,7 @@ func TestResolver_ResolveComponents_Stripe(t *testing.T) { resolveFile, _ := os.ReadFile(baseDir) - info, err := datamodel.ExtractSpecInfoWithDocumentCheck(resolveFile, true) + info, _ := datamodel.ExtractSpecInfoWithDocumentCheck(resolveFile, true) fileFS, err := NewLocalFS(baseDir, os.DirFS(filepath.Dir(baseDir))) if err != nil { diff --git a/index/rolodex.go b/index/rolodex.go index 74ebd16..ef2e083 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -5,6 +5,7 @@ package index import ( "errors" + "fmt" "github.com/pb33f/libopenapi/datamodel" "gopkg.in/yaml.v3" "io" @@ -17,9 +18,12 @@ import ( "time" ) +type HasIndex interface { + GetIndex() *SpecIndex +} + type CanBeIndexed interface { Index(config *SpecIndexConfig) (*SpecIndex, error) - GetIndex() *SpecIndex } type RolodexFile interface { @@ -86,10 +90,10 @@ func (rf *rolodexFile) Name() string { func (rf *rolodexFile) GetIndex() *SpecIndex { if rf.localFile != nil { - return rf.localFile.index + return rf.localFile.GetIndex() } if rf.remoteFile != nil { - // TODO: remote file index + return rf.remoteFile.GetIndex() } return nil } @@ -207,7 +211,6 @@ func (rf *rolodexFile) GetErrors() []error { } func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex { - r := &Rolodex{ indexConfig: indexConfig, localFS: make(map[string]fs.FS), @@ -304,16 +307,23 @@ func (r *Rolodex) IndexTheRolodex() error { indexChan <- idx } - if lfs, ok := fs.(*LocalFS); ok { - for _, f := range lfs.Files { + if lfs, ok := fs.(RolodexFS); ok { + wait := false + for _, f := range lfs.GetFiles() { if idxFile, ko := f.(CanBeIndexed); ko { wg.Add(1) + wait = true go indexFileFunc(idxFile, f.GetFullPath()) } } - wg.Wait() + if wait { + wg.Wait() + } doneChan <- true return + } else { + errChan <- errors.New("rolodex file system is not a RolodexFS") + doneChan <- true } } @@ -440,24 +450,25 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { var errorStack []error var localFile *LocalFile - //var remoteFile *RemoteFile + var remoteFile *RemoteFile if r == nil || r.localFS == nil && r.remoteFS == nil { panic("WHAT NO....") } - for k, v := range r.localFS { + fileLookup := location + isUrl := false + u, _ := url.Parse(location) + if u != nil && u.Scheme != "" { + isUrl = true + } - // check if this is a URL or an abs/rel reference. - fileLookup := location - isUrl := false - u, _ := url.Parse(location) - if u != nil && u.Scheme != "" { - isUrl = true - } + if !isUrl { + + for k, v := range r.localFS { + + // check if this is a URL or an abs/rel reference. - // TODO handle URLs. - if !isUrl { if !filepath.IsAbs(location) { fileLookup, _ = filepath.Abs(filepath.Join(k, location)) } @@ -504,8 +515,52 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { break } } + + } + } else { + + if !r.indexConfig.AllowRemoteLookup { + return nil, fmt.Errorf("remote lookup for '%s' not allowed, please set the index configuration to "+ + "AllowRemoteLookup to true", fileLookup) + } + + for _, v := range r.remoteFS { + f, err := v.Open(fileLookup) + if err == nil { + + if rf, ok := interface{}(f).(*RemoteFile); ok { + remoteFile = rf + break + } else { + + bytes, rErr := io.ReadAll(f) + if rErr != nil { + errorStack = append(errorStack, rErr) + continue + } + s, sErr := f.Stat() + if sErr != nil { + errorStack = append(errorStack, sErr) + continue + } + if len(bytes) > 0 { + remoteFile = &RemoteFile{ + filename: filepath.Base(fileLookup), + name: filepath.Base(fileLookup), + extension: ExtractFileType(fileLookup), + data: bytes, + fullPath: fileLookup, + lastModified: s.ModTime(), + index: r.rootIndex, + } + break + } + } + } + } } + if localFile != nil { return &rolodexFile{ rolodex: r, @@ -514,5 +569,13 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { }, errors.Join(errorStack...) } + if remoteFile != nil { + return &rolodexFile{ + rolodex: r, + location: remoteFile.fullPath, + remoteFile: remoteFile, + }, errors.Join(errorStack...) + } + return nil, errors.Join(errorStack...) } diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 1aeda53..4036cd2 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -20,10 +20,8 @@ type LocalFS struct { entryPointDirectory string baseDirectory string Files map[string]RolodexFile - parseTime int64 logger *slog.Logger readingErrors []error - filters []string } func (l *LocalFS) GetFiles() map[string]RolodexFile { @@ -180,26 +178,23 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { switch extension { case YAML, JSON: - file, readErr := config.DirFS.Open(p) + dirFile, readErr := config.DirFS.Open(p) modTime := time.Now() if readErr != nil { - readingErrors = append(readingErrors, readErr) allErrors = append(allErrors, readErr) logger.Error("[rolodex] cannot open file: ", "file", abs, "error", readErr.Error()) return nil } - stat, statErr := file.Stat() + stat, statErr := dirFile.Stat() if statErr != nil { - readingErrors = append(readingErrors, statErr) allErrors = append(allErrors, statErr) logger.Error("[rolodex] cannot stat file: ", "file", abs, "error", statErr.Error()) } if stat != nil { modTime = stat.ModTime() } - fileData, readErr = io.ReadAll(file) + fileData, readErr = io.ReadAll(dirFile) if readErr != nil { - readingErrors = append(readingErrors, readErr) allErrors = append(allErrors, readErr) logger.Error("cannot read file data: ", "file", abs, "error", readErr.Error()) return nil diff --git a/index/rolodex_ref_extractor.go b/index/rolodex_ref_extractor.go index 3a5c55a..35e5ee2 100644 --- a/index/rolodex_ref_extractor.go +++ b/index/rolodex_ref_extractor.go @@ -12,17 +12,6 @@ import ( // var refRegex = regexp.MustCompile(`['"]?\$ref['"]?\s*:\s*['"]?([^'"]*?)['"]`) var refRegex = regexp.MustCompile(`('\$ref'|"\$ref"|\$ref)\s*:\s*('[^']*'|"[^"]*"|\S*)`) -/* -r := regexp.MustCompile(`('\$ref'|"\$ref"|\$ref)\s*:\s*('[^']*'|"[^"]*"|\S*)`) - matches := r.FindAllStringSubmatch(text, -1) - for _, submatches := range matches { - if len(submatches) > 2 { - fmt.Println("Full match:", submatches[0]) - fmt.Println("JSON Schema reference: ", submatches[2]) - } - } -*/ - type RefType int const ( @@ -108,12 +97,12 @@ func ExtractRefType(ref string) RefType { func ExtractRefs(content string) [][]string { - res := refRegex.FindAllStringSubmatch(content, -1) + return refRegex.FindAllStringSubmatch(content, -1) + + //var results []*ExtractedRef + //for _, r := range res { + // results = append(results, &ExtractedRef{Location: r[1], Type: ExtractRefType(r[1])}) + //} - var results []*ExtractedRef - for _, r := range res { - results = append(results, &ExtractedRef{Location: r[1], Type: ExtractRefType(r[1])}) - } - return res } diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index efee557..f1deb8d 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -6,6 +6,7 @@ package index import ( "errors" "fmt" + "github.com/pb33f/libopenapi/datamodel" "golang.org/x/exp/slog" "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" @@ -19,6 +20,7 @@ import ( ) type RemoteFS struct { + indexConfig *SpecIndexConfig rootURL string rootURLParsed *url.URL RemoteHandlerFunc RemoteURLHandler @@ -41,6 +43,9 @@ type RemoteFile struct { URL *url.URL lastModified time.Time seekingErrors []error + index *SpecIndex + parsed *yaml.Node + offset int64 } func (f *RemoteFile) GetFileName() string { @@ -52,7 +57,25 @@ func (f *RemoteFile) GetContent() string { } func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) { - return nil, errors.New("not implemented") + if f.parsed != nil { + return f.parsed, nil + } + if f.index != nil && f.index.root != nil { + return f.index.root, nil + } + if f.data == nil { + return nil, fmt.Errorf("no data to parse for file: %s", f.fullPath) + } + var root yaml.Node + err := yaml.Unmarshal(f.data, &root) + if err != nil { + return nil, err + } + if f.index != nil && f.index.root == nil { + f.index.root = &root + } + f.parsed = &root + return &root, nil } func (f *RemoteFile) GetFileExtension() FileExtension { @@ -71,6 +94,8 @@ func (f *RemoteFile) GetFullPath() string { return f.fullPath } +// fs.FileInfo interfaces + func (f *RemoteFile) Name() string { return f.name } @@ -91,40 +116,52 @@ func (f *RemoteFile) IsDir() bool { return false } +// fs.File interfaces + func (f *RemoteFile) Sys() interface{} { return nil } -func (f *RemoteFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { - - // TODO - return nil, nil -} -func (f *RemoteFile) GetIndex() *SpecIndex { - - // TODO +func (f *RemoteFile) Close() error { return nil } - -type remoteRolodexFile struct { - f *RemoteFile - offset int64 +func (f *RemoteFile) Stat() (fs.FileInfo, error) { + return f, nil } - -func (f *remoteRolodexFile) Close() error { return nil } -func (f *remoteRolodexFile) Stat() (fs.FileInfo, error) { return f.f, nil } -func (f *remoteRolodexFile) Read(b []byte) (int, error) { - if f.offset >= int64(len(f.f.data)) { +func (f *RemoteFile) Read(b []byte) (int, error) { + if f.offset >= int64(len(f.data)) { return 0, io.EOF } if f.offset < 0 { - return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid} + return 0, &fs.PathError{Op: "read", Path: f.name, Err: fs.ErrInvalid} } - n := copy(b, f.f.data[f.offset:]) + n := copy(b, f.data[f.offset:]) f.offset += int64(n) return n, nil } +func (f *RemoteFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { + + if f.index != nil { + return f.index, nil + } + content := f.data + + // first, we must parse the content of the file + info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, true) + if err != nil { + return nil, err + } + + index := NewSpecIndexWithConfig(info.RootNode, config) + index.specAbsolutePath = f.fullPath + f.index = index + return index, nil +} +func (f *RemoteFile) GetIndex() *SpecIndex { + return f.index +} + type FileExtension int const ( @@ -133,19 +170,39 @@ const ( UNSUPPORTED ) -func NewRemoteFS(rootURL string) (*RemoteFS, error) { +func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) { + remoteRootURL := specIndexConfig.BaseURL + rfs := &RemoteFS{ + logger: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelDebug, + })), + + rootURLParsed: remoteRootURL, + FetchChannel: make(chan *RemoteFile), + } + if remoteRootURL != nil { + rfs.rootURL = remoteRootURL.String() + } + return rfs, nil +} + +func NewRemoteFS() (*RemoteFS, error) { + config := CreateOpenAPIIndexConfig() + return NewRemoteFSWithConfig(config) +} + +func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) { remoteRootURL, err := url.Parse(rootURL) if err != nil { return nil, err } - return &RemoteFS{ - logger: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ - Level: slog.LevelDebug, - })), - rootURL: rootURL, - rootURLParsed: remoteRootURL, - FetchChannel: make(chan *RemoteFile), - }, nil + config := CreateOpenAPIIndexConfig() + config.BaseURL = remoteRootURL + return NewRemoteFSWithConfig(config) +} + +func (i *RemoteFS) SetIndexConfig(config *SpecIndexConfig) { + i.indexConfig = config } func (i *RemoteFS) GetFiles() map[string]RolodexFile { @@ -200,7 +257,7 @@ func (i *RemoteFS) seekRelatives(file *RemoteFile) { fmt.Printf("Found relative HTTP reference: %s\n", ref[1]) } } - if i.remoteRunning == false { + if !i.remoteRunning { i.remoteRunning = true i.remoteWg.Wait() i.remoteRunning = false @@ -215,15 +272,29 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { return nil, err } + remoteParsedOrig, _ := url.Parse(remoteURL) + + // try path first + if r, ok := i.Files.Load(remoteParsedURL.Path); ok { + return r.(*RemoteFile), nil + } + fileExt := ExtractFileType(remoteParsedURL.Path) if fileExt == UNSUPPORTED { return nil, &fs.PathError{Op: "open", Path: remoteURL, Err: fs.ErrInvalid} } - i.logger.Debug("Loading remote file", "file", remoteParsedURL.Path) + // if the remote URL is absolute (http:// or https://), and we have a rootURL defined, we need to override + // the host being defined by this URL, and use the rootURL instead, but keep the path. + if i.rootURLParsed != nil && remoteParsedURL.Host != "" { + remoteParsedURL.Host = i.rootURLParsed.Host + remoteParsedURL.Scheme = i.rootURLParsed.Scheme + } - response, clientErr := i.RemoteHandlerFunc(i.rootURL + remoteURL) + i.logger.Debug("Loading remote file", "file", remoteURL, "remoteURL", remoteParsedURL.String()) + + response, clientErr := i.RemoteHandlerFunc(remoteParsedURL.String()) if clientErr != nil { i.logger.Error("client error", "error", response.StatusCode) @@ -238,7 +309,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { if response.StatusCode >= 400 { i.logger.Error("Unable to fetch remote document %s", "file", remoteParsedURL.Path, "status", response.StatusCode, "resp", string(responseBytes)) - return nil, errors.New(fmt.Sprintf("Unable to fetch remote document: %s", string(responseBytes))) + return nil, fmt.Errorf("unable to fetch remote document: %s", string(responseBytes)) } absolutePath, pathErr := filepath.Abs(remoteParsedURL.Path) @@ -253,10 +324,12 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { lastModifiedTime, parseErr := time.Parse(time.RFC1123, lastModified) if parseErr != nil { - return nil, parseErr + // can't extract last modified, so use now + lastModifiedTime = time.Now() } filename := filepath.Base(remoteParsedURL.Path) + remoteFile := &RemoteFile{ filename: filename, name: remoteParsedURL.Path, @@ -266,14 +339,31 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { URL: remoteParsedURL, lastModified: lastModifiedTime, } + + copiedCfg := *i.indexConfig + + newBase := fmt.Sprintf("%s://%s%s", remoteParsedOrig.Scheme, remoteParsedOrig.Host, + filepath.Dir(remoteParsedOrig.Path)) + newBaseURL, _ := url.Parse(newBase) + + copiedCfg.BaseURL = newBaseURL + copiedCfg.SpecAbsolutePath = remoteURL + idx, _ := remoteFile.Index(&copiedCfg) + + // for each index, we need a resolver + resolver := NewResolver(idx) + idx.resolver = resolver + i.Files.Store(absolutePath, remoteFile) i.logger.Debug("successfully loaded file", "file", absolutePath) i.seekRelatives(remoteFile) - if i.remoteRunning == false { - return &remoteRolodexFile{remoteFile, 0}, errors.Join(i.remoteErrors...) + idx.BuildIndex() + + if !i.remoteRunning { + return remoteFile, errors.Join(i.remoteErrors...) } else { - return &remoteRolodexFile{remoteFile, 0}, nil + return remoteFile, nil } } diff --git a/index/rolodex_remote_loader_test.go b/index/rolodex_remote_loader_test.go index 0820a12..7d1e7dd 100644 --- a/index/rolodex_remote_loader_test.go +++ b/index/rolodex_remote_loader_test.go @@ -4,192 +4,192 @@ package index import ( - "github.com/stretchr/testify/assert" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" + "github.com/stretchr/testify/assert" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" ) var test_httpClient = &http.Client{Timeout: time.Duration(60) * time.Second} func test_buildServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if req.URL.String() == "/file1.yaml" { - rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") - _, _ = rw.Write([]byte(`"$ref": "./deeper/file2.yaml#/components/schemas/Pet"`)) - return - } - if req.URL.String() == "/deeper/file2.yaml" { - rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 08:28:00 GMT") - _, _ = rw.Write([]byte(`"$ref": "/deeper/even_deeper/file3.yaml#/components/schemas/Pet"`)) - return - } + return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + if req.URL.String() == "/file1.yaml" { + rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + _, _ = rw.Write([]byte(`"$ref": "./deeper/file2.yaml#/components/schemas/Pet"`)) + return + } + if req.URL.String() == "/deeper/file2.yaml" { + rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 08:28:00 GMT") + _, _ = rw.Write([]byte(`"$ref": "/deeper/even_deeper/file3.yaml#/components/schemas/Pet"`)) + return + } - if req.URL.String() == "/deeper/even_deeper/file3.yaml" { - rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 10:28:00 GMT") - _, _ = rw.Write([]byte(`"$ref": "../file2.yaml#/components/schemas/Pet"`)) - return - } + if req.URL.String() == "/deeper/even_deeper/file3.yaml" { + rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 10:28:00 GMT") + _, _ = rw.Write([]byte(`"$ref": "../file2.yaml#/components/schemas/Pet"`)) + return + } - rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 12:28:00 GMT") + rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 12:28:00 GMT") - if req.URL.String() == "/deeper/list.yaml" { - _, _ = rw.Write([]byte(`"$ref": "../file2.yaml"`)) - return - } + if req.URL.String() == "/deeper/list.yaml" { + _, _ = rw.Write([]byte(`"$ref": "../file2.yaml"`)) + return + } - if req.URL.String() == "/bag/list.yaml" { - _, _ = rw.Write([]byte(`"$ref": "pocket/list.yaml"\n\n"$ref": "zip/things.yaml"`)) - return - } + if req.URL.String() == "/bag/list.yaml" { + _, _ = rw.Write([]byte(`"$ref": "pocket/list.yaml"\n\n"$ref": "zip/things.yaml"`)) + return + } - if req.URL.String() == "/bag/pocket/list.yaml" { - _, _ = rw.Write([]byte(`"$ref": "../list.yaml"\n\n"$ref": "../../file2.yaml"`)) - return - } + if req.URL.String() == "/bag/pocket/list.yaml" { + _, _ = rw.Write([]byte(`"$ref": "../list.yaml"\n\n"$ref": "../../file2.yaml"`)) + return + } - if req.URL.String() == "/bag/pocket/things.yaml" { - _, _ = rw.Write([]byte(`"$ref": "list.yaml"`)) - return - } + if req.URL.String() == "/bag/pocket/things.yaml" { + _, _ = rw.Write([]byte(`"$ref": "list.yaml"`)) + return + } - if req.URL.String() == "/bag/zip/things.yaml" { - _, _ = rw.Write([]byte(`"$ref": "list.yaml"`)) - return - } + if req.URL.String() == "/bag/zip/things.yaml" { + _, _ = rw.Write([]byte(`"$ref": "list.yaml"`)) + return + } - if req.URL.String() == "/bag/zip/list.yaml" { - _, _ = rw.Write([]byte(`"$ref": "../list.yaml"\n\n"$ref": "../../file1.yaml"\n\n"$ref": "more.yaml""`)) - return - } + if req.URL.String() == "/bag/zip/list.yaml" { + _, _ = rw.Write([]byte(`"$ref": "../list.yaml"\n\n"$ref": "../../file1.yaml"\n\n"$ref": "more.yaml""`)) + return + } - if req.URL.String() == "/bag/zip/more.yaml" { - _, _ = rw.Write([]byte(`"$ref": "../../deeper/list.yaml"\n\n"$ref": "../../bad.yaml"`)) - return - } + if req.URL.String() == "/bag/zip/more.yaml" { + _, _ = rw.Write([]byte(`"$ref": "../../deeper/list.yaml"\n\n"$ref": "../../bad.yaml"`)) + return + } - if req.URL.String() == "/bad.yaml" { - rw.WriteHeader(http.StatusInternalServerError) - _, _ = rw.Write([]byte(`"error, cannot do the thing"`)) - return - } + if req.URL.String() == "/bad.yaml" { + rw.WriteHeader(http.StatusInternalServerError) + _, _ = rw.Write([]byte(`"error, cannot do the thing"`)) + return + } - _, _ = rw.Write([]byte(`OK`)) - })) + _, _ = rw.Write([]byte(`OK`)) + })) } func TestNewRemoteFS_BasicCheck(t *testing.T) { - server := test_buildServer() - defer server.Close() + server := test_buildServer() + defer server.Close() - //remoteFS := NewRemoteFS("https://raw.githubusercontent.com/digitalocean/openapi/main/specification/") - remoteFS, _ := NewRemoteFS(server.URL) - remoteFS.RemoteHandlerFunc = test_httpClient.Get + //remoteFS := NewRemoteFS("https://raw.githubusercontent.com/digitalocean/openapi/main/specification/") + remoteFS, _ := NewRemoteFSWithRootURL(server.URL) + remoteFS.RemoteHandlerFunc = test_httpClient.Get - file, err := remoteFS.Open("/file1.yaml") + file, err := remoteFS.Open("/file1.yaml") - assert.NoError(t, err) + assert.NoError(t, err) - bytes, rErr := io.ReadAll(file) - assert.NoError(t, rErr) + bytes, rErr := io.ReadAll(file) + assert.NoError(t, rErr) - assert.Equal(t, "\"$ref\": \"\"./deeper/file2.yaml#/components/schemas/Pet\"", string(bytes)) + assert.Equal(t, "\"$ref\": \"\"./deeper/file2.yaml#/components/schemas/Pet\"", string(bytes)) - stat, _ := file.Stat() + stat, _ := file.Stat() - assert.Equal(t, "file1.yaml", stat.Name()) - assert.Equal(t, int64(54), stat.Size()) + assert.Equal(t, "file1.yaml", stat.Name()) + assert.Equal(t, int64(54), stat.Size()) - lastMod := stat.ModTime() - assert.Equal(t, "2015-10-21 07:28:00 +0000 GMT", lastMod.String()) + lastMod := stat.ModTime() + assert.Equal(t, "2015-10-21 07:28:00 +0000 GMT", lastMod.String()) } func TestNewRemoteFS_BasicCheck_Relative(t *testing.T) { - server := test_buildServer() - defer server.Close() + server := test_buildServer() + defer server.Close() - remoteFS, _ := NewRemoteFS(server.URL) - remoteFS.RemoteHandlerFunc = test_httpClient.Get + remoteFS, _ := NewRemoteFSWithRootURL(server.URL) + remoteFS.RemoteHandlerFunc = test_httpClient.Get - file, err := remoteFS.Open("/deeper/file2.yaml") + file, err := remoteFS.Open("/deeper/file2.yaml") - assert.NoError(t, err) + assert.NoError(t, err) - bytes, rErr := io.ReadAll(file) - assert.NoError(t, rErr) + bytes, rErr := io.ReadAll(file) + assert.NoError(t, rErr) - assert.Equal(t, "\"$ref\": \"./deeper/even_deeper/file3.yaml#/components/schemas/Pet\"", string(bytes)) + assert.Equal(t, "\"$ref\": \"./deeper/even_deeper/file3.yaml#/components/schemas/Pet\"", string(bytes)) - stat, _ := file.Stat() + stat, _ := file.Stat() - assert.Equal(t, "/deeper/file2.yaml", stat.Name()) - assert.Equal(t, int64(65), stat.Size()) + assert.Equal(t, "/deeper/file2.yaml", stat.Name()) + assert.Equal(t, int64(65), stat.Size()) - lastMod := stat.ModTime() - assert.Equal(t, "2015-10-21 08:28:00 +0000 GMT", lastMod.String()) + lastMod := stat.ModTime() + assert.Equal(t, "2015-10-21 08:28:00 +0000 GMT", lastMod.String()) } func TestNewRemoteFS_BasicCheck_Relative_Deeper(t *testing.T) { - server := test_buildServer() - defer server.Close() + server := test_buildServer() + defer server.Close() - remoteFS, _ := NewRemoteFS(server.URL) - remoteFS.RemoteHandlerFunc = test_httpClient.Get + remoteFS, _ := NewRemoteFSWithRootURL(server.URL) + remoteFS.RemoteHandlerFunc = test_httpClient.Get - file, err := remoteFS.Open("/deeper/even_deeper/file3.yaml") + file, err := remoteFS.Open("/deeper/even_deeper/file3.yaml") - assert.NoError(t, err) + assert.NoError(t, err) - bytes, rErr := io.ReadAll(file) - assert.NoError(t, rErr) + bytes, rErr := io.ReadAll(file) + assert.NoError(t, rErr) - assert.Equal(t, "\"$ref\": \"../file2.yaml#/components/schemas/Pet\"", string(bytes)) + assert.Equal(t, "\"$ref\": \"../file2.yaml#/components/schemas/Pet\"", string(bytes)) - stat, _ := file.Stat() + stat, _ := file.Stat() - assert.Equal(t, "/deeper/even_deeper/file3.yaml", stat.Name()) - assert.Equal(t, int64(47), stat.Size()) + assert.Equal(t, "/deeper/even_deeper/file3.yaml", stat.Name()) + assert.Equal(t, int64(47), stat.Size()) - lastMod := stat.ModTime() - assert.Equal(t, "2015-10-21 10:28:00 +0000 GMT", lastMod.String()) + lastMod := stat.ModTime() + assert.Equal(t, "2015-10-21 10:28:00 +0000 GMT", lastMod.String()) } func TestNewRemoteFS_BasicCheck_SeekRelatives(t *testing.T) { - server := test_buildServer() - defer server.Close() + server := test_buildServer() + defer server.Close() - remoteFS, _ := NewRemoteFS(server.URL) - remoteFS.RemoteHandlerFunc = test_httpClient.Get + remoteFS, _ := NewRemoteFSWithRootURL(server.URL) + remoteFS.RemoteHandlerFunc = test_httpClient.Get - file, err := remoteFS.Open("/bag/list.yaml") + file, err := remoteFS.Open("/bag/list.yaml") - assert.Error(t, err) + assert.Error(t, err) - bytes, rErr := io.ReadAll(file) - assert.NoError(t, rErr) + bytes, rErr := io.ReadAll(file) + assert.NoError(t, rErr) - assert.Equal(t, "\"$ref\": \"pocket/list.yaml\"\\n\\n\"$ref\": \"zip/things.yaml\"", string(bytes)) + assert.Equal(t, "\"$ref\": \"pocket/list.yaml\"\\n\\n\"$ref\": \"zip/things.yaml\"", string(bytes)) - stat, _ := file.Stat() + stat, _ := file.Stat() - assert.Equal(t, "/bag/list.yaml", stat.Name()) - assert.Equal(t, int64(55), stat.Size()) + assert.Equal(t, "/bag/list.yaml", stat.Name()) + assert.Equal(t, int64(55), stat.Size()) - lastMod := stat.ModTime() - assert.Equal(t, "2015-10-21 12:28:00 +0000 GMT", lastMod.String()) + lastMod := stat.ModTime() + assert.Equal(t, "2015-10-21 12:28:00 +0000 GMT", lastMod.String()) - files := remoteFS.GetFiles() - assert.Len(t, remoteFS.remoteErrors, 1) - assert.Len(t, files, 10) + files := remoteFS.GetFiles() + assert.Len(t, remoteFS.remoteErrors, 1) + assert.Len(t, files, 10) - // check correct files are in the cache - assert.Equal(t, "/bag/list.yaml", files["/bag/list.yaml"].GetFullPath()) - assert.Equal(t, "list.yaml", files["/bag/list.yaml"].Name()) + // check correct files are in the cache + assert.Equal(t, "/bag/list.yaml", files["/bag/list.yaml"].GetFullPath()) + assert.Equal(t, "list.yaml", files["/bag/list.yaml"].Name()) } diff --git a/index/search_index.go b/index/search_index.go index e1d8fb5..b468738 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -9,10 +9,9 @@ import ( "strings" ) -// SearchIndexForReference searches the index for a reference, first looking through the mapped references -// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes -// extracted when parsing the OpenAPI Spec. -func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { +func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) []*Reference { + + ref := fullRef.FullDefinition absPath := index.specAbsolutePath if absPath == "" { @@ -22,7 +21,11 @@ func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { uri := strings.Split(ref, "#/") if len(uri) == 2 { if uri[0] != "" { - roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + if strings.HasPrefix(uri[0], "http") { + roloLookup = fullRef.FullDefinition + } else { + roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + } } ref = fmt.Sprintf("#/%s", uri[1]) } else { @@ -38,7 +41,6 @@ func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { return []*Reference{r} } - // TODO: look in the rolodex. if roloLookup != "" { rFile, err := index.rolodex.Open(roloLookup) if err != nil { @@ -70,33 +72,17 @@ func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { } } - panic("should not be here") - fmt.Println(roloLookup) - return nil - - //if r, ok := index.allMappedRefs[ref]; ok { - // return []*Reference{r}jh - //} - //for c := range index.children { - // found := goFindMeSomething(index.children[c], ref) - // if found != nil { - // return found - // } - //} - //return nil -} - -func (index *SpecIndex) SearchAncestryForSeenURI(uri string) *SpecIndex { - //if index.parentIndex == nil { - // return nil - //} - //if index.uri[0] != uri { - // return index.parentIndex.SearchAncestryForSeenURI(uri) - //} - //return index + fmt.Printf("unable to locate reference: %s, within index: %s\n", ref, index.specAbsolutePath) return nil } -func goFindMeSomething(i *SpecIndex, ref string) []*Reference { - return i.SearchIndexForReference(ref) +// SearchIndexForReference searches the index for a reference, first looking through the mapped references +// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes +// extracted when parsing the OpenAPI Spec. +func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { + return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref}) +} + +func (index *SpecIndex) SearchIndexForReferenceWithParent(ref string, reference *Reference) []*Reference { + return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref}) } diff --git a/index/spec_index.go b/index/spec_index.go index 53c6aba..e04fda3 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -103,6 +103,9 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex, avoidBuildOut bool) * // useful for looking up things, the count operations are all run in parallel and then the final calculations are run // the index is ready. func (index *SpecIndex) BuildIndex() { + if index.built { + return + } countFuncs := []func() int{ index.GetOperationCount, index.GetComponentSchemaCount, @@ -132,6 +135,7 @@ func (index *SpecIndex) BuildIndex() { index.GetInlineDuplicateParamCount() index.GetAllDescriptionsCount() index.GetTotalTagsCount() + index.built = true } // GetRootNode returns document root node. @@ -998,7 +1002,6 @@ func (index *SpecIndex) GetOperationCount() int { } } if valid { - fmt.Sprint(p) ref := &Reference{ Definition: m.Value, Name: m.Value, diff --git a/index/spec_index_test.go b/index/spec_index_test.go index de235da..1199357 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -4,342 +4,344 @@ package index import ( - "fmt" - "log" - "net/url" - "os" - "os/exec" - "path/filepath" - "testing" + "fmt" + "log" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "testing" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestSpecIndex_ExtractRefsStripe(t *testing.T) { - stripe, _ := os.ReadFile("../test_specs/stripe.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(stripe, &rootNode) + stripe, _ := os.ReadFile("../test_specs/stripe.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(stripe, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, 385) - assert.Equal(t, 537, len(index.allMappedRefs)) - combined := index.GetAllCombinedReferences() - assert.Equal(t, 537, len(combined)) + assert.Len(t, index.allRefs, 385) + assert.Equal(t, 537, len(index.allMappedRefs)) + combined := index.GetAllCombinedReferences() + assert.Equal(t, 537, len(combined)) - assert.Len(t, index.rawSequencedRefs, 1972) - assert.Equal(t, 246, index.pathCount) - assert.Equal(t, 402, index.operationCount) - assert.Equal(t, 537, index.schemaCount) - assert.Equal(t, 0, index.globalTagsCount) - assert.Equal(t, 0, index.globalLinksCount) - assert.Equal(t, 0, index.componentParamCount) - assert.Equal(t, 143, index.operationParamCount) - assert.Equal(t, 88, index.componentsInlineParamDuplicateCount) - assert.Equal(t, 55, index.componentsInlineParamUniqueCount) - assert.Equal(t, 1516, index.enumCount) - assert.Len(t, index.GetAllEnums(), 1516) - assert.Len(t, index.GetPolyAllOfReferences(), 0) - assert.Len(t, index.GetPolyOneOfReferences(), 275) - assert.Len(t, index.GetPolyAnyOfReferences(), 553) - assert.Len(t, index.GetAllReferenceSchemas(), 1972) - assert.NotNil(t, index.GetRootServersNode()) - assert.Len(t, index.GetAllRootServers(), 1) + assert.Len(t, index.rawSequencedRefs, 1972) + assert.Equal(t, 246, index.pathCount) + assert.Equal(t, 402, index.operationCount) + assert.Equal(t, 537, index.schemaCount) + assert.Equal(t, 0, index.globalTagsCount) + assert.Equal(t, 0, index.globalLinksCount) + assert.Equal(t, 0, index.componentParamCount) + assert.Equal(t, 143, index.operationParamCount) + assert.Equal(t, 88, index.componentsInlineParamDuplicateCount) + assert.Equal(t, 55, index.componentsInlineParamUniqueCount) + assert.Equal(t, 1516, index.enumCount) + assert.Len(t, index.GetAllEnums(), 1516) + assert.Len(t, index.GetPolyAllOfReferences(), 0) + assert.Len(t, index.GetPolyOneOfReferences(), 275) + assert.Len(t, index.GetPolyAnyOfReferences(), 553) + assert.Len(t, index.GetAllReferenceSchemas(), 1972) + assert.NotNil(t, index.GetRootServersNode()) + assert.Len(t, index.GetAllRootServers(), 1) - // not required, but flip the circular result switch on and off. - assert.False(t, index.AllowCircularReferenceResolving()) - index.SetAllowCircularReferenceResolving(true) - assert.True(t, index.AllowCircularReferenceResolving()) + // not required, but flip the circular result switch on and off. + assert.False(t, index.AllowCircularReferenceResolving()) + index.SetAllowCircularReferenceResolving(true) + assert.True(t, index.AllowCircularReferenceResolving()) - // simulate setting of circular references, also pointless but needed for coverage. - assert.Nil(t, index.GetCircularReferences()) - index.SetCircularReferences([]*CircularReferenceResult{new(CircularReferenceResult)}) - assert.Len(t, index.GetCircularReferences(), 1) + // simulate setting of circular references, also pointless but needed for coverage. + assert.Nil(t, index.GetCircularReferences()) + index.SetCircularReferences([]*CircularReferenceResult{new(CircularReferenceResult)}) + assert.Len(t, index.GetCircularReferences(), 1) - assert.Len(t, index.GetRefsByLine(), 537) - assert.Len(t, index.GetLinesWithReferences(), 1972) - assert.Len(t, index.GetAllExternalDocuments(), 0) - assert.Len(t, index.GetAllExternalIndexes(), 0) + assert.Len(t, index.GetRefsByLine(), 537) + assert.Len(t, index.GetLinesWithReferences(), 1972) + assert.Len(t, index.GetAllExternalDocuments(), 0) + assert.Len(t, index.GetAllExternalIndexes(), 0) } func TestSpecIndex_Asana(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/asana.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + asana, _ := os.ReadFile("../test_specs/asana.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(asana, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, 152) - assert.Len(t, index.allMappedRefs, 171) - combined := index.GetAllCombinedReferences() - assert.Equal(t, 171, len(combined)) - assert.Equal(t, 118, index.pathCount) - assert.Equal(t, 152, index.operationCount) - assert.Equal(t, 135, index.schemaCount) - assert.Equal(t, 26, index.globalTagsCount) - assert.Equal(t, 0, index.globalLinksCount) - assert.Equal(t, 30, index.componentParamCount) - assert.Equal(t, 107, index.operationParamCount) - assert.Equal(t, 8, index.componentsInlineParamDuplicateCount) - assert.Equal(t, 69, index.componentsInlineParamUniqueCount) + assert.Len(t, index.allRefs, 152) + assert.Len(t, index.allMappedRefs, 171) + combined := index.GetAllCombinedReferences() + assert.Equal(t, 171, len(combined)) + assert.Equal(t, 118, index.pathCount) + assert.Equal(t, 152, index.operationCount) + assert.Equal(t, 135, index.schemaCount) + assert.Equal(t, 26, index.globalTagsCount) + assert.Equal(t, 0, index.globalLinksCount) + assert.Equal(t, 30, index.componentParamCount) + assert.Equal(t, 107, index.operationParamCount) + assert.Equal(t, 8, index.componentsInlineParamDuplicateCount) + assert.Equal(t, 69, index.componentsInlineParamUniqueCount) } func TestSpecIndex_DigitalOcean(t *testing.T) { - do, _ := os.ReadFile("../test_specs/digitalocean.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(do, &rootNode) + do, _ := os.ReadFile("../test_specs/digitalocean.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(do, &rootNode) - baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") - index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - //AllowRemoteLookup: true, - //AllowFileLookup: true, - }) + baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") + index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ + BaseURL: baseURL, + //AllowRemoteLookup: true, + //AllowFileLookup: true, + }) - assert.Len(t, index.GetAllExternalIndexes(), 291) - assert.NotNil(t, index) + assert.Len(t, index.GetAllExternalIndexes(), 291) + assert.NotNil(t, index) } func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) { - // this is a full checkout of the digitalocean API repo. - tmp, _ := os.MkdirTemp("", "openapi") - cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp) - defer os.RemoveAll(filepath.Join(tmp, "openapi")) - err := cmd.Run() - if err != nil { - log.Fatalf("cmd.Run() failed with %s\n", err) - } - spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml")) - doLocal, _ := os.ReadFile(spec) - var rootNode yaml.Node - _ = yaml.Unmarshal(doLocal, &rootNode) + // this is a full checkout of the digitalocean API repo. + tmp, _ := os.MkdirTemp("", "openapi") + cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp) + defer os.RemoveAll(filepath.Join(tmp, "openapi")) + err := cmd.Run() + if err != nil { + log.Fatalf("cmd.Run() failed with %s\n", err) + } + spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml")) + doLocal, _ := os.ReadFile(spec) + var rootNode yaml.Node + _ = yaml.Unmarshal(doLocal, &rootNode) - config := CreateOpenAPIIndexConfig() - config.BasePath = filepath.Join(tmp, "specification") + config := CreateOpenAPIIndexConfig() + config.BasePath = filepath.Join(tmp, "specification") - index := NewSpecIndexWithConfig(&rootNode, config) + index := NewSpecIndexWithConfig(&rootNode, config) - assert.NotNil(t, index) - assert.Len(t, index.GetAllExternalIndexes(), 296) + assert.NotNil(t, index) + assert.Len(t, index.GetAllExternalIndexes(), 296) - ref := index.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml") - assert.NotNil(t, ref) - assert.Equal(t, "operationId", ref[0].Node.Content[0].Value) + ref := index.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml") + assert.NotNil(t, ref) + assert.Equal(t, "operationId", ref[0].Node.Content[0].Value) - ref = index.SearchIndexForReference("examples/ruby/domains_create.yml") - assert.NotNil(t, ref) - assert.Equal(t, "lang", ref[0].Node.Content[0].Value) + ref = index.SearchIndexForReference("examples/ruby/domains_create.yml") + assert.NotNil(t, ref) + assert.Equal(t, "lang", ref[0].Node.Content[0].Value) - ref = index.SearchIndexForReference("../../shared/responses/server_error.yml") - assert.NotNil(t, ref) - assert.Equal(t, "description", ref[0].Node.Content[0].Value) + ref = index.SearchIndexForReference("../../shared/responses/server_error.yml") + assert.NotNil(t, ref) + assert.Equal(t, "description", ref[0].Node.Content[0].Value) - ref = index.SearchIndexForReference("../models/options.yml") - assert.NotNil(t, ref) + ref = index.SearchIndexForReference("../models/options.yml") + assert.NotNil(t, ref) } func TestSpecIndex_DigitalOcean_LookupsNotAllowed(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/digitalocean.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + asana, _ := os.ReadFile("../test_specs/digitalocean.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(asana, &rootNode) - baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") - index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - }) + baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") + index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ + BaseURL: baseURL, + }) - // no lookups allowed, bits have not been set, so there should just be a bunch of errors. - assert.Len(t, index.GetAllExternalIndexes(), 0) - assert.True(t, len(index.GetReferenceIndexErrors()) > 0) + // no lookups allowed, bits have not been set, so there should just be a bunch of errors. + assert.Len(t, index.GetAllExternalIndexes(), 0) + assert.True(t, len(index.GetReferenceIndexErrors()) > 0) } func TestSpecIndex_BaseURLError(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/digitalocean.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + asana, _ := os.ReadFile("../test_specs/digitalocean.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(asana, &rootNode) - // this should fail because the base url is not a valid url and digital ocean won't be able to resolve - // anything. - baseURL, _ := url.Parse("https://githerbs.com/fresh/herbs/for/you") - index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - //AllowRemoteLookup: true, - //AllowFileLookup: true, - }) + // this should fail because the base url is not a valid url and digital ocean won't be able to resolve + // anything. + baseURL, _ := url.Parse("https://githerbs.com/fresh/herbs/for/you") + index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ + BaseURL: baseURL, + //AllowRemoteLookup: true, + //AllowFileLookup: true, + }) - assert.Len(t, index.GetAllExternalIndexes(), 0) + assert.Len(t, index.GetAllExternalIndexes(), 0) } func TestSpecIndex_k8s(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/k8s.json") - var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + asana, _ := os.ReadFile("../test_specs/k8s.json") + var rootNode yaml.Node + _ = yaml.Unmarshal(asana, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, 558) - assert.Equal(t, 563, len(index.allMappedRefs)) - combined := index.GetAllCombinedReferences() - assert.Equal(t, 563, len(combined)) - assert.Equal(t, 436, index.pathCount) - assert.Equal(t, 853, index.operationCount) - assert.Equal(t, 563, index.schemaCount) - assert.Equal(t, 0, index.globalTagsCount) - assert.Equal(t, 58, index.operationTagsCount) - assert.Equal(t, 0, index.globalLinksCount) - assert.Equal(t, 0, index.componentParamCount) - assert.Equal(t, 36, index.operationParamCount) - assert.Equal(t, 26, index.componentsInlineParamDuplicateCount) - assert.Equal(t, 10, index.componentsInlineParamUniqueCount) - assert.Equal(t, 58, index.GetTotalTagsCount()) - assert.Equal(t, 2524, index.GetRawReferenceCount()) + assert.Len(t, index.allRefs, 558) + assert.Equal(t, 563, len(index.allMappedRefs)) + combined := index.GetAllCombinedReferences() + assert.Equal(t, 563, len(combined)) + assert.Equal(t, 436, index.pathCount) + assert.Equal(t, 853, index.operationCount) + assert.Equal(t, 563, index.schemaCount) + assert.Equal(t, 0, index.globalTagsCount) + assert.Equal(t, 58, index.operationTagsCount) + assert.Equal(t, 0, index.globalLinksCount) + assert.Equal(t, 0, index.componentParamCount) + assert.Equal(t, 36, index.operationParamCount) + assert.Equal(t, 26, index.componentsInlineParamDuplicateCount) + assert.Equal(t, 10, index.componentsInlineParamUniqueCount) + assert.Equal(t, 58, index.GetTotalTagsCount()) + assert.Equal(t, 2524, index.GetRawReferenceCount()) } func TestSpecIndex_PetstoreV2(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/petstorev2.json") - var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + asana, _ := os.ReadFile("../test_specs/petstorev2.json") + var rootNode yaml.Node + _ = yaml.Unmarshal(asana, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, 6) - assert.Len(t, index.allMappedRefs, 6) - assert.Equal(t, 14, index.pathCount) - assert.Equal(t, 20, index.operationCount) - assert.Equal(t, 6, index.schemaCount) - assert.Equal(t, 3, index.globalTagsCount) - assert.Equal(t, 3, index.operationTagsCount) - assert.Equal(t, 0, index.globalLinksCount) - assert.Equal(t, 1, index.componentParamCount) - assert.Equal(t, 1, index.GetComponentParameterCount()) - assert.Equal(t, 11, index.operationParamCount) - assert.Equal(t, 5, index.componentsInlineParamDuplicateCount) - assert.Equal(t, 6, index.componentsInlineParamUniqueCount) - assert.Equal(t, 3, index.GetTotalTagsCount()) - assert.Equal(t, 2, len(index.GetSecurityRequirementReferences())) + assert.Len(t, index.allRefs, 6) + assert.Len(t, index.allMappedRefs, 6) + assert.Equal(t, 14, index.pathCount) + assert.Equal(t, 20, index.operationCount) + assert.Equal(t, 6, index.schemaCount) + assert.Equal(t, 3, index.globalTagsCount) + assert.Equal(t, 3, index.operationTagsCount) + assert.Equal(t, 0, index.globalLinksCount) + assert.Equal(t, 1, index.componentParamCount) + assert.Equal(t, 1, index.GetComponentParameterCount()) + assert.Equal(t, 11, index.operationParamCount) + assert.Equal(t, 5, index.componentsInlineParamDuplicateCount) + assert.Equal(t, 6, index.componentsInlineParamUniqueCount) + assert.Equal(t, 3, index.GetTotalTagsCount()) + assert.Equal(t, 2, len(index.GetSecurityRequirementReferences())) } func TestSpecIndex_XSOAR(t *testing.T) { - xsoar, _ := os.ReadFile("../test_specs/xsoar.json") - var rootNode yaml.Node - _ = yaml.Unmarshal(xsoar, &rootNode) + xsoar, _ := os.ReadFile("../test_specs/xsoar.json") + var rootNode yaml.Node + _ = yaml.Unmarshal(xsoar, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, 209) - assert.Equal(t, 85, index.pathCount) - assert.Equal(t, 88, index.operationCount) - assert.Equal(t, 245, index.schemaCount) - assert.Equal(t, 207, len(index.allMappedRefs)) - assert.Equal(t, 0, index.globalTagsCount) - assert.Equal(t, 0, index.operationTagsCount) - assert.Equal(t, 0, index.globalLinksCount) - assert.Len(t, index.GetRootSecurityReferences(), 1) - assert.NotNil(t, index.GetRootSecurityNode()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Len(t, index.allRefs, 209) + assert.Equal(t, 85, index.pathCount) + assert.Equal(t, 88, index.operationCount) + assert.Equal(t, 245, index.schemaCount) + assert.Equal(t, 207, len(index.allMappedRefs)) + assert.Equal(t, 0, index.globalTagsCount) + assert.Equal(t, 0, index.operationTagsCount) + assert.Equal(t, 0, index.globalLinksCount) + assert.Len(t, index.GetRootSecurityReferences(), 1) + assert.NotNil(t, index.GetRootSecurityNode()) } func TestSpecIndex_PetstoreV3(t *testing.T) { - petstore, _ := os.ReadFile("../test_specs/petstorev3.json") - var rootNode yaml.Node - _ = yaml.Unmarshal(petstore, &rootNode) + petstore, _ := os.ReadFile("../test_specs/petstorev3.json") + var rootNode yaml.Node + _ = yaml.Unmarshal(petstore, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, 7) - assert.Len(t, index.allMappedRefs, 7) - assert.Equal(t, 13, index.pathCount) - assert.Equal(t, 19, index.operationCount) - assert.Equal(t, 8, index.schemaCount) - assert.Equal(t, 3, index.globalTagsCount) - assert.Equal(t, 3, index.operationTagsCount) - assert.Equal(t, 0, index.globalLinksCount) - assert.Equal(t, 0, index.componentParamCount) - assert.Equal(t, 9, index.operationParamCount) - assert.Equal(t, 4, index.componentsInlineParamDuplicateCount) - assert.Equal(t, 5, index.componentsInlineParamUniqueCount) - assert.Equal(t, 3, index.GetTotalTagsCount()) - assert.Equal(t, 90, index.GetAllDescriptionsCount()) - assert.Equal(t, 19, index.GetAllSummariesCount()) - assert.Len(t, index.GetAllDescriptions(), 90) - assert.Len(t, index.GetAllSummaries(), 19) + assert.Len(t, index.allRefs, 7) + assert.Len(t, index.allMappedRefs, 7) + assert.Equal(t, 13, index.pathCount) + assert.Equal(t, 19, index.operationCount) + assert.Equal(t, 8, index.schemaCount) + assert.Equal(t, 3, index.globalTagsCount) + assert.Equal(t, 3, index.operationTagsCount) + assert.Equal(t, 0, index.globalLinksCount) + assert.Equal(t, 0, index.componentParamCount) + assert.Equal(t, 9, index.operationParamCount) + assert.Equal(t, 4, index.componentsInlineParamDuplicateCount) + assert.Equal(t, 5, index.componentsInlineParamUniqueCount) + assert.Equal(t, 3, index.GetTotalTagsCount()) + assert.Equal(t, 90, index.GetAllDescriptionsCount()) + assert.Equal(t, 19, index.GetAllSummariesCount()) + assert.Len(t, index.GetAllDescriptions(), 90) + assert.Len(t, index.GetAllSummaries(), 19) } var mappedRefs = 15 func TestSpecIndex_BurgerShop(t *testing.T) { - burgershop, _ := os.ReadFile("../test_specs/burgershop.openapi.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(burgershop, &rootNode) + burgershop, _ := os.ReadFile("../test_specs/burgershop.openapi.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(burgershop, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.allRefs, mappedRefs) - assert.Len(t, index.allMappedRefs, mappedRefs) - assert.Equal(t, mappedRefs, len(index.GetMappedReferences())) - assert.Equal(t, mappedRefs, len(index.GetMappedReferencesSequenced())) + assert.Len(t, index.allRefs, mappedRefs) + assert.Len(t, index.allMappedRefs, mappedRefs) + assert.Equal(t, mappedRefs, len(index.GetMappedReferences())) + assert.Equal(t, mappedRefs, len(index.GetMappedReferencesSequenced())) - assert.Equal(t, 6, index.pathCount) - assert.Equal(t, 6, index.GetPathCount()) + assert.Equal(t, 6, index.pathCount) + assert.Equal(t, 6, index.GetPathCount()) - assert.Equal(t, 6, len(index.GetAllComponentSchemas())) - assert.Equal(t, 56, len(index.GetAllSchemas())) + assert.Equal(t, 6, len(index.GetAllComponentSchemas())) + assert.Equal(t, 56, len(index.GetAllSchemas())) - assert.Equal(t, 34, len(index.GetAllSequencedReferences())) - assert.NotNil(t, index.GetSchemasNode()) - assert.NotNil(t, index.GetParametersNode()) + assert.Equal(t, 34, len(index.GetAllSequencedReferences())) + assert.NotNil(t, index.GetSchemasNode()) + assert.NotNil(t, index.GetParametersNode()) - assert.Equal(t, 5, index.operationCount) - assert.Equal(t, 5, index.GetOperationCount()) + assert.Equal(t, 5, index.operationCount) + assert.Equal(t, 5, index.GetOperationCount()) - assert.Equal(t, 6, index.schemaCount) - assert.Equal(t, 6, index.GetComponentSchemaCount()) + assert.Equal(t, 6, index.schemaCount) + assert.Equal(t, 6, index.GetComponentSchemaCount()) - assert.Equal(t, 2, index.globalTagsCount) - assert.Equal(t, 2, index.GetGlobalTagsCount()) - assert.Equal(t, 2, index.GetTotalTagsCount()) + assert.Equal(t, 2, index.globalTagsCount) + assert.Equal(t, 2, index.GetGlobalTagsCount()) + assert.Equal(t, 2, index.GetTotalTagsCount()) - assert.Equal(t, 2, index.operationTagsCount) - assert.Equal(t, 2, index.GetOperationTagsCount()) + assert.Equal(t, 2, index.operationTagsCount) + assert.Equal(t, 2, index.GetOperationTagsCount()) - assert.Equal(t, 3, index.globalLinksCount) - assert.Equal(t, 3, index.GetGlobalLinksCount()) + assert.Equal(t, 3, index.globalLinksCount) + assert.Equal(t, 3, index.GetGlobalLinksCount()) - assert.Equal(t, 1, index.globalCallbacksCount) - assert.Equal(t, 1, index.GetGlobalCallbacksCount()) + assert.Equal(t, 1, index.globalCallbacksCount) + assert.Equal(t, 1, index.GetGlobalCallbacksCount()) - assert.Equal(t, 2, index.componentParamCount) - assert.Equal(t, 2, index.GetComponentParameterCount()) + assert.Equal(t, 2, index.componentParamCount) + assert.Equal(t, 2, index.GetComponentParameterCount()) - assert.Equal(t, 4, index.operationParamCount) - assert.Equal(t, 4, index.GetOperationsParameterCount()) + assert.Equal(t, 4, index.operationParamCount) + assert.Equal(t, 4, index.GetOperationsParameterCount()) - assert.Equal(t, 0, index.componentsInlineParamDuplicateCount) - assert.Equal(t, 0, index.GetInlineDuplicateParamCount()) + assert.Equal(t, 0, index.componentsInlineParamDuplicateCount) + assert.Equal(t, 0, index.GetInlineDuplicateParamCount()) - assert.Equal(t, 2, index.componentsInlineParamUniqueCount) - assert.Equal(t, 2, index.GetInlineUniqueParamCount()) + assert.Equal(t, 2, index.componentsInlineParamUniqueCount) + assert.Equal(t, 2, index.GetInlineUniqueParamCount()) - assert.Equal(t, 1, len(index.GetAllRequestBodies())) - assert.NotNil(t, index.GetRootNode()) - assert.NotNil(t, index.GetGlobalTagsNode()) - assert.NotNil(t, index.GetPathsNode()) - assert.NotNil(t, index.GetDiscoveredReferences()) - assert.Equal(t, 1, len(index.GetPolyReferences())) - assert.NotNil(t, index.GetOperationParameterReferences()) - assert.Equal(t, 3, len(index.GetAllSecuritySchemes())) - assert.Equal(t, 2, len(index.GetAllParameters())) - assert.Equal(t, 1, len(index.GetAllResponses())) - assert.Equal(t, 2, len(index.GetInlineOperationDuplicateParameters())) - assert.Equal(t, 0, len(index.GetReferencesWithSiblings())) - assert.Equal(t, mappedRefs, len(index.GetAllReferences())) - assert.Equal(t, 0, len(index.GetOperationParametersIndexErrors())) - assert.Equal(t, 5, len(index.GetAllPaths())) - assert.Equal(t, 5, len(index.GetOperationTags())) - assert.Equal(t, 3, len(index.GetAllParametersFromOperations())) + assert.Equal(t, 1, len(index.GetAllRequestBodies())) + assert.NotNil(t, index.GetRootNode()) + assert.NotNil(t, index.GetGlobalTagsNode()) + assert.NotNil(t, index.GetPathsNode()) + assert.NotNil(t, index.GetDiscoveredReferences()) + assert.Equal(t, 1, len(index.GetPolyReferences())) + assert.NotNil(t, index.GetOperationParameterReferences()) + assert.Equal(t, 3, len(index.GetAllSecuritySchemes())) + assert.Equal(t, 2, len(index.GetAllParameters())) + assert.Equal(t, 1, len(index.GetAllResponses())) + assert.Equal(t, 2, len(index.GetInlineOperationDuplicateParameters())) + assert.Equal(t, 0, len(index.GetReferencesWithSiblings())) + assert.Equal(t, mappedRefs, len(index.GetAllReferences())) + assert.Equal(t, 0, len(index.GetOperationParametersIndexErrors())) + assert.Equal(t, 5, len(index.GetAllPaths())) + assert.Equal(t, 5, len(index.GetOperationTags())) + assert.Equal(t, 3, len(index.GetAllParametersFromOperations())) } func TestSpecIndex_GetAllParametersFromOperations(t *testing.T) { - yml := `openapi: 3.0.0 + yml := `openapi: 3.0.0 servers: - url: http://localhost:8080 paths: @@ -355,47 +357,47 @@ paths: schema: type: string` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 1, len(index.GetAllParametersFromOperations())) - assert.Equal(t, 1, len(index.GetOperationParametersIndexErrors())) + assert.Equal(t, 1, len(index.GetAllParametersFromOperations())) + assert.Equal(t, 1, len(index.GetOperationParametersIndexErrors())) } func TestSpecIndex_BurgerShop_AllTheComponents(t *testing.T) { - burgershop, _ := os.ReadFile("../test_specs/all-the-components.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(burgershop, &rootNode) + burgershop, _ := os.ReadFile("../test_specs/all-the-components.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(burgershop, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 1, len(index.GetAllHeaders())) - assert.Equal(t, 1, len(index.GetAllLinks())) - assert.Equal(t, 1, len(index.GetAllCallbacks())) - assert.Equal(t, 1, len(index.GetAllExamples())) - assert.Equal(t, 1, len(index.GetAllResponses())) - assert.Equal(t, 2, len(index.GetAllRootServers())) - assert.Equal(t, 2, len(index.GetAllOperationsServers())) + assert.Equal(t, 1, len(index.GetAllHeaders())) + assert.Equal(t, 1, len(index.GetAllLinks())) + assert.Equal(t, 1, len(index.GetAllCallbacks())) + assert.Equal(t, 1, len(index.GetAllExamples())) + assert.Equal(t, 1, len(index.GetAllResponses())) + assert.Equal(t, 2, len(index.GetAllRootServers())) + assert.Equal(t, 2, len(index.GetAllOperationsServers())) } func TestSpecIndex_SwaggerResponses(t *testing.T) { - yml := `swagger: 2.0 + yml := `swagger: 2.0 responses: niceResponse: description: hi` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 1, len(index.GetAllResponses())) + assert.Equal(t, 1, len(index.GetAllResponses())) } func TestSpecIndex_NoNameParam(t *testing.T) { - yml := `paths: + yml := `paths: /users/{id}: parameters: - in: path @@ -407,100 +409,165 @@ func TestSpecIndex_NoNameParam(t *testing.T) { name: id - in: query` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 2, len(index.GetOperationParametersIndexErrors())) + assert.Equal(t, 2, len(index.GetOperationParametersIndexErrors())) } func TestSpecIndex_NoRoot(t *testing.T) { - index := NewSpecIndex(nil) - refs := index.ExtractRefs(nil, nil, nil, 0, false, "") - docs := index.ExtractExternalDocuments(nil) - assert.Nil(t, docs) - assert.Nil(t, refs) - assert.Nil(t, index.FindComponent("nothing", nil)) - assert.Equal(t, -1, index.GetOperationCount()) - assert.Equal(t, -1, index.GetPathCount()) - assert.Equal(t, -1, index.GetGlobalTagsCount()) - assert.Equal(t, -1, index.GetOperationTagsCount()) - assert.Equal(t, -1, index.GetTotalTagsCount()) - assert.Equal(t, -1, index.GetOperationsParameterCount()) - assert.Equal(t, -1, index.GetComponentParameterCount()) - assert.Equal(t, -1, index.GetComponentSchemaCount()) - assert.Equal(t, -1, index.GetGlobalLinksCount()) + index := NewSpecIndex(nil) + refs := index.ExtractRefs(nil, nil, nil, 0, false, "") + docs := index.ExtractExternalDocuments(nil) + assert.Nil(t, docs) + assert.Nil(t, refs) + assert.Nil(t, index.FindComponent("nothing", nil)) + assert.Equal(t, -1, index.GetOperationCount()) + assert.Equal(t, -1, index.GetPathCount()) + assert.Equal(t, -1, index.GetGlobalTagsCount()) + assert.Equal(t, -1, index.GetOperationTagsCount()) + assert.Equal(t, -1, index.GetTotalTagsCount()) + assert.Equal(t, -1, index.GetOperationsParameterCount()) + assert.Equal(t, -1, index.GetComponentParameterCount()) + assert.Equal(t, -1, index.GetComponentSchemaCount()) + assert.Equal(t, -1, index.GetGlobalLinksCount()) +} + +func test_buildMixedRefServer() *httptest.Server { + + bs, _ := os.ReadFile("../test_specs/burgershop.openapi.yaml") + return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + if req.URL.String() == "/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml" { + rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + _, _ = rw.Write(bs) + return + } + + _, _ = rw.Write([]byte(`OK`)) + })) } func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { - spec, _ := os.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(spec, &rootNode) - cwd, _ := os.Getwd() + // create a test server. + server := test_buildMixedRefServer() + defer server.Close() - index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - //AllowRemoteLookup: true, - // AllowFileLookup: true, - BasePath: cwd, - }) + // create a new config that allows local and remote to be mixed up. + cf := CreateOpenAPIIndexConfig() + cf.AvoidBuildIndex = true + cf.AllowRemoteLookup = true + cf.AvoidCircularReferenceCheck = true + cf.BasePath = "../test_specs" - assert.Len(t, index.allRefs, 5) - assert.Len(t, index.allMappedRefs, 5) - assert.Equal(t, 5, index.GetPathCount()) - assert.Equal(t, 5, index.GetOperationCount()) - assert.Equal(t, 1, index.GetComponentSchemaCount()) - assert.Equal(t, 2, index.GetGlobalTagsCount()) - assert.Equal(t, 3, index.GetTotalTagsCount()) - assert.Equal(t, 2, index.GetOperationTagsCount()) - assert.Equal(t, 0, index.GetGlobalLinksCount()) - assert.Equal(t, 0, index.GetComponentParameterCount()) - assert.Equal(t, 2, index.GetOperationsParameterCount()) - assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) - assert.Equal(t, 1, index.GetInlineUniqueParamCount()) + // setting this baseURL will override the base + //cf.BaseURL, _ = url.Parse(server.URL) + + cFile := "../test_specs/mixedref-burgershop.openapi.yaml" + yml, _ := os.ReadFile(cFile) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + // create a new rolodex + rolo := NewRolodex(cf) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&rootNode) + + // create a new remote fs and set the config for indexing. + //remoteFS, _ := NewRemoteFSWithRootURL(server.URL) + remoteFS, _ := NewRemoteFS() + remoteFS.SetIndexConfig(cf) + + // set our remote handler func + + c := http.Client{} + + remoteFS.RemoteHandlerFunc = c.Get + + // configure the local filesystem. + fsCfg := LocalFSConfig{ + BaseDirectory: cf.BasePath, + FileFilters: []string{"burgershop.openapi.yaml"}, + DirFS: os.DirFS(cf.BasePath), + } + + // create a new local filesystem. + fileFS, err := NewLocalFSWithConfig(&fsCfg) + assert.NoError(t, err) + + // add file systems to the rolodex + rolo.AddLocalFS(cf.BasePath, fileFS) + rolo.AddRemoteFS(server.URL, remoteFS) + + // index the rolodex. + indexedErr := rolo.IndexTheRolodex() + rolo.BuildIndexes() + + assert.NoError(t, indexedErr) + + index := rolo.GetRootIndex() + rolo.CheckForCircularReferences() + + assert.Len(t, index.allRefs, 5) + assert.Len(t, index.allMappedRefs, 5) + assert.Equal(t, 5, index.GetPathCount()) + assert.Equal(t, 5, index.GetOperationCount()) + assert.Equal(t, 1, index.GetComponentSchemaCount()) + assert.Equal(t, 2, index.GetGlobalTagsCount()) + assert.Equal(t, 3, index.GetTotalTagsCount()) + assert.Equal(t, 2, index.GetOperationTagsCount()) + assert.Equal(t, 0, index.GetGlobalLinksCount()) + assert.Equal(t, 0, index.GetComponentParameterCount()) + assert.Equal(t, 2, index.GetOperationsParameterCount()) + assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) + assert.Equal(t, 1, index.GetInlineUniqueParamCount()) + assert.Len(t, index.refErrors, 0) + assert.Len(t, index.GetCircularReferences(), 0) } func TestSpecIndex_TestEmptyBrokenReferences(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/badref-burgershop.openapi.yaml") - var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + asana, _ := os.ReadFile("../test_specs/badref-burgershop.openapi.yaml") + var rootNode yaml.Node + _ = yaml.Unmarshal(asana, &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 5, index.GetPathCount()) - assert.Equal(t, 5, index.GetOperationCount()) - assert.Equal(t, 5, index.GetComponentSchemaCount()) - assert.Equal(t, 2, index.GetGlobalTagsCount()) - assert.Equal(t, 3, index.GetTotalTagsCount()) - assert.Equal(t, 2, index.GetOperationTagsCount()) - assert.Equal(t, 2, index.GetGlobalLinksCount()) - assert.Equal(t, 0, index.GetComponentParameterCount()) - assert.Equal(t, 2, index.GetOperationsParameterCount()) - assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) - assert.Equal(t, 1, index.GetInlineUniqueParamCount()) - assert.Len(t, index.refErrors, 7) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Equal(t, 5, index.GetPathCount()) + assert.Equal(t, 5, index.GetOperationCount()) + assert.Equal(t, 5, index.GetComponentSchemaCount()) + assert.Equal(t, 2, index.GetGlobalTagsCount()) + assert.Equal(t, 3, index.GetTotalTagsCount()) + assert.Equal(t, 2, index.GetOperationTagsCount()) + assert.Equal(t, 2, index.GetGlobalLinksCount()) + assert.Equal(t, 0, index.GetComponentParameterCount()) + assert.Equal(t, 2, index.GetOperationsParameterCount()) + assert.Equal(t, 1, index.GetInlineDuplicateParamCount()) + assert.Equal(t, 1, index.GetInlineUniqueParamCount()) + assert.Len(t, index.refErrors, 7) } func TestTagsNoDescription(t *testing.T) { - yml := `tags: + yml := `tags: - name: one - name: two - three: three` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 3, index.GetGlobalTagsCount()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Equal(t, 3, index.GetGlobalTagsCount()) } func TestGlobalCallbacksNoIndexTest(t *testing.T) { - idx := new(SpecIndex) - assert.Equal(t, -1, idx.GetGlobalCallbacksCount()) + idx := new(SpecIndex) + assert.Equal(t, -1, idx.GetGlobalCallbacksCount()) } func TestMultipleCallbacksPerOperationVerb(t *testing.T) { - yml := `components: + yml := `components: callbacks: callbackA: "{$request.query.queryUrl}": @@ -529,15 +596,15 @@ paths: callbackA: $ref: '#/components/callbacks/CallbackA'` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, 4, index.GetGlobalCallbacksCount()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Equal(t, 4, index.GetGlobalCallbacksCount()) } func TestSpecIndex_ExtractComponentsFromRefs(t *testing.T) { - yml := `components: + yml := `components: schemas: pizza: properties: @@ -546,15 +613,15 @@ func TestSpecIndex_ExtractComponentsFromRefs(t *testing.T) { something: description: something` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.GetReferenceIndexErrors(), 1) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Len(t, index.GetReferenceIndexErrors(), 1) } func TestSpecIndex_FindComponent_WithACrazyAssPath(t *testing.T) { - yml := `paths: + yml := `paths: /crazy/ass/references: get: parameters: @@ -588,19 +655,19 @@ func TestSpecIndex_FindComponent_WithACrazyAssPath(t *testing.T) { $ref: "#/paths/~1crazy~1ass~1references/get/responses/404/content/application~1xml;%20charset=utf-8/schema" description: Not Found.` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Equal(t, "#/paths/~1crazy~1ass~1references/get/parameters/0", - index.FindComponent("#/paths/~1crazy~1ass~1references/get/responses/404/content/application~1xml;%20charset=utf-8/schema", nil).Node.Content[1].Value) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Equal(t, "#/paths/~1crazy~1ass~1references/get/parameters/0", + index.FindComponent("#/paths/~1crazy~1ass~1references/get/responses/404/content/application~1xml;%20charset=utf-8/schema", nil).Node.Content[1].Value) - assert.Equal(t, "a param", - index.FindComponent("#/paths/~1crazy~1ass~1references/get/parameters/0", nil).Node.Content[1].Value) + assert.Equal(t, "a param", + index.FindComponent("#/paths/~1crazy~1ass~1references/get/parameters/0", nil).Node.Content[1].Value) } func TestSpecIndex_FindComponenth(t *testing.T) { - yml := `components: + yml := `components: schemas: pizza: properties: @@ -609,15 +676,15 @@ func TestSpecIndex_FindComponenth(t *testing.T) { something: description: something` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Nil(t, index.FindComponent("I-do-not-exist", nil)) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Nil(t, index.FindComponent("I-do-not-exist", nil)) } func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) { - yml := `components: + yml := `components: schemas: pizza: properties: @@ -626,187 +693,187 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) { something: description: something` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Nil(t, index.performExternalLookup(nil)) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + assert.Nil(t, index.lookupRolodex(nil)) } func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} - _, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something") - assert.Error(t, err) + index := new(SpecIndex) + index.seenRemoteSources = make(map[string]*yaml.Node) + index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} + _, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something") + assert.Error(t, err) } func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") - assert.Error(t, err) - assert.Nil(t, a) - assert.Nil(t, b) + index := new(SpecIndex) + index.seenRemoteSources = make(map[string]*yaml.Node) + index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} + a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") + assert.Error(t, err) + assert.Nil(t, a) + assert.Nil(t, b) } // Discovered in issue https://github.com/pb33f/libopenapi/issues/37 func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") - assert.NoError(t, err) - assert.NotNil(t, a) - assert.NotNil(t, b) + index := new(SpecIndex) + index.seenRemoteSources = make(map[string]*yaml.Node) + index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} + a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") + assert.NoError(t, err) + assert.NotNil(t, a) + assert.NotNil(t, b) } // Discovered in issue https://github.com/daveshanley/vacuum/issues/225 func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { - cwd, _ := os.Getwd() - index := new(SpecIndex) - index.config = &SpecIndexConfig{BasePath: cwd} + cwd, _ := os.Getwd() + index := new(SpecIndex) + index.config = &SpecIndexConfig{BasePath: cwd} - _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") + _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) + defer os.Remove("coffee-time.yaml") - index.seenRemoteSources = make(map[string]*yaml.Node) - a, b, err := index.lookupFileReference("coffee-time.yaml") - assert.NoError(t, err) - assert.NotNil(t, a) - assert.NotNil(t, b) + index.seenRemoteSources = make(map[string]*yaml.Node) + a, b, err := index.lookupFileReference("coffee-time.yaml") + assert.NoError(t, err) + assert.NotNil(t, a) + assert.NotNil(t, b) } func TestSpecIndex_CheckBadURLRef(t *testing.T) { - yml := `openapi: 3.1.0 + yml := `openapi: 3.1.0 paths: /cakes: post: parameters: - $ref: 'httpsss://badurl'` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, index.refErrors, 2) + assert.Len(t, index.refErrors, 2) } func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { - yml := `openapi: 3.1.0 + yml := `openapi: 3.1.0 paths: /cakes: post: parameters: - $ref: 'httpsss://badurl'` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - c := CreateClosedAPIIndexConfig() - idx := NewSpecIndexWithConfig(&rootNode, c) + c := CreateClosedAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, c) - assert.Len(t, idx.refErrors, 2) - assert.Equal(t, "remote lookups are not permitted, "+ - "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error()) + assert.Len(t, idx.refErrors, 2) + assert.Equal(t, "remote lookups are not permitted, "+ + "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error()) } func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) { - _ = os.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") + _ = os.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) + defer os.Remove("coffee-time.yaml") - yml := `openapi: 3.0.3 + yml := `openapi: 3.0.3 paths: /cakes: post: parameters: - $ref: 'coffee-time.yaml'` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.NotNil(t, index.GetAllParametersFromOperations()["/cakes"]["post"]["coffee-time.yaml"][0].Node) + assert.NotNil(t, index.GetAllParametersFromOperations()["/cakes"]["post"]["coffee-time.yaml"][0].Node) } func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) { - index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{ - //AllowRemoteLookup: true, - }) - index.seenRemoteSources = make(map[string]*yaml.Node) - a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey") - assert.Error(t, err) - assert.Nil(t, a) - assert.Nil(t, b) + index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{ + //AllowRemoteLookup: true, + }) + index.seenRemoteSources = make(map[string]*yaml.Node) + a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey") + assert.Error(t, err) + assert.Nil(t, a) + assert.Nil(t, b) } func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) { - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - _, _, err := index.lookupFileReference("not-a-reference") - assert.Error(t, err) + index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) + _, _, err := index.lookupFileReference("not-a-reference") + assert.Error(t, err) } func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) { - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{} - _, _, err := index.lookupFileReference("magic-money-file.json#something") - assert.Error(t, err) + index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) + index.seenRemoteSources = make(map[string]*yaml.Node) + index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{} + _, _, err := index.lookupFileReference("magic-money-file.json#something") + assert.Error(t, err) } func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) { - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - _, _, err := index.lookupFileReference("chickers.json#no-rice") - assert.Error(t, err) + index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) + _, _, err := index.lookupFileReference("chickers.json#no-rice") + assert.Error(t, err) } func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) { - _ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664) - defer os.Remove("chickers.yaml") - var root yaml.Node - index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig()) - _, _, err := index.lookupFileReference("chickers.yaml#no-rice") - assert.Error(t, err) + _ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664) + defer os.Remove("chickers.yaml") + var root yaml.Node + index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig()) + _, _, err := index.lookupFileReference("chickers.yaml#no-rice") + assert.Error(t, err) } func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) { - _ = os.WriteFile("embie.yaml", []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy"), 0o664) - defer os.Remove("embie.yaml") + _ = os.WriteFile("embie.yaml", []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy"), 0o664) + defer os.Remove("embie.yaml") - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - index.seenRemoteSources = make(map[string]*yaml.Node) - k, doc, err := index.lookupFileReference("embie.yaml#/.naughty") - assert.NoError(t, err) - assert.NotNil(t, doc) - assert.Nil(t, k) + index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) + index.seenRemoteSources = make(map[string]*yaml.Node) + k, doc, err := index.lookupFileReference("embie.yaml#/.naughty") + assert.NoError(t, err) + assert.NotNil(t, doc) + assert.Nil(t, k) } func TestSpecIndex_lookupFileReference(t *testing.T) { - _ = os.WriteFile("fox.yaml", []byte("good:\n - puppy: dog\n - puppy: forever-more"), 0o664) - defer os.Remove("fox.yaml") + _ = os.WriteFile("fox.yaml", []byte("good:\n - puppy: dog\n - puppy: forever-more"), 0o664) + defer os.Remove("fox.yaml") - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - index.seenRemoteSources = make(map[string]*yaml.Node) - k, doc, err := index.lookupFileReference("fox.yaml#/good") - assert.NoError(t, err) - assert.NotNil(t, doc) - assert.NotNil(t, k) + index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) + index.seenRemoteSources = make(map[string]*yaml.Node) + k, doc, err := index.lookupFileReference("fox.yaml#/good") + assert.NoError(t, err) + assert.NotNil(t, doc) + assert.NotNil(t, k) } func TestSpecIndex_parameterReferencesHavePaths(t *testing.T) { - _ = os.WriteFile("paramour.yaml", []byte(`components: + _ = os.WriteFile("paramour.yaml", []byte(`components: parameters: param3: name: param3 in: query schema: type: string`), 0o664) - defer os.Remove("paramour.yaml") + defer os.Remove("paramour.yaml") - yml := `paths: + yml := `paths: /: parameters: - $ref: '#/components/parameters/param1' @@ -833,35 +900,35 @@ components: schema: type: string` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - params := index.GetAllParametersFromOperations() + params := index.GetAllParametersFromOperations() - if assert.Contains(t, params, "/") { - if assert.Contains(t, params["/"], "top") { - if assert.Contains(t, params["/"]["top"], "#/components/parameters/param1") { - assert.Equal(t, "$.components.parameters.param1", params["/"]["top"]["#/components/parameters/param1"][0].Path) - } - if assert.Contains(t, params["/"]["top"], "paramour.yaml#/components/parameters/param3") { - assert.Equal(t, "$.components.parameters.param3", params["/"]["top"]["paramour.yaml#/components/parameters/param3"][0].Path) - } - } - if assert.Contains(t, params["/"], "get") { - if assert.Contains(t, params["/"]["get"], "#/components/parameters/param2") { - assert.Equal(t, "$.components.parameters.param2", params["/"]["get"]["#/components/parameters/param2"][0].Path) - } - if assert.Contains(t, params["/"]["get"], "test") { - assert.Equal(t, "$.paths./.get.parameters[2]", params["/"]["get"]["test"][0].Path) - } - } - } + if assert.Contains(t, params, "/") { + if assert.Contains(t, params["/"], "top") { + if assert.Contains(t, params["/"]["top"], "#/components/parameters/param1") { + assert.Equal(t, "$.components.parameters.param1", params["/"]["top"]["#/components/parameters/param1"][0].Path) + } + if assert.Contains(t, params["/"]["top"], "paramour.yaml#/components/parameters/param3") { + assert.Equal(t, "$.components.parameters.param3", params["/"]["top"]["paramour.yaml#/components/parameters/param3"][0].Path) + } + } + if assert.Contains(t, params["/"], "get") { + if assert.Contains(t, params["/"]["get"], "#/components/parameters/param2") { + assert.Equal(t, "$.components.parameters.param2", params["/"]["get"]["#/components/parameters/param2"][0].Path) + } + if assert.Contains(t, params["/"]["get"], "test") { + assert.Equal(t, "$.paths./.get.parameters[2]", params["/"]["get"]["test"][0].Path) + } + } + } } func TestSpecIndex_serverReferencesHaveParentNodesAndPaths(t *testing.T) { - yml := `servers: + yml := `servers: - url: https://api.example.com/v1 paths: /: @@ -871,59 +938,59 @@ paths: servers: - url: https://api.example.com/v3` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - rootServers := index.GetAllRootServers() + rootServers := index.GetAllRootServers() - for i, server := range rootServers { - assert.NotNil(t, server.ParentNode) - assert.Equal(t, fmt.Sprintf("$.servers[%d]", i), server.Path) - } + for i, server := range rootServers { + assert.NotNil(t, server.ParentNode) + assert.Equal(t, fmt.Sprintf("$.servers[%d]", i), server.Path) + } - opServers := index.GetAllOperationsServers() + opServers := index.GetAllOperationsServers() - for path, ops := range opServers { - for op, servers := range ops { - for i, server := range servers { - assert.NotNil(t, server.ParentNode) + for path, ops := range opServers { + for op, servers := range ops { + for i, server := range servers { + assert.NotNil(t, server.ParentNode) - opPath := fmt.Sprintf(".%s", op) - if op == "top" { - opPath = "" - } + opPath := fmt.Sprintf(".%s", op) + if op == "top" { + opPath = "" + } - assert.Equal(t, fmt.Sprintf("$.paths.%s%s.servers[%d]", path, opPath, i), server.Path) - } - } - } + assert.Equal(t, fmt.Sprintf("$.paths.%s%s.servers[%d]", path, opPath, i), server.Path) + } + } + } } func TestSpecIndex_schemaComponentsHaveParentsAndPaths(t *testing.T) { - yml := `components: + yml := `components: schemas: Pet: type: object Dog: type: object` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - schemas := index.GetAllSchemas() + schemas := index.GetAllSchemas() - for _, schema := range schemas { - assert.NotNil(t, schema.ParentNode) - assert.Equal(t, fmt.Sprintf("$.components.schemas.%s", schema.Name), schema.Path) - } + for _, schema := range schemas { + assert.NotNil(t, schema.ParentNode) + assert.Equal(t, fmt.Sprintf("$.components.schemas.%s", schema.Name), schema.Path) + } } func TestSpecIndex_ParamsWithDuplicateNamesButUniqueInTypes(t *testing.T) { - yml := `openapi: 3.1.0 + yml := `openapi: 3.1.0 info: title: Test version: 0.0.1 @@ -959,19 +1026,19 @@ paths: "200": description: OK` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, idx.paramAllRefs, 4) - assert.Len(t, idx.paramInlineDuplicateNames, 2) - assert.Len(t, idx.operationParamErrors, 0) - assert.Len(t, idx.refErrors, 0) + assert.Len(t, idx.paramAllRefs, 4) + assert.Len(t, idx.paramInlineDuplicateNames, 2) + assert.Len(t, idx.operationParamErrors, 0) + assert.Len(t, idx.refErrors, 0) } func TestSpecIndex_ParamsWithDuplicateNamesAndSameInTypes(t *testing.T) { - yml := `openapi: 3.1.0 + yml := `openapi: 3.1.0 info: title: Test version: 0.0.1 @@ -1007,19 +1074,19 @@ paths: "200": description: OK` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Len(t, idx.paramAllRefs, 3) - assert.Len(t, idx.paramInlineDuplicateNames, 2) - assert.Len(t, idx.operationParamErrors, 1) - assert.Len(t, idx.refErrors, 0) + assert.Len(t, idx.paramAllRefs, 3) + assert.Len(t, idx.paramInlineDuplicateNames, 2) + assert.Len(t, idx.operationParamErrors, 1) + assert.Len(t, idx.refErrors, 0) } func TestSpecIndex_foundObjectsWithProperties(t *testing.T) { - yml := `paths: + yml := `paths: /test: get: responses: @@ -1047,64 +1114,64 @@ components: type: object additionalProperties: true` - var rootNode yaml.Node - yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + yaml.Unmarshal([]byte(yml), &rootNode) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - objects := index.GetAllObjectsWithProperties() - assert.Len(t, objects, 3) + objects := index.GetAllObjectsWithProperties() + assert.Len(t, objects, 3) } // Example of how to load in an OpenAPI Specification and index it. func ExampleNewSpecIndex() { - // define a rootNode to hold our raw spec AST. - var rootNode yaml.Node + // define a rootNode to hold our raw spec AST. + var rootNode yaml.Node - // load in the stripe OpenAPI specification into bytes (it's pretty meaty) - stripeSpec, _ := os.ReadFile("../test_specs/stripe.yaml") + // load in the stripe OpenAPI specification into bytes (it's pretty meaty) + stripeSpec, _ := os.ReadFile("../test_specs/stripe.yaml") - // unmarshal spec into our rootNode - _ = yaml.Unmarshal(stripeSpec, &rootNode) + // unmarshal spec into our rootNode + _ = yaml.Unmarshal(stripeSpec, &rootNode) - // create a new specification index. - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + // create a new specification index. + index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - // print out some statistics - fmt.Printf("There are %d references\n"+ - "%d paths\n"+ - "%d operations\n"+ - "%d component schemas\n"+ - "%d reference schemas\n"+ - "%d inline schemas\n"+ - "%d inline schemas that are objects or arrays\n"+ - "%d total schemas\n"+ - "%d enums\n"+ - "%d polymorphic references", - len(index.GetAllCombinedReferences()), - len(index.GetAllPaths()), - index.GetOperationCount(), - len(index.GetAllComponentSchemas()), - len(index.GetAllReferenceSchemas()), - len(index.GetAllInlineSchemas()), - len(index.GetAllInlineSchemaObjects()), - len(index.GetAllSchemas()), - len(index.GetAllEnums()), - len(index.GetPolyOneOfReferences())+len(index.GetPolyAnyOfReferences())) - // Output: There are 537 references - // 246 paths - // 402 operations - // 537 component schemas - // 1972 reference schemas - // 11749 inline schemas - // 2612 inline schemas that are objects or arrays - // 14258 total schemas - // 1516 enums - // 828 polymorphic references + // print out some statistics + fmt.Printf("There are %d references\n"+ + "%d paths\n"+ + "%d operations\n"+ + "%d component schemas\n"+ + "%d reference schemas\n"+ + "%d inline schemas\n"+ + "%d inline schemas that are objects or arrays\n"+ + "%d total schemas\n"+ + "%d enums\n"+ + "%d polymorphic references", + len(index.GetAllCombinedReferences()), + len(index.GetAllPaths()), + index.GetOperationCount(), + len(index.GetAllComponentSchemas()), + len(index.GetAllReferenceSchemas()), + len(index.GetAllInlineSchemas()), + len(index.GetAllInlineSchemaObjects()), + len(index.GetAllSchemas()), + len(index.GetAllEnums()), + len(index.GetPolyOneOfReferences())+len(index.GetPolyAnyOfReferences())) + // Output: There are 537 references + // 246 paths + // 402 operations + // 537 component schemas + // 1972 reference schemas + // 11749 inline schemas + // 2612 inline schemas that are objects or arrays + // 14258 total schemas + // 1516 enums + // 828 polymorphic references } func TestSpecIndex_GetAllPathsHavePathAndParent(t *testing.T) { - yml := `openapi: 3.1.0 + yml := `openapi: 3.1.0 info: title: Test version: 0.0.1 @@ -1130,19 +1197,19 @@ paths: "200": description: OK` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) - idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - paths := idx.GetAllPaths() + paths := idx.GetAllPaths() - assert.Equal(t, "$.paths./test.get", paths["/test"]["get"].Path) - assert.Equal(t, 9, paths["/test"]["get"].ParentNode.Line) - assert.Equal(t, "$.paths./test.post", paths["/test"]["post"].Path) - assert.Equal(t, 13, paths["/test"]["post"].ParentNode.Line) - assert.Equal(t, "$.paths./test2.delete", paths["/test2"]["delete"].Path) - assert.Equal(t, 18, paths["/test2"]["delete"].ParentNode.Line) - assert.Equal(t, "$.paths./test2.put", paths["/test2"]["put"].Path) - assert.Equal(t, 22, paths["/test2"]["put"].ParentNode.Line) + assert.Equal(t, "$.paths./test.get", paths["/test"]["get"].Path) + assert.Equal(t, 9, paths["/test"]["get"].ParentNode.Line) + assert.Equal(t, "$.paths./test.post", paths["/test"]["post"].Path) + assert.Equal(t, 13, paths["/test"]["post"].ParentNode.Line) + assert.Equal(t, "$.paths./test2.delete", paths["/test2"]["delete"].Path) + assert.Equal(t, 18, paths["/test2"]["delete"].ParentNode.Line) + assert.Equal(t, "$.paths./test2.put", paths["/test2"]["put"].Path) + assert.Equal(t, 22, paths["/test2"]["put"].ParentNode.Line) } diff --git a/index/utility_methods.go b/index/utility_methods.go index 109b48d..133c304 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -441,9 +441,5 @@ func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bo } } - if strings.HasSuffix(p, "/") { - p = p[:len(p)-1] - } - return p - + return strings.TrimSuffix(p, "/") } diff --git a/test_specs/mixedref-burgershop.openapi.yaml b/test_specs/mixedref-burgershop.openapi.yaml index a722ee0..001de5d 100644 --- a/test_specs/mixedref-burgershop.openapi.yaml +++ b/test_specs/mixedref-burgershop.openapi.yaml @@ -234,7 +234,7 @@ paths: content: application/json: schema: - $ref: 'https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml' + $ref: 'https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml#/components/schemas/Error' components: schemas: Error: