From b295e8fd5cee9b953389be9b38c54d60de969b15 Mon Sep 17 00:00:00 2001 From: quobix Date: Thu, 19 Oct 2023 15:18:33 -0400 Subject: [PATCH] bashing through usecases and updating tests as we go. so many things that can go wrong. have to catch them all. Signed-off-by: quobix --- index/extract_refs.go | 1 + index/find_component.go | 19 +- index/find_component_test.go | 2 +- index/index_model.go | 22 +- index/index_utils.go | 4 +- index/rolodex.go | 25 +- index/rolodex_file_loader.go | 12 + index/rolodex_remote_loader.go | 49 +++- index/search_index.go | 28 ++- index/spec_index.go | 8 +- index/spec_index_test.go | 415 +++++++++++++++++++++------------ 11 files changed, 404 insertions(+), 181 deletions(-) diff --git a/index/extract_refs.go b/index/extract_refs.go index 35be220..46e2300 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -189,6 +189,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, var p string uri := strings.Split(value, "#/") + if strings.HasPrefix(value, "http") || filepath.IsAbs(value) { if len(uri) == 2 { _, p = utils.ConvertComponentIdIntoFriendlyPathSearch(fmt.Sprintf("#/%s", uri[1])) diff --git a/index/find_component.go b/index/find_component.go index 9e2ebc2..d4c2c49 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -62,7 +62,16 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re } } else { if !strings.Contains(componentId, "#") { - return index.lookupRolodex(uri) + + // does it contain a file extension? + fileExt := filepath.Ext(componentId) + if fileExt != "" { + return index.lookupRolodex(uri) + } + + // root search + return index.FindComponentInRoot(componentId) + } return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[0])) } @@ -368,6 +377,7 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { if index.specAbsolutePath != "" { if index.config.BaseURL != nil { + // consider the file remote. //if strings.Contains(file, "../../") { // extract the base path from the specAbsolutePath for this index. @@ -384,8 +394,11 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { //absoluteFileLocation = loc } else { - panic("nooooooo") - absoluteFileLocation, _ = filepath.Abs(filepath.Join(index.config.BaseURL.Path, file)) + + // consider the file local + + dir := filepath.Dir(index.config.SpecAbsolutePath) + absoluteFileLocation, _ = filepath.Abs(filepath.Join(dir, file)) } } else { absoluteFileLocation = file diff --git a/index/find_component_test.go b/index/find_component_test.go index de31e42..8ca94c2 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -43,7 +43,7 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) { _ = yaml.Unmarshal([]byte(yml), &rootNode) cf := CreateOpenAPIIndexConfig() - cf.AvoidBuildIndex = true + cf.AvoidCircularReferenceCheck = true cf.BasePath = "../test_specs" rolo := NewRolodex(cf) diff --git a/index/index_model.go b/index/index_model.go index d95f468..2d4da64 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -268,17 +268,17 @@ type SpecIndex struct { enumCount int descriptionCount int summaryCount int - seenRemoteSources map[string]*yaml.Node - seenLocalSources map[string]*yaml.Node - refLock sync.Mutex - componentLock sync.RWMutex - errorLock sync.RWMutex - circularReferences []*CircularReferenceResult // only available when the resolver has been used. - allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. - config *SpecIndexConfig // configuration for the index - httpClient *http.Client - componentIndexChan chan bool - polyComponentIndexChan chan bool + //seenRemoteSources map[string]*yaml.Node + //seenLocalSources map[string]*yaml.Node + refLock sync.Mutex + componentLock sync.RWMutex + errorLock sync.RWMutex + circularReferences []*CircularReferenceResult // only available when the resolver has been used. + allowCircularReferences bool // decide if you want to error out, or allow circular references, default is false. + config *SpecIndexConfig // configuration for the index + httpClient *http.Client + componentIndexChan chan bool + polyComponentIndexChan chan bool specAbsolutePath string resolver *Resolver diff --git a/index/index_utils.go b/index/index_utils.go index e74400c..e648e84 100644 --- a/index/index_utils.go +++ b/index/index_utils.go @@ -82,8 +82,8 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) { index.securityRequirementRefs = make(map[string]map[string][]*Reference) index.polymorphicRefs = make(map[string]*Reference) index.refsWithSiblings = make(map[string]Reference) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenLocalSources = make(map[string]*yaml.Node) + //index.seenRemoteSources = make(map[string]*yaml.Node) + //index.seenLocalSources = make(map[string]*yaml.Node) index.opServersRefs = make(map[string]map[string][]*Reference) index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second} index.componentIndexChan = make(chan bool) diff --git a/index/rolodex.go b/index/rolodex.go index 84f92b1..1aa2378 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -258,10 +258,6 @@ func (r *Rolodex) IndexTheRolodex() error { return nil } - // disable index building, it will need to be run after the rolodex indexed - // at a high level. - r.indexConfig.AvoidBuildIndex = true - var caughtErrors []error var indexBuildQueue []*SpecIndex @@ -373,6 +369,19 @@ func (r *Rolodex) IndexTheRolodex() error { if r.rootNode != nil { + // if there is a base path, then we need to set the root spec config to point to a theoretical root.yaml + // which does not exist, but is used to formulate the absolute path to root references correctly. + if r.indexConfig.BasePath != "" && r.indexConfig.BaseURL == nil { + + basePath := r.indexConfig.BasePath + if !filepath.IsAbs(basePath) { + basePath, _ = filepath.Abs(basePath) + } + r.indexConfig.SpecAbsolutePath = filepath.Join(basePath, "root.yaml") + } + + // todo: variation with no base path, but a base URL. + index := NewSpecIndexWithConfig(r.rootNode, r.indexConfig) resolver := NewResolver(index) if r.indexConfig.IgnoreArrayCircularReferences { @@ -382,9 +391,7 @@ func (r *Rolodex) IndexTheRolodex() error { resolver.IgnorePolymorphicCircularReferences() } - if !r.indexConfig.AvoidBuildIndex { - index.BuildIndex() - } + index.BuildIndex() if !r.indexConfig.AvoidCircularReferenceCheck { resolvingErrors := resolver.CheckForCircularReferences() @@ -393,10 +400,14 @@ func (r *Rolodex) IndexTheRolodex() error { } } r.rootIndex = index + if len(index.refErrors) > 0 { + caughtErrors = append(caughtErrors, index.refErrors...) + } } r.indexingDuration = time.Since(started) r.indexed = true r.caughtErrors = caughtErrors + r.built = true return errors.Join(caughtErrors...) } diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 4036cd2..a51db6a 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -17,6 +17,7 @@ import ( ) type LocalFS struct { + indexConfig *SpecIndexConfig entryPointDirectory string baseDirectory string Files map[string]RolodexFile @@ -28,7 +29,18 @@ func (l *LocalFS) GetFiles() map[string]RolodexFile { return l.Files } +func (l *LocalFS) GetErrors() []error { + return l.readingErrors +} + func (l *LocalFS) Open(name string) (fs.File, error) { + + if l.indexConfig != nil && !l.indexConfig.AllowFileLookup { + return nil, &fs.PathError{Op: "open", Path: name, + Err: fmt.Errorf("file lookup for '%s' not allowed, set the index configuration "+ + "to AllowFileLookup to be true", name)} + } + if !filepath.IsAbs(name) { var absErr error name, absErr = filepath.Abs(filepath.Join(l.baseDirectory, name)) diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index a3d5445..12fb5c8 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -27,6 +27,7 @@ type RemoteFS struct { rootURLParsed *url.URL RemoteHandlerFunc RemoteURLHandler Files syncmap.Map + ProcessingFiles syncmap.Map FetchTime int64 FetchChannel chan *RemoteFile remoteWg sync.WaitGroup @@ -235,6 +236,10 @@ func (i *RemoteFS) GetFiles() map[string]RolodexFile { return files } +func (i *RemoteFS) GetErrors() []error { + return i.remoteErrors +} + func (i *RemoteFS) seekRelatives(file *RemoteFile) { extractedRefs := ExtractRefs(string(file.data)) @@ -288,6 +293,11 @@ func (i *RemoteFS) seekRelatives(file *RemoteFile) { func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { + if i.indexConfig != nil && !i.indexConfig.AllowRemoteLookup { + return nil, fmt.Errorf("remote lookup for '%s' is not allowed, please set "+ + "AllowRemoteLookup to true as part of the index configuration", remoteURL) + } + remoteParsedURL, err := url.Parse(remoteURL) if err != nil { return nil, err @@ -298,6 +308,20 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { return r.(*RemoteFile), nil } + // if we're processing, we need to block and wait for the file to be processed + // try path first + if _, ok := i.ProcessingFiles.Load(remoteParsedURL.Path); ok { + i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String()) + for { + if wf, ko := i.Files.Load(remoteParsedURL.Path); ko { + return wf.(*RemoteFile), nil + } + } + } + + // add to processing + i.ProcessingFiles.Store(remoteParsedURL.Path, true) + fileExt := ExtractFileType(remoteParsedURL.Path) if fileExt == UNSUPPORTED { @@ -314,7 +338,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { } } - i.logger.Debug("Loading remote file", "file", remoteURL, "remoteURL", remoteParsedURL.String()) + i.logger.Debug("loading remote file", "file", remoteURL, "remoteURL", remoteParsedURL.String()) // no handler func? use the default client. if i.RemoteHandlerFunc == nil { @@ -323,20 +347,32 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { response, clientErr := i.RemoteHandlerFunc(remoteParsedURL.String()) if clientErr != nil { + + i.remoteErrors = append(i.remoteErrors, clientErr) + // remove from processing + i.ProcessingFiles.Delete(remoteParsedURL.Path) if response != nil { i.logger.Error("client error", "error", clientErr, "status", response.StatusCode) } else { - i.logger.Error("no response for request", "error", clientErr.Error()) + i.logger.Error("client error, empty body", "error", clientErr.Error()) } return nil, clientErr } responseBytes, readError := io.ReadAll(response.Body) if readError != nil { + + // remove from processing + i.ProcessingFiles.Delete(remoteParsedURL.Path) + return nil, readError } if response.StatusCode >= 400 { + + // remove from processing + i.ProcessingFiles.Delete(remoteParsedURL.Path) + i.logger.Error("Unable to fetch remote document", "file", remoteParsedURL.Path, "status", response.StatusCode, "resp", string(responseBytes)) return nil, fmt.Errorf("unable to fetch remote document: %s", string(responseBytes)) @@ -344,6 +380,8 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { absolutePath, pathErr := filepath.Abs(remoteParsedURL.Path) if pathErr != nil { + // remove from processing + i.ProcessingFiles.Delete(remoteParsedURL.Path) return nil, pathErr } @@ -394,11 +432,16 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { i.Files.Store(absolutePath, remoteFile) - i.logger.Debug("successfully loaded file", "file", absolutePath) + if len(remoteFile.data) > 0 { + i.logger.Debug("successfully loaded file", "file", absolutePath) + } i.seekRelatives(remoteFile) idx.BuildIndex() + // remove from processing + i.ProcessingFiles.Delete(remoteParsedURL.Path) + if !i.remoteRunning { return remoteFile, errors.Join(i.remoteErrors...) } else { diff --git a/index/search_index.go b/index/search_index.go index 76aa785..2508a5e 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -24,12 +24,27 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) * if strings.HasPrefix(uri[0], "http") { roloLookup = fullRef.FullDefinition } else { - roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + if filepath.IsAbs(uri[0]) { + roloLookup = uri[0] + } else { + if filepath.Ext(absPath) != "" { + absPath = filepath.Dir(absPath) + } + roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + } } } ref = fmt.Sprintf("#/%s", uri[1]) } else { - roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + if filepath.IsAbs(uri[0]) { + roloLookup = uri[0] + } else { + if filepath.Ext(absPath) != "" { + absPath = filepath.Dir(absPath) + } + roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + } + ref = uri[0] } @@ -65,6 +80,15 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) * return s } } + + // does component exist in the root? + node, _ := rFile.GetContentAsYAMLNode() + if node != nil { + found := idx.FindComponent(ref, node) + if found != nil { + return found + } + } } } diff --git a/index/spec_index.go b/index/spec_index.go index e04fda3..30638db 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -430,10 +430,10 @@ func (index *SpecIndex) GetAllOperationsServers() map[string]map[string][]*Refer return index.opServersRefs } -// GetAllExternalIndexes will return all indexes for external documents -func (index *SpecIndex) GetAllExternalIndexes() map[string]*SpecIndex { - return index.externalSpecIndex -} +//// GetAllExternalIndexes will return all indexes for external documents +//func (index *SpecIndex) GetAllExternalIndexes() map[string]*SpecIndex { +// return index.externalSpecIndex +//} // SetAllowCircularReferenceResolving will flip a bit that can be used by any consumers to determine if they want // to allow or disallow circular references to be resolved or visited diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 06c3052..d66ab0e 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -5,6 +5,7 @@ package index import ( "fmt" + "github.com/pb33f/libopenapi/utils" "log" "log/slog" "net/http" @@ -64,7 +65,6 @@ func TestSpecIndex_ExtractRefsStripe(t *testing.T) { assert.Len(t, index.GetRefsByLine(), 537) assert.Len(t, index.GetLinesWithReferences(), 1972) assert.Len(t, index.GetAllExternalDocuments(), 0) - assert.Len(t, index.GetAllExternalIndexes(), 0) } func TestSpecIndex_Asana(t *testing.T) { @@ -97,7 +97,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification" baseURL, _ := url.Parse(location) - // create a new config that allows local and remote to be mixed up. + // create a new config that allows remote lookups. cf := &SpecIndexConfig{} cf.AvoidBuildIndex = true cf.AllowRemoteLookup = true @@ -121,7 +121,6 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { // create a handler that uses an env variable to capture any GITHUB_TOKEN in the OS ENV // and inject it into the request header, so this does not fail when running lots of local tests. if os.Getenv("GITHUB_TOKEN") != "" { - client := &http.Client{ Timeout: time.Second * 60, } @@ -130,7 +129,6 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", os.Getenv("GITHUB_TOKEN"))) return client.Do(request) }) - } // add remote filesystem @@ -143,19 +141,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { files := remoteFS.GetFiles() fileLen := len(files) assert.Equal(t, 1646, fileLen) - - // - // - // - // - // - //assert.NoError(t, indexedErr) - // - //index := rolo.GetRootIndex() - //rolo.CheckForCircularReferences() - // - //assert.Len(t, index.GetAllExternalIndexes(), 291) - //assert.NotNil(t, index) + assert.Len(t, remoteFS.GetErrors(), 0) } func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) { @@ -163,69 +149,153 @@ func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) { tmp, _ := os.MkdirTemp("", "openapi") cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp) defer os.RemoveAll(filepath.Join(tmp, "openapi")) + err := cmd.Run() if err != nil { log.Fatalf("cmd.Run() failed with %s\n", err) } + spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml")) doLocal, _ := os.ReadFile(spec) + var rootNode yaml.Node _ = yaml.Unmarshal(doLocal, &rootNode) - config := CreateOpenAPIIndexConfig() - config.BasePath = filepath.Join(tmp, "specification") + basePath := filepath.Join(tmp, "specification") - index := NewSpecIndexWithConfig(&rootNode, config) + // create a new config that allows local and remote to be mixed up. + cf := CreateOpenAPIIndexConfig() + cf.AvoidBuildIndex = true + cf.AllowRemoteLookup = true + cf.AvoidCircularReferenceCheck = true + cf.BasePath = basePath + + // create a new rolodex + rolo := NewRolodex(cf) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&rootNode) + + // configure the local filesystem. + fsCfg := LocalFSConfig{ + BaseDirectory: cf.BasePath, + DirFS: os.DirFS(cf.BasePath), + } + + // create a new local filesystem. + fileFS, fsErr := NewLocalFSWithConfig(&fsCfg) + assert.NoError(t, fsErr) + + files := fileFS.GetFiles() + fileLen := len(files) + + assert.Equal(t, 1684, fileLen) + + rolo.AddLocalFS(basePath, fileFS) + + rErr := rolo.IndexTheRolodex() + + assert.NoError(t, rErr) + + index := rolo.GetRootIndex() assert.NotNil(t, index) - assert.Len(t, index.GetAllExternalIndexes(), 296) - ref := index.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml") - assert.NotNil(t, ref) - assert.Equal(t, "operationId", ref.Node.Content[0].Value) - - ref = index.SearchIndexForReference("examples/ruby/domains_create.yml") - assert.NotNil(t, ref) - assert.Equal(t, "lang", ref.Node.Content[0].Value) - - ref = index.SearchIndexForReference("../../shared/responses/server_error.yml") - assert.NotNil(t, ref) - assert.Equal(t, "description", ref.Node.Content[0].Value) - - ref = index.SearchIndexForReference("../models/options.yml") - assert.NotNil(t, ref) + assert.Len(t, index.GetMappedReferencesSequenced(), 296) + assert.Len(t, index.GetMappedReferences(), 296) + assert.Len(t, fileFS.GetErrors(), 0) } func TestSpecIndex_DigitalOcean_LookupsNotAllowed(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/digitalocean.yaml") + do, _ := os.ReadFile("../test_specs/digitalocean.yaml") var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + _ = yaml.Unmarshal(do, &rootNode) - baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") - index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - }) + location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification" + baseURL, _ := url.Parse(location) + + // create a new config that does not allow remote lookups. + cf := &SpecIndexConfig{} + cf.AvoidBuildIndex = true + cf.AvoidCircularReferenceCheck = true + cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + // setting this baseURL will override the base + cf.BaseURL = baseURL + + // create a new rolodex + rolo := NewRolodex(cf) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&rootNode) + + // create a new remote fs and set the config for indexing. + remoteFS, _ := NewRemoteFSWithConfig(cf) + + // add remote filesystem + rolo.AddRemoteFS(location, remoteFS) + + // index the rolodex. + indexedErr := rolo.IndexTheRolodex() + assert.Error(t, indexedErr) + assert.Len(t, utils.UnwrapErrors(indexedErr), 291) + + index := rolo.GetRootIndex() + + files := remoteFS.GetFiles() + fileLen := len(files) + assert.Equal(t, 0, fileLen) + assert.Len(t, remoteFS.GetErrors(), 0) // no lookups allowed, bits have not been set, so there should just be a bunch of errors. - assert.Len(t, index.GetAllExternalIndexes(), 0) assert.True(t, len(index.GetReferenceIndexErrors()) > 0) } func TestSpecIndex_BaseURLError(t *testing.T) { - asana, _ := os.ReadFile("../test_specs/digitalocean.yaml") + + do, _ := os.ReadFile("../test_specs/digitalocean.yaml") var rootNode yaml.Node - _ = yaml.Unmarshal(asana, &rootNode) + _ = yaml.Unmarshal(do, &rootNode) - // this should fail because the base url is not a valid url and digital ocean won't be able to resolve - // anything. - baseURL, _ := url.Parse("https://githerbs.com/fresh/herbs/for/you") - index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - //AllowRemoteLookup: true, - //AllowFileLookup: true, - }) + location := "https://githerbsandcoffeeandcode.com/fresh/herbs/for/you" // not gonna work bro. + baseURL, _ := url.Parse(location) + + // create a new config that allows remote lookups. + cf := &SpecIndexConfig{} + cf.AvoidBuildIndex = true + cf.AllowRemoteLookup = true + cf.AvoidCircularReferenceCheck = true + cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + // setting this baseURL will override the base + cf.BaseURL = baseURL + + // create a new rolodex + rolo := NewRolodex(cf) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&rootNode) + + // create a new remote fs and set the config for indexing. + remoteFS, _ := NewRemoteFSWithConfig(cf) + + // add remote filesystem + rolo.AddRemoteFS(location, remoteFS) + + // index the rolodex. + indexedErr := rolo.IndexTheRolodex() + assert.Error(t, indexedErr) + assert.Len(t, utils.UnwrapErrors(indexedErr), 291) + + files := remoteFS.GetFiles() + fileLen := len(files) + assert.Equal(t, 0, fileLen) + assert.GreaterOrEqual(t, len(remoteFS.GetErrors()), 200) - assert.Len(t, index.GetAllExternalIndexes(), 0) } func TestSpecIndex_k8s(t *testing.T) { @@ -494,13 +564,9 @@ func test_buildMixedRefServer() *httptest.Server { bs, _ := os.ReadFile("../test_specs/burgershop.openapi.yaml") return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if req.URL.String() == "/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml" { - rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") - _, _ = rw.Write(bs) - return - } + rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + _, _ = rw.Write(bs) - _, _ = rw.Write([]byte(`OK`)) })) } @@ -516,6 +582,9 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { cf.AllowRemoteLookup = true cf.AvoidCircularReferenceCheck = true cf.BasePath = "../test_specs" + cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelError, + })) // setting this baseURL will override the base cf.BaseURL, _ = url.Parse(server.URL) @@ -720,7 +789,7 @@ func TestSpecIndex_FindComponent_WithACrazyAssPath(t *testing.T) { index.FindComponent("#/paths/~1crazy~1ass~1references/get/parameters/0", nil).Node.Content[1].Value) } -func TestSpecIndex_FindComponenth(t *testing.T) { +func TestSpecIndex_FindComponent(t *testing.T) { yml := `components: schemas: pizza: @@ -754,34 +823,34 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) { assert.Nil(t, index.lookupRolodex(nil)) } -func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} - _, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something") - assert.Error(t, err) -} +//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) { +// index := new(SpecIndex) +// index.seenRemoteSources = make(map[string]*yaml.Node) +// index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} +// _, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something") +// assert.Error(t, err) +//} -func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") - assert.Error(t, err) - assert.Nil(t, a) - assert.Nil(t, b) -} +//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing.T) { +// index := new(SpecIndex) +// index.seenRemoteSources = make(map[string]*yaml.Node) +// index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{} +// a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey") +// assert.Error(t, err) +// assert.Nil(t, a) +// assert.Nil(t, b) +//} // Discovered in issue https://github.com/pb33f/libopenapi/issues/37 -func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) { - index := new(SpecIndex) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} - a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") - assert.NoError(t, err) - assert.NotNil(t, a) - assert.NotNil(t, b) -} +//func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) { +// index := new(SpecIndex) +// index.seenRemoteSources = make(map[string]*yaml.Node) +// index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{} +// a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json") +// assert.NoError(t, err) +// assert.NotNil(t, a) +// assert.NotNil(t, b) +//} // Discovered in issue https://github.com/daveshanley/vacuum/issues/225 func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { @@ -792,29 +861,13 @@ func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) defer os.Remove("coffee-time.yaml") - index.seenRemoteSources = make(map[string]*yaml.Node) + //index.seenRemoteSources = make(map[string]*yaml.Node) a, b, err := index.lookupFileReference("coffee-time.yaml") assert.NoError(t, err) assert.NotNil(t, a) assert.NotNil(t, b) } -func TestSpecIndex_CheckBadURLRef(t *testing.T) { - yml := `openapi: 3.1.0 -paths: - /cakes: - post: - parameters: - - $ref: 'httpsss://badurl'` - - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) - - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - - assert.Len(t, index.refErrors, 2) -} - func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { yml := `openapi: 3.1.0 paths: @@ -829,15 +882,34 @@ paths: c := CreateClosedAPIIndexConfig() idx := NewSpecIndexWithConfig(&rootNode, c) - assert.Len(t, idx.refErrors, 2) - assert.Equal(t, "remote lookups are not permitted, "+ - "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error()) + assert.Len(t, idx.refErrors, 1) } func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) { - _ = os.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664) + c := []byte("name: time for coffee") + + _ = os.WriteFile("coffee-time.yaml", c, 0o664) defer os.Remove("coffee-time.yaml") + // create a new config that allows local and remote to be mixed up. + cf := CreateOpenAPIIndexConfig() + cf.AvoidCircularReferenceCheck = true + cf.BasePath = "." + + // create a new rolodex + rolo := NewRolodex(cf) + + // configure the local filesystem. + fsCfg := LocalFSConfig{ + BaseDirectory: cf.BasePath, + FileFilters: []string{"coffee-time.yaml"}, + DirFS: os.DirFS(cf.BasePath), + } + + // create a new local filesystem. + fileFS, err := NewLocalFSWithConfig(&fsCfg) + assert.NoError(t, err) + yml := `openapi: 3.0.3 paths: /cakes: @@ -845,24 +917,32 @@ paths: parameters: - $ref: 'coffee-time.yaml'` - var rootNode yaml.Node - _ = yaml.Unmarshal([]byte(yml), &rootNode) + var coffee yaml.Node + _ = yaml.Unmarshal([]byte(yml), &coffee) - index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&coffee) + + rolo.AddLocalFS(cf.BasePath, fileFS) + rErr := rolo.IndexTheRolodex() + + assert.NoError(t, rErr) + + index := rolo.GetRootIndex() assert.NotNil(t, index.GetAllParametersFromOperations()["/cakes"]["post"]["coffee-time.yaml"][0].Node) } -func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) { - index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{ - //AllowRemoteLookup: true, - }) - index.seenRemoteSources = make(map[string]*yaml.Node) - a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey") - assert.Error(t, err) - assert.Nil(t, a) - assert.Nil(t, b) -} +//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) { +// index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{ +// //AllowRemoteLookup: true, +// }) +// index.seenRemoteSources = make(map[string]*yaml.Node) +// a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey") +// assert.Error(t, err) +// assert.Nil(t, a) +// assert.Nil(t, b) +//} func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) { index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) @@ -870,39 +950,79 @@ func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) { assert.Error(t, err) } -func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) { - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - index.seenRemoteSources = make(map[string]*yaml.Node) - index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{} - _, _, err := index.lookupFileReference("magic-money-file.json#something") - assert.Error(t, err) -} - -func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) { - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - _, _, err := index.lookupFileReference("chickers.json#no-rice") - assert.Error(t, err) -} - -func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) { - _ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664) - defer os.Remove("chickers.yaml") - var root yaml.Node - index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig()) - _, _, err := index.lookupFileReference("chickers.yaml#no-rice") - assert.Error(t, err) -} +// +//func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) { +// index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) +// index.seenRemoteSources = make(map[string]*yaml.Node) +// index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{} +// _, _, err := index.lookupFileReference("magic-money-file.json#something") +// assert.Error(t, err) +//} +// +//func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) { +// index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) +// _, _, err := index.lookupFileReference("chickers.json#no-rice") +// assert.Error(t, err) +//} +// +//func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) { +// _ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664) +// defer os.Remove("chickers.yaml") +// var root yaml.Node +// index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig()) +// _, _, err := index.lookupFileReference("chickers.yaml#no-rice") +// assert.Error(t, err) +//} func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) { - _ = os.WriteFile("embie.yaml", []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy"), 0o664) + + embie := []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy") + + _ = os.WriteFile("embie.yaml", embie, 0o664) defer os.Remove("embie.yaml") - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - index.seenRemoteSources = make(map[string]*yaml.Node) - k, doc, err := index.lookupFileReference("embie.yaml#/.naughty") + // create a new config that allows local and remote to be mixed up. + cf := CreateOpenAPIIndexConfig() + cf.AvoidBuildIndex = true + cf.AvoidCircularReferenceCheck = true + cf.BasePath = "." + + // create a new rolodex + rolo := NewRolodex(cf) + + var myPuppy yaml.Node + _ = yaml.Unmarshal(embie, &myPuppy) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&myPuppy) + + // configure the local filesystem. + fsCfg := LocalFSConfig{ + BaseDirectory: cf.BasePath, + FileFilters: []string{"embie.yaml"}, + DirFS: os.DirFS(cf.BasePath), + } + + // create a new local filesystem. + fileFS, err := NewLocalFSWithConfig(&fsCfg) assert.NoError(t, err) - assert.NotNil(t, doc) - assert.Nil(t, k) + + rolo.AddLocalFS(cf.BasePath, fileFS) + rErr := rolo.IndexTheRolodex() + + assert.NoError(t, rErr) + + embieRoloFile, fErr := rolo.Open("embie.yaml") + + assert.NoError(t, fErr) + assert.NotNil(t, embieRoloFile) + + index := rolo.GetRootIndex() + //index.seenRemoteSources = make(map[string]*yaml.Node) + absoluteRef, _ := filepath.Abs("embie.yaml#/naughty") + fRef := index.SearchIndexForReference(absoluteRef) + assert.NotNil(t, fRef) + } func TestSpecIndex_lookupFileReference(t *testing.T) { @@ -918,7 +1038,6 @@ func TestSpecIndex_lookupFileReference(t *testing.T) { // create a new config that allows local and remote to be mixed up. cf := CreateOpenAPIIndexConfig() cf.AvoidBuildIndex = true - cf.AllowRemoteLookup = true cf.AvoidCircularReferenceCheck = true cf.BasePath = "."