diff --git a/index/extract_refs.go b/index/extract_refs.go index 46e2300..d3c5af8 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -6,6 +6,7 @@ package index import ( "errors" "fmt" + "net/url" "path/filepath" "strings" @@ -207,8 +208,13 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, // determine absolute path to this definition // TODO: come and clean this mess up. + var iroot string + if strings.HasPrefix(index.specAbsolutePath, "http") { + iroot = index.specAbsolutePath + } else { + iroot = filepath.Dir(index.specAbsolutePath) + } - iroot := filepath.Dir(index.specAbsolutePath) var componentName string var fullDefinitionPath string if len(uri) == 2 { @@ -239,6 +245,15 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, } else { fullDefinitionPath = uri[0] componentName = uri[0] + if strings.HasPrefix(iroot, "http") { + if !filepath.IsAbs(uri[0]) { + u, _ := url.Parse(iroot) + pathDir := filepath.Dir(u.Path) + pathAbs, _ := filepath.Abs(filepath.Join(pathDir, uri[0])) + u.Path = pathAbs + fullDefinitionPath = u.String() + } + } } } } diff --git a/index/find_component.go b/index/find_component.go index d4c2c49..767c25a 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -99,220 +99,220 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) { close(d) } -func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { - // split string to remove file reference - //uri := strings.Split(ref, "#") - // - //// have we already seen this remote source? - //var parsedRemoteDocument *yaml.Node - //alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0]) - // - //if alreadySeen { - // parsedRemoteDocument = foundDocument - //} else { - // - // d := make(chan bool) - // var body []byte - // var err error - // - // go func(uri string) { - // bc := make(chan []byte) - // ec := make(chan error) - // var getter = httpClient.Get - // if index.config != nil && index.config.RemoteURLHandler != nil { - // getter = index.config.RemoteURLHandler - // } - // - // // if we have a remote handler, use it instead of the default. - // if index.config != nil && index.config.FSHandler != nil { - // go func() { - // remoteFS := index.config.FSHandler - // remoteFile, rErr := remoteFS.Open(uri) - // if rErr != nil { - // e := fmt.Errorf("unable to open remote file: %s", rErr) - // ec <- e - // return - // } - // b, ioErr := io.ReadAll(remoteFile) - // if ioErr != nil { - // e := fmt.Errorf("unable to read remote file bytes: %s", ioErr) - // ec <- e - // return - // } - // bc <- b - // }() - // } else { - // go getRemoteDoc(getter, uri, bc, ec) - // } - // select { - // case v := <-bc: - // body = v - // break - // case er := <-ec: - // err = er - // break - // } - // if len(body) > 0 { - // var remoteDoc yaml.Node - // er := yaml.Unmarshal(body, &remoteDoc) - // if er != nil { - // err = er - // d <- true - // return - // } - // parsedRemoteDocument = &remoteDoc - // if index.config != nil { - // index.config.seenRemoteSources.Store(uri, &remoteDoc) - // } - // } - // d <- true - // }(uri[0]) - // - // // wait for double go fun. - // <-d - // if err != nil { - // // no bueno. - // return nil, nil, err - // } - //} - // - //// lookup item from reference by using a path query. - //var query string - //if len(uri) >= 2 { - // query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) - //} else { - // query = "$" - //} - // - //query, err := url.PathUnescape(query) - //if err != nil { - // return nil, nil, err - //} - // - //// remove any URL encoding - //query = strings.Replace(query, "~1", "./", 1) - //query = strings.ReplaceAll(query, "~1", "/") - // - //path, err := yamlpath.NewPath(query) - //if err != nil { - // return nil, nil, err - //} - //result, _ := path.Find(parsedRemoteDocument) - //if len(result) == 1 { - // return result[0], parsedRemoteDocument, nil - //} - return nil, nil, nil -} +//func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { +// // split string to remove file reference +// //uri := strings.Split(ref, "#") +// // +// //// have we already seen this remote source? +// //var parsedRemoteDocument *yaml.Node +// //alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0]) +// // +// //if alreadySeen { +// // parsedRemoteDocument = foundDocument +// //} else { +// // +// // d := make(chan bool) +// // var body []byte +// // var err error +// // +// // go func(uri string) { +// // bc := make(chan []byte) +// // ec := make(chan error) +// // var getter = httpClient.Get +// // if index.config != nil && index.config.RemoteURLHandler != nil { +// // getter = index.config.RemoteURLHandler +// // } +// // +// // // if we have a remote handler, use it instead of the default. +// // if index.config != nil && index.config.FSHandler != nil { +// // go func() { +// // remoteFS := index.config.FSHandler +// // remoteFile, rErr := remoteFS.Open(uri) +// // if rErr != nil { +// // e := fmt.Errorf("unable to open remote file: %s", rErr) +// // ec <- e +// // return +// // } +// // b, ioErr := io.ReadAll(remoteFile) +// // if ioErr != nil { +// // e := fmt.Errorf("unable to read remote file bytes: %s", ioErr) +// // ec <- e +// // return +// // } +// // bc <- b +// // }() +// // } else { +// // go getRemoteDoc(getter, uri, bc, ec) +// // } +// // select { +// // case v := <-bc: +// // body = v +// // break +// // case er := <-ec: +// // err = er +// // break +// // } +// // if len(body) > 0 { +// // var remoteDoc yaml.Node +// // er := yaml.Unmarshal(body, &remoteDoc) +// // if er != nil { +// // err = er +// // d <- true +// // return +// // } +// // parsedRemoteDocument = &remoteDoc +// // if index.config != nil { +// // index.config.seenRemoteSources.Store(uri, &remoteDoc) +// // } +// // } +// // d <- true +// // }(uri[0]) +// // +// // // wait for double go fun. +// // <-d +// // if err != nil { +// // // no bueno. +// // return nil, nil, err +// // } +// //} +// // +// //// lookup item from reference by using a path query. +// //var query string +// //if len(uri) >= 2 { +// // query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) +// //} else { +// // query = "$" +// //} +// // +// //query, err := url.PathUnescape(query) +// //if err != nil { +// // return nil, nil, err +// //} +// // +// //// remove any URL encoding +// //query = strings.Replace(query, "~1", "./", 1) +// //query = strings.ReplaceAll(query, "~1", "/") +// // +// //path, err := yamlpath.NewPath(query) +// //if err != nil { +// // return nil, nil, err +// //} +// //result, _ := path.Find(parsedRemoteDocument) +// //if len(result) == 1 { +// // return result[0], parsedRemoteDocument, nil +// //} +// return nil, nil, nil +//} -func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, error) { - // split string to remove file reference - uri := strings.Split(ref, "#") - file := strings.ReplaceAll(uri[0], "file:", "") - //filePath := filepath.Dir(file) - //fileName := filepath.Base(file) - absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) - - // extract the document from the rolodex. - rFile, rError := index.rolodex.Open(absoluteFileLocation) - if rError != nil { - return nil, nil, rError - } - - parsedDocument, err := rFile.GetContentAsYAMLNode() - if err != nil { - return nil, nil, err - } - - //if index.seenRemoteSources[file] != nil { - // parsedDocument = index.seenRemoteSources[file] - //} else { - // - // base := index.config.BasePath - // fileToRead := filepath.Join(base, filePath, fileName) - // var body []byte - // var err error - // - // // if we have an FS handler, use it instead of the default behavior - // if index.config != nil && index.config.FSHandler != nil { - // remoteFS := index.config.FSHandler - // remoteFile, rErr := remoteFS.Open(fileToRead) - // if rErr != nil { - // e := fmt.Errorf("unable to open file: %s", rErr) - // return nil, nil, e - // } - // body, err = io.ReadAll(remoteFile) - // if err != nil { - // e := fmt.Errorf("unable to read file bytes: %s", err) - // return nil, nil, e - // } - // - // } else { - // - // // try and read the file off the local file system, if it fails - // // check for a baseURL and then ask our remote lookup function to go try and get it. - // body, err = os.ReadFile(fileToRead) - // - // if err != nil { - // - // // if we have a baseURL, then we can try and get the file from there. - // if index.config != nil && index.config.BaseURL != nil { - // - // u := index.config.BaseURL - // remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) - // a, b, e := index.lookupRemoteReference(remoteRef) - // if e != nil { - // // give up, we can't find the file, not locally, not remotely. It's toast. - // return nil, nil, e - // } - // return a, b, nil - // - // } else { - // // no baseURL? then we can't do anything, give up. - // return nil, nil, err - // } - // } - // } - // var remoteDoc yaml.Node - // err = yaml.Unmarshal(body, &remoteDoc) - // if err != nil { - // return nil, nil, err - // } - // parsedDocument = &remoteDoc - // if index.seenLocalSources != nil { - // index.sourceLock.Lock() - // index.seenLocalSources[file] = &remoteDoc - // index.sourceLock.Unlock() - // } - //} - - // lookup item from reference by using a path query. - var query string - if len(uri) >= 2 { - query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) - } else { - query = "$" - } - - query, err = url.PathUnescape(query) - if err != nil { - return nil, nil, err - } - - // remove any URL encoding - query = strings.Replace(query, "~1", "./", 1) - query = strings.ReplaceAll(query, "~1", "/") - - path, err := yamlpath.NewPath(query) - if err != nil { - return nil, nil, err - } - result, _ := path.Find(parsedDocument) - if len(result) == 1 { - return result[0], parsedDocument, nil - } - - return nil, parsedDocument, nil -} +//func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, error) { +// // split string to remove file reference +// uri := strings.Split(ref, "#") +// file := strings.ReplaceAll(uri[0], "file:", "") +// //filePath := filepath.Dir(file) +// //fileName := filepath.Base(file) +// absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) +// +// // extract the document from the rolodex. +// rFile, rError := index.rolodex.Open(absoluteFileLocation) +// if rError != nil { +// return nil, nil, rError +// } +// +// parsedDocument, err := rFile.GetContentAsYAMLNode() +// if err != nil { +// return nil, nil, err +// } +// +// //if index.seenRemoteSources[file] != nil { +// // parsedDocument = index.seenRemoteSources[file] +// //} else { +// // +// // base := index.config.BasePath +// // fileToRead := filepath.Join(base, filePath, fileName) +// // var body []byte +// // var err error +// // +// // // if we have an FS handler, use it instead of the default behavior +// // if index.config != nil && index.config.FSHandler != nil { +// // remoteFS := index.config.FSHandler +// // remoteFile, rErr := remoteFS.Open(fileToRead) +// // if rErr != nil { +// // e := fmt.Errorf("unable to open file: %s", rErr) +// // return nil, nil, e +// // } +// // body, err = io.ReadAll(remoteFile) +// // if err != nil { +// // e := fmt.Errorf("unable to read file bytes: %s", err) +// // return nil, nil, e +// // } +// // +// // } else { +// // +// // // try and read the file off the local file system, if it fails +// // // check for a baseURL and then ask our remote lookup function to go try and get it. +// // body, err = os.ReadFile(fileToRead) +// // +// // if err != nil { +// // +// // // if we have a baseURL, then we can try and get the file from there. +// // if index.config != nil && index.config.BaseURL != nil { +// // +// // u := index.config.BaseURL +// // remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) +// // a, b, e := index.lookupRemoteReference(remoteRef) +// // if e != nil { +// // // give up, we can't find the file, not locally, not remotely. It's toast. +// // return nil, nil, e +// // } +// // return a, b, nil +// // +// // } else { +// // // no baseURL? then we can't do anything, give up. +// // return nil, nil, err +// // } +// // } +// // } +// // var remoteDoc yaml.Node +// // err = yaml.Unmarshal(body, &remoteDoc) +// // if err != nil { +// // return nil, nil, err +// // } +// // parsedDocument = &remoteDoc +// // if index.seenLocalSources != nil { +// // index.sourceLock.Lock() +// // index.seenLocalSources[file] = &remoteDoc +// // index.sourceLock.Unlock() +// // } +// //} +// +// // lookup item from reference by using a path query. +// var query string +// if len(uri) >= 2 { +// query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) +// } else { +// query = "$" +// } +// +// query, err = url.PathUnescape(query) +// if err != nil { +// return nil, nil, err +// } +// +// // remove any URL encoding +// query = strings.Replace(query, "~1", "./", 1) +// query = strings.ReplaceAll(query, "~1", "/") +// +// path, err := yamlpath.NewPath(query) +// if err != nil { +// return nil, nil, err +// } +// result, _ := path.Find(parsedDocument) +// if len(result) == 1 { +// return result[0], parsedDocument, nil +// } +// +// return nil, parsedDocument, nil +//} func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference { // check component for url encoding. diff --git a/index/find_component_test.go b/index/find_component_test.go index 8ca94c2..4ad1480 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -6,6 +6,8 @@ package index import ( "errors" "fmt" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" "io" "io/fs" "net/http" @@ -13,9 +15,6 @@ import ( "os" "reflect" "testing" - - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" ) func TestSpecIndex_performExternalLookup(t *testing.T) { @@ -177,6 +176,7 @@ components: //} func TestSpecIndex_LocateRemoteDocsWithRemoteURLHandler(t *testing.T) { + // This test will push the index to do try and locate remote references that use relative references spec := `openapi: 3.0.2 info: @@ -191,13 +191,38 @@ paths: var rootNode yaml.Node _ = yaml.Unmarshal([]byte(spec), &rootNode) - c := CreateOpenAPIIndexConfig() - c.RemoteURLHandler = httpClient.Get + //location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification" + //baseURL, _ := url.Parse(location) - index := NewSpecIndexWithConfig(&rootNode, c) + // create a new config that allows remote lookups. + cf := &SpecIndexConfig{} + cf.AllowRemoteLookup = true + cf.AvoidCircularReferenceCheck = true + + // setting this baseURL will override the base + //cf.BaseURL = baseURL + + // create a new rolodex + rolo := NewRolodex(cf) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&rootNode) + + // create a new remote fs and set the config for indexing. + remoteFS, _ := NewRemoteFSWithConfig(cf) + + // add remote filesystem + rolo.AddRemoteFS("", remoteFS) + + // index the rolodex. + indexedErr := rolo.IndexTheRolodex() + + assert.NoError(t, indexedErr) + + index := rolo.GetRootIndex() // extract crs param from index - crsParam := index.GetMappedReferences()["https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"] + crsParam := index.GetMappedReferences()["#/components/parameters/crs"] assert.NotNil(t, crsParam) assert.True(t, crsParam.IsRemote) assert.Equal(t, "crs", crsParam.Node.Content[1].Value) diff --git a/index/spec_index_test.go b/index/spec_index_test.go index d66ab0e..32e71f0 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -853,20 +853,20 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) { //} // Discovered in issue https://github.com/daveshanley/vacuum/issues/225 -func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { - cwd, _ := os.Getwd() - index := new(SpecIndex) - index.config = &SpecIndexConfig{BasePath: cwd} - - _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) - defer os.Remove("coffee-time.yaml") - - //index.seenRemoteSources = make(map[string]*yaml.Node) - a, b, err := index.lookupFileReference("coffee-time.yaml") - assert.NoError(t, err) - assert.NotNil(t, a) - assert.NotNil(t, b) -} +//func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { +// cwd, _ := os.Getwd() +// index := new(SpecIndex) +// index.config = &SpecIndexConfig{BasePath: cwd} +// +// _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) +// defer os.Remove("coffee-time.yaml") +// +// //index.seenRemoteSources = make(map[string]*yaml.Node) +// a, b, err := index.lookupFileReference("coffee-time.yaml") +// assert.NoError(t, err) +// assert.NotNil(t, a) +// assert.NotNil(t, b) +//} func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { yml := `openapi: 3.1.0 @@ -944,11 +944,11 @@ paths: // assert.Nil(t, b) //} -func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) { - index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) - _, _, err := index.lookupFileReference("not-a-reference") - assert.Error(t, err) -} +//func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) { +// index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) +// _, _, err := index.lookupFileReference("not-a-reference") +// assert.Error(t, err) +//} // //func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) {