All spec_index tests pass

It’s so, so much faster than before, intelligent and ready for scale. I’m excited!

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-10-20 11:38:29 -04:00
parent b295e8fd5c
commit 1bf772ab69
4 changed files with 280 additions and 240 deletions

View File

@@ -6,6 +6,7 @@ package index
import ( import (
"errors" "errors"
"fmt" "fmt"
"net/url"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -207,8 +208,13 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
// determine absolute path to this definition // determine absolute path to this definition
// TODO: come and clean this mess up. // TODO: come and clean this mess up.
var iroot string
if strings.HasPrefix(index.specAbsolutePath, "http") {
iroot = index.specAbsolutePath
} else {
iroot = filepath.Dir(index.specAbsolutePath)
}
iroot := filepath.Dir(index.specAbsolutePath)
var componentName string var componentName string
var fullDefinitionPath string var fullDefinitionPath string
if len(uri) == 2 { if len(uri) == 2 {
@@ -239,6 +245,15 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
} else { } else {
fullDefinitionPath = uri[0] fullDefinitionPath = uri[0]
componentName = uri[0] componentName = uri[0]
if strings.HasPrefix(iroot, "http") {
if !filepath.IsAbs(uri[0]) {
u, _ := url.Parse(iroot)
pathDir := filepath.Dir(u.Path)
pathAbs, _ := filepath.Abs(filepath.Join(pathDir, uri[0]))
u.Path = pathAbs
fullDefinitionPath = u.String()
}
}
} }
} }
} }

View File

@@ -99,220 +99,220 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
close(d) close(d)
} }
func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) { //func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
// split string to remove file reference // // split string to remove file reference
//uri := strings.Split(ref, "#") // //uri := strings.Split(ref, "#")
// // //
//// have we already seen this remote source? // //// have we already seen this remote source?
//var parsedRemoteDocument *yaml.Node // //var parsedRemoteDocument *yaml.Node
//alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0]) // //alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
// // //
//if alreadySeen { // //if alreadySeen {
// parsedRemoteDocument = foundDocument // // parsedRemoteDocument = foundDocument
//} else { // //} else {
// // //
// d := make(chan bool) // // d := make(chan bool)
// var body []byte // // var body []byte
// var err error // // var err error
// // //
// go func(uri string) { // // go func(uri string) {
// bc := make(chan []byte) // // bc := make(chan []byte)
// ec := make(chan error) // // ec := make(chan error)
// var getter = httpClient.Get // // var getter = httpClient.Get
// if index.config != nil && index.config.RemoteURLHandler != nil { // // if index.config != nil && index.config.RemoteURLHandler != nil {
// getter = index.config.RemoteURLHandler // // getter = index.config.RemoteURLHandler
// } // // }
// // //
// // if we have a remote handler, use it instead of the default. // // // if we have a remote handler, use it instead of the default.
// if index.config != nil && index.config.FSHandler != nil { // // if index.config != nil && index.config.FSHandler != nil {
// go func() { // // go func() {
// remoteFS := index.config.FSHandler // // remoteFS := index.config.FSHandler
// remoteFile, rErr := remoteFS.Open(uri) // // remoteFile, rErr := remoteFS.Open(uri)
// if rErr != nil { // // if rErr != nil {
// e := fmt.Errorf("unable to open remote file: %s", rErr) // // e := fmt.Errorf("unable to open remote file: %s", rErr)
// ec <- e // // ec <- e
// return // // return
// } // // }
// b, ioErr := io.ReadAll(remoteFile) // // b, ioErr := io.ReadAll(remoteFile)
// if ioErr != nil { // // if ioErr != nil {
// e := fmt.Errorf("unable to read remote file bytes: %s", ioErr) // // e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
// ec <- e // // ec <- e
// return // // return
// } // // }
// bc <- b // // bc <- b
// }() // // }()
// } else { // // } else {
// go getRemoteDoc(getter, uri, bc, ec) // // go getRemoteDoc(getter, uri, bc, ec)
// } // // }
// select { // // select {
// case v := <-bc: // // case v := <-bc:
// body = v // // body = v
// break // // break
// case er := <-ec: // // case er := <-ec:
// err = er // // err = er
// break // // break
// } // // }
// if len(body) > 0 { // // if len(body) > 0 {
// var remoteDoc yaml.Node // // var remoteDoc yaml.Node
// er := yaml.Unmarshal(body, &remoteDoc) // // er := yaml.Unmarshal(body, &remoteDoc)
// if er != nil { // // if er != nil {
// err = er // // err = er
// d <- true // // d <- true
// return // // return
// } // // }
// parsedRemoteDocument = &remoteDoc // // parsedRemoteDocument = &remoteDoc
// if index.config != nil { // // if index.config != nil {
// index.config.seenRemoteSources.Store(uri, &remoteDoc) // // index.config.seenRemoteSources.Store(uri, &remoteDoc)
// } // // }
// } // // }
// d <- true // // d <- true
// }(uri[0]) // // }(uri[0])
// // //
// // wait for double go fun. // // // wait for double go fun.
// <-d // // <-d
// if err != nil { // // if err != nil {
// // no bueno. // // // no bueno.
// return nil, nil, err // // return nil, nil, err
// } // // }
//} // //}
// // //
//// lookup item from reference by using a path query. // //// lookup item from reference by using a path query.
//var query string // //var query string
//if len(uri) >= 2 { // //if len(uri) >= 2 {
// query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) // // query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
//} else { // //} else {
// query = "$" // // query = "$"
//} // //}
// // //
//query, err := url.PathUnescape(query) // //query, err := url.PathUnescape(query)
//if err != nil { // //if err != nil {
// return nil, nil, err // // return nil, nil, err
//} // //}
// // //
//// remove any URL encoding // //// remove any URL encoding
//query = strings.Replace(query, "~1", "./", 1) // //query = strings.Replace(query, "~1", "./", 1)
//query = strings.ReplaceAll(query, "~1", "/") // //query = strings.ReplaceAll(query, "~1", "/")
// // //
//path, err := yamlpath.NewPath(query) // //path, err := yamlpath.NewPath(query)
//if err != nil { // //if err != nil {
// return nil, nil, err // // return nil, nil, err
//} // //}
//result, _ := path.Find(parsedRemoteDocument) // //result, _ := path.Find(parsedRemoteDocument)
//if len(result) == 1 { // //if len(result) == 1 {
// return result[0], parsedRemoteDocument, nil // // return result[0], parsedRemoteDocument, nil
//} // //}
return nil, nil, nil // return nil, nil, nil
} //}
func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, error) { //func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, error) {
// split string to remove file reference // // split string to remove file reference
uri := strings.Split(ref, "#") // uri := strings.Split(ref, "#")
file := strings.ReplaceAll(uri[0], "file:", "") // file := strings.ReplaceAll(uri[0], "file:", "")
//filePath := filepath.Dir(file) // //filePath := filepath.Dir(file)
//fileName := filepath.Base(file) // //fileName := filepath.Base(file)
absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file)) // absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file))
//
// extract the document from the rolodex. // // extract the document from the rolodex.
rFile, rError := index.rolodex.Open(absoluteFileLocation) // rFile, rError := index.rolodex.Open(absoluteFileLocation)
if rError != nil { // if rError != nil {
return nil, nil, rError // return nil, nil, rError
} // }
//
parsedDocument, err := rFile.GetContentAsYAMLNode() // parsedDocument, err := rFile.GetContentAsYAMLNode()
if err != nil { // if err != nil {
return nil, nil, err // return nil, nil, err
} // }
//
//if index.seenRemoteSources[file] != nil { // //if index.seenRemoteSources[file] != nil {
// parsedDocument = index.seenRemoteSources[file] // // parsedDocument = index.seenRemoteSources[file]
//} else { // //} else {
// // //
// base := index.config.BasePath // // base := index.config.BasePath
// fileToRead := filepath.Join(base, filePath, fileName) // // fileToRead := filepath.Join(base, filePath, fileName)
// var body []byte // // var body []byte
// var err error // // var err error
// // //
// // if we have an FS handler, use it instead of the default behavior // // // if we have an FS handler, use it instead of the default behavior
// if index.config != nil && index.config.FSHandler != nil { // // if index.config != nil && index.config.FSHandler != nil {
// remoteFS := index.config.FSHandler // // remoteFS := index.config.FSHandler
// remoteFile, rErr := remoteFS.Open(fileToRead) // // remoteFile, rErr := remoteFS.Open(fileToRead)
// if rErr != nil { // // if rErr != nil {
// e := fmt.Errorf("unable to open file: %s", rErr) // // e := fmt.Errorf("unable to open file: %s", rErr)
// return nil, nil, e // // return nil, nil, e
// } // // }
// body, err = io.ReadAll(remoteFile) // // body, err = io.ReadAll(remoteFile)
// if err != nil { // // if err != nil {
// e := fmt.Errorf("unable to read file bytes: %s", err) // // e := fmt.Errorf("unable to read file bytes: %s", err)
// return nil, nil, e // // return nil, nil, e
// } // // }
// // //
// } else { // // } else {
// // //
// // try and read the file off the local file system, if it fails // // // try and read the file off the local file system, if it fails
// // check for a baseURL and then ask our remote lookup function to go try and get it. // // // check for a baseURL and then ask our remote lookup function to go try and get it.
// body, err = os.ReadFile(fileToRead) // // body, err = os.ReadFile(fileToRead)
// // //
// if err != nil { // // if err != nil {
// // //
// // if we have a baseURL, then we can try and get the file from there. // // // if we have a baseURL, then we can try and get the file from there.
// if index.config != nil && index.config.BaseURL != nil { // // if index.config != nil && index.config.BaseURL != nil {
// // //
// u := index.config.BaseURL // // u := index.config.BaseURL
// remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true) // // remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
// a, b, e := index.lookupRemoteReference(remoteRef) // // a, b, e := index.lookupRemoteReference(remoteRef)
// if e != nil { // // if e != nil {
// // give up, we can't find the file, not locally, not remotely. It's toast. // // // give up, we can't find the file, not locally, not remotely. It's toast.
// return nil, nil, e // // return nil, nil, e
// } // // }
// return a, b, nil // // return a, b, nil
// // //
// } else { // // } else {
// // no baseURL? then we can't do anything, give up. // // // no baseURL? then we can't do anything, give up.
// return nil, nil, err // // return nil, nil, err
// } // // }
// } // // }
// } // // }
// var remoteDoc yaml.Node // // var remoteDoc yaml.Node
// err = yaml.Unmarshal(body, &remoteDoc) // // err = yaml.Unmarshal(body, &remoteDoc)
// if err != nil { // // if err != nil {
// return nil, nil, err // // return nil, nil, err
// } // // }
// parsedDocument = &remoteDoc // // parsedDocument = &remoteDoc
// if index.seenLocalSources != nil { // // if index.seenLocalSources != nil {
// index.sourceLock.Lock() // // index.sourceLock.Lock()
// index.seenLocalSources[file] = &remoteDoc // // index.seenLocalSources[file] = &remoteDoc
// index.sourceLock.Unlock() // // index.sourceLock.Unlock()
// } // // }
//} // //}
//
// lookup item from reference by using a path query. // // lookup item from reference by using a path query.
var query string // var query string
if len(uri) >= 2 { // if len(uri) >= 2 {
query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", ".")) // query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
} else { // } else {
query = "$" // query = "$"
} // }
//
query, err = url.PathUnescape(query) // query, err = url.PathUnescape(query)
if err != nil { // if err != nil {
return nil, nil, err // return nil, nil, err
} // }
//
// remove any URL encoding // // remove any URL encoding
query = strings.Replace(query, "~1", "./", 1) // query = strings.Replace(query, "~1", "./", 1)
query = strings.ReplaceAll(query, "~1", "/") // query = strings.ReplaceAll(query, "~1", "/")
//
path, err := yamlpath.NewPath(query) // path, err := yamlpath.NewPath(query)
if err != nil { // if err != nil {
return nil, nil, err // return nil, nil, err
} // }
result, _ := path.Find(parsedDocument) // result, _ := path.Find(parsedDocument)
if len(result) == 1 { // if len(result) == 1 {
return result[0], parsedDocument, nil // return result[0], parsedDocument, nil
} // }
//
return nil, parsedDocument, nil // return nil, parsedDocument, nil
} //}
func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference { func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference {
// check component for url encoding. // check component for url encoding.

View File

@@ -6,6 +6,8 @@ package index
import ( import (
"errors" "errors"
"fmt" "fmt"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
"io" "io"
"io/fs" "io/fs"
"net/http" "net/http"
@@ -13,9 +15,6 @@ import (
"os" "os"
"reflect" "reflect"
"testing" "testing"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
) )
func TestSpecIndex_performExternalLookup(t *testing.T) { func TestSpecIndex_performExternalLookup(t *testing.T) {
@@ -177,6 +176,7 @@ components:
//} //}
func TestSpecIndex_LocateRemoteDocsWithRemoteURLHandler(t *testing.T) { func TestSpecIndex_LocateRemoteDocsWithRemoteURLHandler(t *testing.T) {
// This test will push the index to do try and locate remote references that use relative references // This test will push the index to do try and locate remote references that use relative references
spec := `openapi: 3.0.2 spec := `openapi: 3.0.2
info: info:
@@ -191,13 +191,38 @@ paths:
var rootNode yaml.Node var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode) _ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig() //location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification"
c.RemoteURLHandler = httpClient.Get //baseURL, _ := url.Parse(location)
index := NewSpecIndexWithConfig(&rootNode, c) // create a new config that allows remote lookups.
cf := &SpecIndexConfig{}
cf.AllowRemoteLookup = true
cf.AvoidCircularReferenceCheck = true
// setting this baseURL will override the base
//cf.BaseURL = baseURL
// create a new rolodex
rolo := NewRolodex(cf)
// set the rolodex root node to the root node of the spec.
rolo.SetRootNode(&rootNode)
// create a new remote fs and set the config for indexing.
remoteFS, _ := NewRemoteFSWithConfig(cf)
// add remote filesystem
rolo.AddRemoteFS("", remoteFS)
// index the rolodex.
indexedErr := rolo.IndexTheRolodex()
assert.NoError(t, indexedErr)
index := rolo.GetRootIndex()
// extract crs param from index // extract crs param from index
crsParam := index.GetMappedReferences()["https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"] crsParam := index.GetMappedReferences()["#/components/parameters/crs"]
assert.NotNil(t, crsParam) assert.NotNil(t, crsParam)
assert.True(t, crsParam.IsRemote) assert.True(t, crsParam.IsRemote)
assert.Equal(t, "crs", crsParam.Node.Content[1].Value) assert.Equal(t, "crs", crsParam.Node.Content[1].Value)

View File

@@ -853,20 +853,20 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) {
//} //}
// Discovered in issue https://github.com/daveshanley/vacuum/issues/225 // Discovered in issue https://github.com/daveshanley/vacuum/issues/225
func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) { //func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
cwd, _ := os.Getwd() // cwd, _ := os.Getwd()
index := new(SpecIndex) // index := new(SpecIndex)
index.config = &SpecIndexConfig{BasePath: cwd} // index.config = &SpecIndexConfig{BasePath: cwd}
//
_ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664) // _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664)
defer os.Remove("coffee-time.yaml") // defer os.Remove("coffee-time.yaml")
//
//index.seenRemoteSources = make(map[string]*yaml.Node) // //index.seenRemoteSources = make(map[string]*yaml.Node)
a, b, err := index.lookupFileReference("coffee-time.yaml") // a, b, err := index.lookupFileReference("coffee-time.yaml")
assert.NoError(t, err) // assert.NoError(t, err)
assert.NotNil(t, a) // assert.NotNil(t, a)
assert.NotNil(t, b) // assert.NotNil(t, b)
} //}
func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) {
yml := `openapi: 3.1.0 yml := `openapi: 3.1.0
@@ -944,11 +944,11 @@ paths:
// assert.Nil(t, b) // assert.Nil(t, b)
//} //}
func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) { //func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) {
index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig()) // index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
_, _, err := index.lookupFileReference("not-a-reference") // _, _, err := index.lookupFileReference("not-a-reference")
assert.Error(t, err) // assert.Error(t, err)
} //}
// //
//func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) { //func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) {