LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-06 12:37:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

First sweep at cleaning up dead code

Browse Source 
first round of a number I am sure, lots to clean.

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-10-21 18:26:21 -04:00
parent be7e477529
commit 28047d08d2
15 changed files with 225 additions and 1478 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
index/extract_refs.go
View File

@@ -261,12 +261,12 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
// if the index has a base URL, use that to resolve the path. // if the index has a base URL, use that to resolve the path.
if index.config.BaseURL != nil { if index.config.BaseURL != nil {
url := *index.config.BaseURL u := *index.config.BaseURL
abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0]))
abs, _ := filepath.Abs(filepath.Join(url.Path, uri[0])) u.Path = abs
url.Path = abs fullDefinitionPath = fmt.Sprintf("%s#/%s", u.String(), uri[1])
fullDefinitionPath = fmt.Sprintf("%s#/%s", url.String(), uri[1])
componentName = fmt.Sprintf("#/%s", uri[1]) componentName = fmt.Sprintf("#/%s", uri[1])
} else { } else {
abs, _ := filepath.Abs(filepath.Join(iroot, uri[0])) abs, _ := filepath.Abs(filepath.Join(iroot, uri[0]))
@@ -314,12 +314,12 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
// if the index has a base URL, use that to resolve the path. // if the index has a base URL, use that to resolve the path.
if index.config.BaseURL != nil { if index.config.BaseURL != nil {
url := *index.config.BaseURL u := *index.config.BaseURL
abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0]))
abs, _ := filepath.Abs(filepath.Join(url.Path, uri[0])) u.Path = abs
url.Path = abs fullDefinitionPath = u.String()
fullDefinitionPath = url.String()
componentName = uri[0] componentName = uri[0]
} else { } else {
abs, _ := filepath.Abs(filepath.Join(iroot, uri[0])) abs, _ := filepath.Abs(filepath.Join(iroot, uri[0]))
@@ -331,7 +331,6 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
} }
} }
//componentName = filepath.Base(uri[0])
} }
} }
} }
@@ -412,16 +411,6 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
continue continue
} }
//if len(uri) == 2 {
// if uri[0] == "" {
// index.allRefs[componentName] = ref
// } else {
// index.allRefs[value] = ref
// }
//} else {
// index.allRefs[value] = ref
//}
index.allRefs[fullDefinitionPath] = ref index.allRefs[fullDefinitionPath] = ref
found = append(found, ref) found = append(found, ref)
} }
@@ -585,14 +574,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
} }
} }
} }
//if len(seenPath) > 0 {
// seenPath = seenPath[:len(seenPath)-1]
//}
} }
//if len(seenPath) > 0 {
// seenPath = seenPath[:len(seenPath)-1]
//}
index.refCount = len(index.allRefs) index.refCount = len(index.allRefs)

367
index/find_component.go
View File

@@ -5,7 +5,6 @@ package index
import ( import (
"fmt" "fmt"
"net/http"
"net/url" "net/url"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -23,30 +22,6 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
return nil return nil
} }
//remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
// if index.config.AllowRemoteLookup {
// return index.lookupRemoteReference(id)
// } else {
// return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
// "please set AllowRemoteLookup to true in the configuration")
// }
//}
//
//fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
// if index.config.AllowFileLookup {
// return index.lookupFileReference(id)
// } else {
// return nil, nil, fmt.Errorf("local lookups are not permitted, " +
// "please set AllowFileLookup to true in the configuration")
// }
//}
//witch DetermineReferenceResolveType(componentId) {
//case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case.
//return index.FindComponentInRoot(componentId)
//case HttpResolve, FileResolve:
uri := strings.Split(componentId, "#/") uri := strings.Split(componentId, "#/")
if len(uri) == 2 { if len(uri) == 2 {
if uri[0] != "" { if uri[0] != "" {
@@ -73,243 +48,8 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
} }
return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[0])) return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[0]))
} }
//}
//return nil
} }
type RemoteURLHandler = func(url string) (*http.Response, error)
//func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
// resp, err := g(u)
// if err != nil {
// e <- err
// close(e)
// close(d)
// return
// }
// var body []byte
// body, _ = io.ReadAll(resp.Body)
// d <- body
// close(e)
// close(d)
//}
//func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
// // split string to remove file reference
// //uri := strings.Split(ref, "#")
// //
// //// have we already seen this remote source?
// //var parsedRemoteDocument *yaml.Node
// //alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
// //
// //if alreadySeen {
// // parsedRemoteDocument = foundDocument
// //} else {
// //
// // d := make(chan bool)
// // var body []byte
// // var err error
// //
// // go func(uri string) {
// // bc := make(chan []byte)
// // ec := make(chan error)
// // var getter = httpClient.Get
// // if index.config != nil && index.config.RemoteURLHandler != nil {
// // getter = index.config.RemoteURLHandler
// // }
// //
// // // if we have a remote handler, use it instead of the default.
// // if index.config != nil && index.config.FSHandler != nil {
// // go func() {
// // remoteFS := index.config.FSHandler
// // remoteFile, rErr := remoteFS.Open(uri)
// // if rErr != nil {
// // e := fmt.Errorf("unable to open remote file: %s", rErr)
// // ec <- e
// // return
// // }
// // b, ioErr := io.ReadAll(remoteFile)
// // if ioErr != nil {
// // e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
// // ec <- e
// // return
// // }
// // bc <- b
// // }()
// // } else {
// // go getRemoteDoc(getter, uri, bc, ec)
// // }
// // select {
// // case v := <-bc:
// // body = v
// // break
// // case er := <-ec:
// // err = er
// // break
// // }
// // if len(body) > 0 {
// // var remoteDoc yaml.Node
// // er := yaml.Unmarshal(body, &remoteDoc)
// // if er != nil {
// // err = er
// // d <- true
// // return
// // }
// // parsedRemoteDocument = &remoteDoc
// // if index.config != nil {
// // index.config.seenRemoteSources.Store(uri, &remoteDoc)
// // }
// // }
// // d <- true
// // }(uri[0])
// //
// // // wait for double go fun.
// // <-d
// // if err != nil {
// // // no bueno.
// // return nil, nil, err
// // }
// //}
// //
// //// lookup item from reference by using a path query.
// //var query string
// //if len(uri) >= 2 {
// // query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
// //} else {
// // query = "$"
// //}
// //
// //query, err := url.PathUnescape(query)
// //if err != nil {
// // return nil, nil, err
// //}
// //
// //// remove any URL encoding
// //query = strings.Replace(query, "~1", "./", 1)
// //query = strings.ReplaceAll(query, "~1", "/")
// //
// //path, err := yamlpath.NewPath(query)
// //if err != nil {
// // return nil, nil, err
// //}
// //result, _ := path.Find(parsedRemoteDocument)
// //if len(result) == 1 {
// // return result[0], parsedRemoteDocument, nil
// //}
// return nil, nil, nil
//}
//func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node, error) {
// // split string to remove file reference
// uri := strings.Split(ref, "#")
// file := strings.ReplaceAll(uri[0], "file:", "")
// //filePath := filepath.Dir(file)
// //fileName := filepath.Base(file)
// absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file))
//
// // extract the document from the rolodex.
// rFile, rError := index.rolodex.Open(absoluteFileLocation)
// if rError != nil {
// return nil, nil, rError
// }
//
// parsedDocument, err := rFile.GetContentAsYAMLNode()
// if err != nil {
// return nil, nil, err
// }
//
// //if index.seenRemoteSources[file] != nil {
// // parsedDocument = index.seenRemoteSources[file]
// //} else {
// //
// // base := index.config.BasePath
// // fileToRead := filepath.Join(base, filePath, fileName)
// // var body []byte
// // var err error
// //
// // // if we have an FS handler, use it instead of the default behavior
// // if index.config != nil && index.config.FSHandler != nil {
// // remoteFS := index.config.FSHandler
// // remoteFile, rErr := remoteFS.Open(fileToRead)
// // if rErr != nil {
// // e := fmt.Errorf("unable to open file: %s", rErr)
// // return nil, nil, e
// // }
// // body, err = io.ReadAll(remoteFile)
// // if err != nil {
// // e := fmt.Errorf("unable to read file bytes: %s", err)
// // return nil, nil, e
// // }
// //
// // } else {
// //
// // // try and read the file off the local file system, if it fails
// // // check for a baseURL and then ask our remote lookup function to go try and get it.
// // body, err = os.ReadFile(fileToRead)
// //
// // if err != nil {
// //
// // // if we have a baseURL, then we can try and get the file from there.
// // if index.config != nil && index.config.BaseURL != nil {
// //
// // u := index.config.BaseURL
// // remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
// // a, b, e := index.lookupRemoteReference(remoteRef)
// // if e != nil {
// // // give up, we can't find the file, not locally, not remotely. It's toast.
// // return nil, nil, e
// // }
// // return a, b, nil
// //
// // } else {
// // // no baseURL? then we can't do anything, give up.
// // return nil, nil, err
// // }
// // }
// // }
// // var remoteDoc yaml.Node
// // err = yaml.Unmarshal(body, &remoteDoc)
// // if err != nil {
// // return nil, nil, err
// // }
// // parsedDocument = &remoteDoc
// // if index.seenLocalSources != nil {
// // index.sourceLock.Lock()
// // index.seenLocalSources[file] = &remoteDoc
// // index.sourceLock.Unlock()
// // }
// //}
//
// // lookup item from reference by using a path query.
// var query string
// if len(uri) >= 2 {
// query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
// } else {
// query = "$"
// }
//
// query, err = url.PathUnescape(query)
// if err != nil {
// return nil, nil, err
// }
//
// // remove any URL encoding
// query = strings.Replace(query, "~1", "./", 1)
// query = strings.ReplaceAll(query, "~1", "/")
//
// path, err := yamlpath.NewPath(query)
// if err != nil {
// return nil, nil, err
// }
// result, _ := path.Find(parsedDocument)
// if len(result) == 1 {
// return result[0], parsedDocument, nil
// }
//
// return nil, parsedDocument, nil
//}
func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference { func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference {
// check component for url encoding. // check component for url encoding.
if strings.Contains(componentId, "%") { if strings.Contains(componentId, "%") {
@@ -333,7 +73,6 @@ func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Refer
fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId) fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId)
// extract properties // extract properties
ref := &Reference{ ref := &Reference{
FullDefinition: fullDef, FullDefinition: fullDef,
Definition: componentId, Definition: componentId,
@@ -356,6 +95,9 @@ func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference {
} }
func (index *SpecIndex) lookupRolodex(uri []string) *Reference { func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
if index.rolodex == nil {
return nil
}
if len(uri) > 0 { if len(uri) > 0 {
@@ -373,9 +115,6 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
if index.specAbsolutePath != "" { if index.specAbsolutePath != "" {
if index.config.BaseURL != nil { if index.config.BaseURL != nil {
// consider the file remote.
//if strings.Contains(file, "../../") {
// extract the base path from the specAbsolutePath for this index. // extract the base path from the specAbsolutePath for this index.
sap, _ := url.Parse(index.specAbsolutePath) sap, _ := url.Parse(index.specAbsolutePath)
newPath, _ := filepath.Abs(filepath.Join(filepath.Dir(sap.Path), file)) newPath, _ := filepath.Abs(filepath.Join(filepath.Dir(sap.Path), file))
@@ -383,16 +122,10 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
sap.Path = newPath sap.Path = newPath
f := sap.String() f := sap.String()
absoluteFileLocation = f absoluteFileLocation = f
//}
//loc := fmt.Sprintf("%s%s", index.config.BaseURL.Path, file)
//absoluteFileLocation = loc
} else { } else {
// consider the file local // consider the file local
dir := filepath.Dir(index.config.SpecAbsolutePath) dir := filepath.Dir(index.config.SpecAbsolutePath)
absoluteFileLocation, _ = filepath.Abs(filepath.Join(dir, file)) absoluteFileLocation, _ = filepath.Abs(filepath.Join(dir, file))
} }
@@ -430,98 +163,6 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
parsedDocument = index.root parsedDocument = index.root
} }
//fmt.Printf("parsedDocument: %v\n", parsedDocument)
//index.externalLock.RLock()
//externalSpecIndex := index.externalSpecIndex[uri[0]]
//index.externalLock.RUnlock()
//if externalSpecIndex == nil {
// _, newRoot, err := lookupFunction(componentId)
// if err != nil {
// indexError := &IndexingError{
// Err: err,
// Node: parent,
// Path: componentId,
// }
// index.errorLock.Lock()
// index.refErrors = append(index.refErrors, indexError)
// index.errorLock.Unlock()
// return nil
// }
//
// // cool, cool, lets index this spec also. This is a recursive action and will keep going
// // until all remote references have been found.
// var bp *url.URL
// var bd string
//
// if index.config.BaseURL != nil {
// bp = index.config.BaseURL
// }
// if index.config.BasePath != "" {
// bd = index.config.BasePath
// }
//
// var path, newBasePath string
// var newUrl *url.URL
//
// if bp != nil {
// path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false)
// newUrl, _ = url.Parse(path)
// newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path)))
// }
// if bd != "" {
// if len(uri[0]) > 0 {
// // if there is no base url defined, but we can know we have been requested remotely,
// // set the base url to the remote url base path.
// // first check if the first param is actually a URL
// io, er := url.ParseRequestURI(uri[0])
// if er != nil {
// newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
// } else {
// if newUrl == nil || newUrl.String() != io.String() {
// newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path)))
// }
// newBasePath = filepath.Dir(filepath.Join(bd, uri[1]))
// }
// } else {
// newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
// }
// }
//
// if newUrl != nil || newBasePath != "" {
// newConfig := &SpecIndexConfig{
// BaseURL: newUrl,
// BasePath: newBasePath,
// AllowRemoteLookup: index.config.AllowRemoteLookup,
// AllowFileLookup: index.config.AllowFileLookup,
// ParentIndex: index,
// seenRemoteSources: index.config.seenRemoteSources,
// remoteLock: index.config.remoteLock,
// uri: uri,
// AvoidBuildIndex: index.config.AvoidBuildIndex,
// }
//
// var newIndex *SpecIndex
// seen := index.SearchAncestryForSeenURI(uri[0])
// if seen == nil {
//
// newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
// index.refLock.Lock()
// index.externalLock.Lock()
// index.externalSpecIndex[uri[0]] = newIndex
// index.externalLock.Unlock()
// newIndex.relativePath = path
// newIndex.parentIndex = index
// index.AddChild(newIndex)
// index.refLock.Unlock()
// externalSpecIndex = newIndex
// } else {
// externalSpecIndex = seen
// }
// }
//}
wholeFile := false wholeFile := false
query := "" query := ""
if len(uri) < 2 { if len(uri) < 2 {
@@ -539,8 +180,6 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
parsedDocument = parsedDocument.Content[0] parsedDocument = parsedDocument.Content[0]
} }
// TODO: remote locations
foundRef = &Reference{ foundRef = &Reference{
FullDefinition: absoluteFileLocation, FullDefinition: absoluteFileLocation,
Definition: fileName, Definition: fileName,

451
index/find_component_test.go
View File

@@ -82,91 +82,23 @@ components:
assert.Len(t, index.GetReferenceIndexErrors(), 1) assert.Len(t, index.GetReferenceIndexErrors(), 1)
} }
//func TestSpecIndex_FindComponentInRoot(t *testing.T) { func TestSpecIndex_FindComponentInRoot(t *testing.T) {
// yml := `openapi: 3.1.0 yml := `openapi: 3.1.0
//components: components:
// schemas: schemas:
// thing: thing:
// properties: properties:
// thong: hi!` thong: hi!`
// var rootNode yaml.Node var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(yml), &rootNode) _ = yaml.Unmarshal([]byte(yml), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./")
// assert.Nil(t, thing)
// assert.Len(t, index.GetReferenceIndexErrors(), 0)
//}
//func TestSpecIndex_FailLookupRemoteComponent_badPath(t *testing.T) { c := CreateOpenAPIIndexConfig()
// yml := `openapi: 3.1.0 index := NewSpecIndexWithConfig(&rootNode, c)
//components:
// schemas:
// thing:
// properties:
// thong:
// $ref: 'https://pb33f.io/site.webmanifest#/....$.ok../oh#/$$_-'`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(yml), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./")
// assert.Nil(t, thing)
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
//}
//func TestSpecIndex_FailLookupRemoteComponent_Ok_butNotFound(t *testing.T) { thing := index.FindComponentInRoot("#/$splish/$.../slash#$///./")
// yml := `openapi: 3.1.0 assert.Nil(t, thing)
//components: assert.Len(t, index.GetReferenceIndexErrors(), 0)
// schemas: }
// thing:
// properties:
// thong:
// $ref: 'https://pb33f.io/site.webmanifest#/valid-but-missing'`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(yml), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// thing := index.FindComponentInRoot("#/valid-but-missing")
// assert.Nil(t, thing)
// assert.Len(t, index.GetReferenceIndexErrors(), 1)
//}
// disabled test because remote host is flaky.
//func TestSpecIndex_LocateRemoteDocsWithNoBaseURLSupplied(t *testing.T) {
// // This test will push the index to do try and locate remote references that use relative references
// spec := `openapi: 3.0.2
//info:
// title: Test
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// // extract crs param from index
// crsParam := index.GetMappedReferences()["https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"]
// assert.NotNil(t, crsParam)
// assert.True(t, crsParam.IsRemote)
// assert.Equal(t, "crs", crsParam.Node.Content[1].Value)
// assert.Equal(t, "query", crsParam.Node.Content[3].Value)
// assert.Equal(t, "form", crsParam.Node.Content[9].Value)
//}
func TestSpecIndex_LocateRemoteDocsWithRemoteURLHandler(t *testing.T) { func TestSpecIndex_LocateRemoteDocsWithRemoteURLHandler(t *testing.T) {
@@ -184,17 +116,11 @@ paths:
var rootNode yaml.Node var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode) _ = yaml.Unmarshal([]byte(spec), &rootNode)
//location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification"
//baseURL, _ := url.Parse(location)
// create a new config that allows remote lookups. // create a new config that allows remote lookups.
cf := &SpecIndexConfig{} cf := &SpecIndexConfig{}
cf.AllowRemoteLookup = true cf.AllowRemoteLookup = true
cf.AvoidCircularReferenceCheck = true cf.AvoidCircularReferenceCheck = true
// setting this baseURL will override the base
//cf.BaseURL = baseURL
// create a new rolodex // create a new rolodex
rolo := NewRolodex(cf) rolo := NewRolodex(cf)
@@ -263,350 +189,3 @@ paths:
index := NewSpecIndexWithConfig(&rootNode, c) index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 1) assert.Len(t, index.GetReferenceIndexErrors(), 1)
} }
//
//func TestGetRemoteDoc(t *testing.T) {
// // Mock HTTP server
// server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
// rw.Write([]byte(`OK`))
// }))
// // Close the server when test finishes
// defer server.Close()
//
// // Channel for data and error
// dataChan := make(chan []byte)
// errorChan := make(chan error)
//
// go getRemoteDoc(http.Get, server.URL, dataChan, errorChan)
//
// data := <-dataChan
// err := <-errorChan
//
// if err != nil {
// t.Errorf("Expected no error, got %v", err)
// }
//
// expectedData := []byte(`OK`)
// if !reflect.DeepEqual(data, expectedData) {
// t.Errorf("Expected %v, got %v", expectedData, data)
// }
//}
//
//type FS struct{}
//type FSBadOpen struct{}
//type FSBadRead struct{}
//
//type file struct {
// name string
// data string
//}
//
//type openFile struct {
// f *file
// offset int64
//}
//
//func (f *openFile) Close() error { return nil }
//func (f *openFile) Stat() (fs.FileInfo, error) { return nil, nil }
//func (f *openFile) Read(b []byte) (int, error) {
// if f.offset >= int64(len(f.f.data)) {
// return 0, io.EOF
// }
// if f.offset < 0 {
// return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid}
// }
// n := copy(b, f.f.data[f.offset:])
// f.offset += int64(n)
// return n, nil
//}
//
////type badFileOpen struct{}
////
////func (f *badFileOpen) Close() error { return errors.New("bad file close") }
////func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
////func (f *badFileOpen) Read(b []byte) (int, error) {
//// return 0, nil
////}
//
//type badFileRead struct {
// f *file
// offset int64
//}
//
//func (f *badFileRead) Close() error { return errors.New("bad file close") }
//func (f *badFileRead) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
//func (f *badFileRead) Read(b []byte) (int, error) {
// return 0, fmt.Errorf("bad file read")
//}
//
//func (f FS) Open(name string) (fs.File, error) {
//
// data := `type: string
//name: something
//in: query`
//
// return &openFile{&file{"test.yaml", data}, 0}, nil
//}
//
//func (f FSBadOpen) Open(name string) (fs.File, error) {
// return nil, errors.New("bad file open")
//}
//
//func (f FSBadRead) Open(name string) (fs.File, error) {
// return &badFileRead{&file{}, 0}, nil
//}
//
//func TestSpecIndex_UseRemoteHandler(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FS{}
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// // extract crs param from index
// crsParam := index.GetMappedReferences()["https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"]
// assert.NotNil(t, crsParam)
// assert.True(t, crsParam.IsRemote)
// assert.Equal(t, "string", crsParam.Node.Content[1].Value)
// assert.Equal(t, "something", crsParam.Node.Content[3].Value)
// assert.Equal(t, "query", crsParam.Node.Content[5].Value)
//}
//
//func TestSpecIndex_UseFileHandler(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "some-file-that-does-not-exist.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FS{}
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// // extract crs param from index
// crsParam := index.GetMappedReferences()["some-file-that-does-not-exist.yaml"]
// assert.NotNil(t, crsParam)
// assert.True(t, crsParam.IsRemote)
// assert.Equal(t, "string", crsParam.Node.Content[1].Value)
// assert.Equal(t, "something", crsParam.Node.Content[3].Value)
// assert.Equal(t, "query", crsParam.Node.Content[5].Value)
//}
//
//func TestSpecIndex_UseRemoteHandler_Error_Open(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://-i-cannot-be-opened.com"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadOpen{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to open remote file: bad file open", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseFileHandler_Error_Open(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test File Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "I-can-never-be-opened.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadOpen{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to open file: bad file open", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'I-can-never-be-opened.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseRemoteHandler_Error_Read(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://-i-cannot-be-opened.com"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadRead{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to read remote file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseFileHandler_Error_Read(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test File Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "I-am-impossible-to-open-forever.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadRead{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to read file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'I-am-impossible-to-open-forever.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseFileHandler_ErrorReference(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test File Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FS{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//func TestSpecIndex_Complex_Local_File_Design(t *testing.T) {
//
// main := `openapi: 3.1.0
//paths:
// /anything/circularReference:
// get:
// operationId: circularReferenceGet
// responses:
// "200":
// description: OK
// content:
// application/json:
// schema:
// $ref: "components.yaml#/components/schemas/validCircularReferenceObject"
// /anything/oneOfCircularReference:
// get:
// operationId: oneOfCircularReferenceGet
// tags:
// - generation
// responses:
// "200":
// description: OK
// content:
// application/json:
// schema:
// $ref: "components.yaml#/components/schemas/oneOfCircularReferenceObject"`
//
// components := `components:
// schemas:
// validCircularReferenceObject:
// type: object
// properties:
// circular:
// type: array
// items:
// $ref: "#/components/schemas/validCircularReferenceObject"
// oneOfCircularReferenceObject:
// type: object
// properties:
// child:
// oneOf:
// - $ref: "#/components/schemas/oneOfCircularReferenceObject"
// - $ref: "#/components/schemas/simpleObject"
// required:
// - child
// simpleObject:
// description: "simple"
// type: object
// properties:
// str:
// type: string
// description: "A string property."
// example: "example" `
//
// _ = os.WriteFile("components.yaml", []byte(components), 0644)
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(main), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}

46
index/index_model.go
View File

@@ -35,7 +35,6 @@ type Reference struct {
Circular bool Circular bool
Seen bool Seen bool
IsRemote bool IsRemote bool
//FileLocation string
RemoteLocation string RemoteLocation string
Path string // this won't always be available. Path string // this won't always be available.
RequiredRefProperties map[string][]string // definition names (eg, #/definitions/One) to a list of required properties on this definition which reference that definition RequiredRefProperties map[string][]string // definition names (eg, #/definitions/One) to a list of required properties on this definition which reference that definition
@@ -100,11 +99,6 @@ type SpecIndexConfig struct {
AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
AllowFileLookup bool // Allow file lookups for references. Defaults to false AllowFileLookup bool // Allow file lookups for references. Defaults to false
// ParentIndex allows the index to be created with knowledge of a parent, before being parsed. This allows
// a breakglass to be used to prevent loops, checking the tree before cursing down.
// deprecated: Use the Rolodex instead, this is no longer needed, indexes are finite and do not have children.
//ParentIndex *SpecIndex
// If set to true, the index will not be built out, which means only the foundational elements will be // If set to true, the index will not be built out, which means only the foundational elements will be
// parsed and added to the index. This is useful to avoid building out an index if the specification is // parsed and added to the index. This is useful to avoid building out an index if the specification is
// broken up into references and want it fully resolved. // broken up into references and want it fully resolved.
@@ -112,8 +106,12 @@ type SpecIndexConfig struct {
// Use the `BuildIndex()` method on the index to build it out once resolved/ready. // Use the `BuildIndex()` method on the index to build it out once resolved/ready.
AvoidBuildIndex bool AvoidBuildIndex bool
// If set to true, the index will not check for circular references automatically, this should be triggered
// manually, otherwise resolving may explode.
AvoidCircularReferenceCheck bool AvoidCircularReferenceCheck bool
// Logger is a logger that will be used for logging errors and warnings. If not set, the default logger
// will be used, set to the Error level.
Logger *slog.Logger Logger *slog.Logger
// SpecInfo is a pointer to the SpecInfo struct that contains the root node and the spec version. It's the // SpecInfo is a pointer to the SpecInfo struct that contains the root node and the spec version. It's the
@@ -125,6 +123,8 @@ type SpecIndexConfig struct {
// of its own automatically. // of its own automatically.
Rolodex *Rolodex Rolodex *Rolodex
// The absolute path to the spec file for the index. Will be absolute, either as a http link or a file.
// If the index is for a single file spec, then the root will be empty.
SpecAbsolutePath string SpecAbsolutePath string
// IgnorePolymorphicCircularReferences will skip over checking for circular references in polymorphic schemas. // IgnorePolymorphicCircularReferences will skip over checking for circular references in polymorphic schemas.
@@ -139,14 +139,7 @@ type SpecIndexConfig struct {
// this is disabled by default, which means array circular references will be checked. // this is disabled by default, which means array circular references will be checked.
IgnoreArrayCircularReferences bool IgnoreArrayCircularReferences bool
// SkipCircularReferenceCheck will skip over checking for circular references. This is disabled by default, which
// means circular references will be checked. This is useful for developers building out models that should be
// indexed later on.
//SkipCircularReferenceCheck bool
// private fields // private fields
//seenRemoteSources *syncmap.Map
//remoteLock *sync.Mutex
uri []string uri []string
} }
@@ -155,11 +148,9 @@ type SpecIndexConfig struct {
// //
// The default BasePath is the current working directory. // The default BasePath is the current working directory.
func CreateOpenAPIIndexConfig() *SpecIndexConfig { func CreateOpenAPIIndexConfig() *SpecIndexConfig {
//cw, _ := os.Getwd()
return &SpecIndexConfig{ return &SpecIndexConfig{
AllowRemoteLookup: true, AllowRemoteLookup: true,
AllowFileLookup: true, AllowFileLookup: true,
//seenRemoteSources: &syncmap.Map{},
} }
} }
@@ -168,13 +159,7 @@ func CreateOpenAPIIndexConfig() *SpecIndexConfig {
// //
// The default BasePath is the current working directory. // The default BasePath is the current working directory.
func CreateClosedAPIIndexConfig() *SpecIndexConfig { func CreateClosedAPIIndexConfig() *SpecIndexConfig {
//cw, _ := os.Getwd() return &SpecIndexConfig{}
return &SpecIndexConfig{
// BasePath: cw,
//AllowRemoteLookup: false,
//AllowFileLookup: false,
//seenRemoteSources: &syncmap.Map{},
}
} }
// SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and // SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and
@@ -267,8 +252,6 @@ type SpecIndex struct {
enumCount int enumCount int
descriptionCount int descriptionCount int
summaryCount int summaryCount int
//seenRemoteSources map[string]*yaml.Node
//seenLocalSources map[string]*yaml.Node
refLock sync.Mutex refLock sync.Mutex
componentLock sync.RWMutex componentLock sync.RWMutex
errorLock sync.RWMutex errorLock sync.RWMutex
@@ -278,16 +261,11 @@ type SpecIndex struct {
httpClient *http.Client httpClient *http.Client
componentIndexChan chan bool componentIndexChan chan bool
polyComponentIndexChan chan bool polyComponentIndexChan chan bool
specAbsolutePath string specAbsolutePath string
resolver *Resolver resolver *Resolver
cache syncmap.Map cache syncmap.Map
built bool built bool
//parentIndex *SpecIndex
uri []string uri []string
//children []*SpecIndex
} }
// GetResolver returns the resolver for this index. // GetResolver returns the resolver for this index.
@@ -300,16 +278,6 @@ func (index *SpecIndex) GetConfig() *SpecIndexConfig {
return index.config return index.config
} }
//// AddChild adds a child index to this index, a child index is an index created from a remote or file reference.
//func (index *SpecIndex) AddChild(child *SpecIndex) {
// index.children = append(index.children, child)
//}
//
//// GetChildren returns the children of this index.
//func (index *SpecIndex) GetChildren() []*SpecIndex {
// return index.children
//}
// ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the
// URI based document. Decides if the reference is local, remote or in a file. // URI based document. Decides if the reference is local, remote or in a file.
type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error) type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error)

16
index/index_model_test.go
View File

@@ -8,22 +8,6 @@ import (
"testing" "testing"
) )
//func TestSpecIndex_Children(t *testing.T) {
// idx1 := new(SpecIndex)
// idx2 := new(SpecIndex)
// idx3 := new(SpecIndex)
// idx4 := new(SpecIndex)
// idx5 := new(SpecIndex)
// idx1.AddChild(idx2)
// idx1.AddChild(idx3)
// idx3.AddChild(idx4)
// idx4.AddChild(idx5)
// assert.Equal(t, 2, len(idx1.GetChildren()))
// assert.Equal(t, 1, len(idx3.GetChildren()))
// assert.Equal(t, 1, len(idx4.GetChildren()))
// assert.Equal(t, 0, len(idx5.GetChildren()))
//}
func TestSpecIndex_GetConfig(t *testing.T) { func TestSpecIndex_GetConfig(t *testing.T) {
idx1 := new(SpecIndex) idx1 := new(SpecIndex)
c := SpecIndexConfig{} c := SpecIndexConfig{}

2
index/index_utils.go
View File

@@ -82,8 +82,6 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) {
index.securityRequirementRefs = make(map[string]map[string][]*Reference) index.securityRequirementRefs = make(map[string]map[string][]*Reference)
index.polymorphicRefs = make(map[string]*Reference) index.polymorphicRefs = make(map[string]*Reference)
index.refsWithSiblings = make(map[string]Reference) index.refsWithSiblings = make(map[string]Reference)
//index.seenRemoteSources = make(map[string]*yaml.Node)
//index.seenLocalSources = make(map[string]*yaml.Node)
index.opServersRefs = make(map[string]map[string][]*Reference) index.opServersRefs = make(map[string]map[string][]*Reference)
index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second} index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second}
index.componentIndexChan = make(chan bool) index.componentIndexChan = make(chan bool)

103
index/resolver.go
View File

@@ -369,35 +369,11 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
} }
var found []*Reference var found []*Reference
//var ignoredPoly []*index.Reference
//var ignoredArray []*index.Reference
if len(node.Content) > 0 { if len(node.Content) > 0 {
for i, n := range node.Content { for i, n := range node.Content {
if utils.IsNodeMap(n) || utils.IsNodeArray(n) { if utils.IsNodeMap(n) || utils.IsNodeArray(n) {
//var anyvn, allvn, onevn, arrayTypevn *yaml.Node
// extract polymorphic references
//if len(n.Content) > 1 {
//_, anyvn = utils.FindKeyNodeTop("anyOf", n.Content)
//_, allvn = utils.FindKeyNodeTop("allOf", n.Content)
//_, onevn = utils.FindKeyNodeTop("oneOf", n.Content)
//_, arrayTypevn = utils.FindKeyNodeTop("type", n.Content)
//}
//if anyvn != nil || allvn != nil || onevn != nil {
// if resolver.IgnorePoly {
// ignoredPoly = append(ignoredPoly, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...)
// }
//}
//if arrayTypevn != nil {
// if arrayTypevn.Value == "array" {
// if resolver.IgnoreArray {
// ignoredArray = append(ignoredArray, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...)
// }
// }
//}
found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, resolve)...) found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, resolve)...)
} }
@@ -411,8 +387,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
var locatedRef *Reference var locatedRef *Reference
var fullDef string var fullDef string
//exp := strings.Split(ref.FullDefinition, "#/")
var definition string var definition string
// explode value // explode value
@@ -445,13 +419,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0])) fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0]))
} }
//// split the full def into parts
//fileDef := strings.Split(ref.FullDefinition, "#/")
//
//// extract the location of the ref and build a full def path.
//
//fullDef = fmt.Sprintf("%s#/%s", fileDef[0], exp[1])
} }
} else { } else {
@@ -518,65 +485,10 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
} else { } else {
fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0])) fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0]))
}
} }
} }
} }
// }
//if len(exp) == 2 {
// if exp[0] != "" {
// fullDef = fmt.Sprintf("%s%s", exp[0], value)
// } else {
//
// //// check if location is relative
// //if filepath.IsAbs(exp)
// //
// //
//
// fullDef = value
// }
// definition = fmt.Sprintf("#/%s", exp[1])
//} else {
// if filepath.IsAbs(value) {
//
// // todo implement.
//
// } else {
// if strings.HasPrefix(value, "http") {
// fullDef = value
// definition = value
// } else {
// if ref.FullDefinition != "" {
// if strings.HasPrefix(ref.FullDefinition, "http") {
// u, _ := url.Parse(ref.FullDefinition)
// pathDir := filepath.Dir(u.Path)
// pathAbs, _ := filepath.Abs(filepath.Join(pathDir, value))
// u.Path = pathAbs
// fullDef = u.String()
// } else {
// if filepath.IsAbs(value) {
// fullDef = value
// } else {
//
// // extract file from value
// uri := strings.Split(value, "#/")
// if len(uri) == 2 {
//
// } else {
//
// }
//
// fullDef, _ = filepath.Abs(
// filepath.Join(
// filepath.Dir(ref.FullDefinition), value))
// }
// }
// }
// }
// }
//}
searchRef := &Reference{ searchRef := &Reference{
Definition: definition, Definition: definition,
@@ -585,16 +497,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
IsRemote: true, IsRemote: true,
} }
// we're searching a remote document, we need to build a full path to the reference
//if ref.IsRemote {
// if ref.RemoteLocation != "" {
// searchRef .RemoteLocation = ref.RemoteLocationFullDefinition: fmt.Sprintf("%s%s", ref.RemoteLocation, value),
// RemoteLocation: ref.RemoteLocation,
// IsRemote: true,
// }
// }
//}
locatedRef = resolver.specIndex.SearchIndexForReferenceByReference(searchRef) locatedRef = resolver.specIndex.SearchIndexForReferenceByReference(searchRef)
if locatedRef == nil { if locatedRef == nil {
@@ -715,12 +617,9 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
} }
break break
} }
} }
} }
} }
//resolver.ignoredPolyReferences = ignoredPoly
resolver.relativesSeen += len(found) resolver.relativesSeen += len(found)
return found return found
} }

17
index/rolodex.go
View File

@@ -282,7 +282,6 @@ func (r *Rolodex) IndexTheRolodex() error {
// for each index, we need a resolver // for each index, we need a resolver
resolver := NewResolver(idx) resolver := NewResolver(idx)
// idx.resolver = resolver
// check if the config has been set to ignore circular references in arrays and polymorphic schemas // check if the config has been set to ignore circular references in arrays and polymorphic schemas
if copiedConfig.IgnoreArrayCircularReferences { if copiedConfig.IgnoreArrayCircularReferences {
@@ -291,12 +290,6 @@ func (r *Rolodex) IndexTheRolodex() error {
if copiedConfig.IgnorePolymorphicCircularReferences { if copiedConfig.IgnorePolymorphicCircularReferences {
resolver.IgnorePolymorphicCircularReferences() resolver.IgnorePolymorphicCircularReferences()
} }
//if !copiedConfig.AvoidCircularReferenceCheck {
// resolvingErrors := resolver.CheckForCircularReferences()
// for e := range resolvingErrors {
// caughtErrors = append(caughtErrors, resolvingErrors[e])
// }
//}
if err != nil { if err != nil {
errChan <- err errChan <- err
@@ -364,10 +357,8 @@ func (r *Rolodex) IndexTheRolodex() error {
caughtErrors = append(caughtErrors, errs[e]) caughtErrors = append(caughtErrors, errs[e])
} }
} }
//}
// indexed and built every supporting file, we can build the root index (our entry point) // indexed and built every supporting file, we can build the root index (our entry point)
if r.rootNode != nil { if r.rootNode != nil {
// if there is a base path, then we need to set the root spec config to point to a theoretical root.yaml // if there is a base path, then we need to set the root spec config to point to a theoretical root.yaml
@@ -541,17 +532,14 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) {
// if there was no file found locally, then search the remote FS. // if there was no file found locally, then search the remote FS.
for _, v := range r.remoteFS { for _, v := range r.remoteFS {
f, err := v.Open(location) f, err := v.Open(location)
if err != nil { if err != nil {
errorStack = append(errorStack, err) errorStack = append(errorStack, err)
continue continue
} }
//fmt.Printf("found remote file: %s\n", fileLookup) if f != nil {
//fmt.Print(f)
return f.(*RemoteFile), nil return f.(*RemoteFile), nil
}
} }
} }
@@ -595,7 +583,6 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) {
} }
} }
} }
} }
} }

51
index/rolodex_ref_extractor.go
View File

@@ -57,54 +57,3 @@ func ExtractFileType(ref string) FileExtension {
} }
return UNSUPPORTED return UNSUPPORTED
} }
func ExtractRefValues(ref string) (location, id string) {
split := strings.Split(ref, "#/")
if len(split) > 1 && split[0] != "" {
location = split[0]
id = split[1]
}
if len(split) > 1 && split[0] == "" {
id = split[1]
}
if len(split) == 1 {
location = ref
}
return
}
func ExtractRefType(ref string) RefType {
if strings.HasPrefix(ref, "http") {
return HTTP
}
if strings.HasPrefix(ref, "/") {
return File
}
if strings.HasPrefix(ref, "..") {
return File
}
if strings.HasPrefix(ref, "./") {
return File
}
split := strings.Split(ref, "#/")
if len(split) > 1 && split[0] != "" {
return File
}
if strings.HasSuffix(ref, ".yaml") {
return File
}
if strings.HasSuffix(ref, ".json") {
return File
}
return Local
}
func ExtractRefs(content string) [][]string {
return refRegex.FindAllStringSubmatch(content, -1)
//var results []*ExtractedRef
//for _, r := range res {
// results = append(results, &ExtractedRef{Location: r[1], Type: ExtractRefType(r[1])})
//}
}

145
index/rolodex_ref_extractor_test.go
View File

@@ -8,151 +8,6 @@ import (
"testing" "testing"
) )
func TestExtractRefs_Local(t *testing.T) {
test := `openapi: 3.0
paths:
/burgers:
post:
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/Nine'
components:
schemas:
One:
description: "test one"
properties:
things:
"$ref": "#/components/schemas/Two"
required:
- things
Two:
description: "test two"
properties:
testThing:
"$ref": "#/components/schemas/One"
anyOf:
- "$ref": "#/components/schemas/Four"
required:
- testThing
- anyOf
Three:
description: "test three"
properties:
tester:
"$ref": "#/components/schemas/Four"
bester:
"$ref": "#/components/schemas/Seven"
yester:
"$ref": "#/components/schemas/Seven"
required:
- tester
- bester
- yester
Four:
description: "test four"
properties:
lemons:
"$ref": "#/components/schemas/Nine"
required:
- lemons
Five:
properties:
rice:
"$ref": "#/components/schemas/Six"
required:
- rice
Six:
properties:
mints:
"$ref": "#/components/schemas/Nine"
required:
- mints
Seven:
properties:
wow:
"$ref": "#/components/schemas/Three"
required:
- wow
Nine:
description: done.
Ten:
properties:
yeah:
"$ref": "#/components/schemas/Ten"
required:
- yeah`
results := ExtractRefs(test)
assert.Len(t, results, 12)
}
func TestExtractRefs_File(t *testing.T) {
test := `openapi: 3.0
paths:
/burgers:
post:
requestBody:
content:
application/json:
schema:
$ref: 'pizza.yaml#/components/schemas/Nine'
components:
schemas:
One:
description: "test one"
properties:
things:
"$ref": "../../fish.yaml#/components/schemas/Two"
required:
- things
Two:
description: "test two"
properties:
testThing:
"$ref": "../../../lost/no.yaml#/components/schemas/One"
anyOf:
- "$ref": "why.yaml#/components/schemas/Four"
required:
- testThing
- anyOf
Three:
description: "test three"
properties:
tester:
"$ref": "no_more.yaml"
bester:
"$ref": 'why.yaml'
yester:
"$ref": "../../yes.yaml"
required:
- tester
- bester
- yester`
results := ExtractRefs(test)
assert.Len(t, results, 7)
}
func TestExtractRefType(t *testing.T) {
assert.Equal(t, Local, ExtractRefType("#/components/schemas/One"))
assert.Equal(t, File, ExtractRefType("pizza.yaml#/components/schemas/One"))
assert.Equal(t, File, ExtractRefType("/pizza.yaml#/components/schemas/One"))
assert.Equal(t, File, ExtractRefType("/something/pizza.yaml#/components/schemas/One"))
assert.Equal(t, File, ExtractRefType("./pizza.yaml#/components/schemas/One"))
assert.Equal(t, File, ExtractRefType("../pizza.yaml#/components/schemas/One"))
assert.Equal(t, File, ExtractRefType("../../../pizza.yaml#/components/schemas/One"))
assert.Equal(t, HTTP, ExtractRefType("http://yeah.com/pizza.yaml#/components/schemas/One"))
assert.Equal(t, HTTP, ExtractRefType("https://yeah.com/pizza.yaml#/components/schemas/One"))
}
func TestExtractedRef_GetFile(t *testing.T) { func TestExtractedRef_GetFile(t *testing.T) {
a := &ExtractedRef{Location: "#/components/schemas/One", Type: Local} a := &ExtractedRef{Location: "#/components/schemas/One", Type: Local}

4
index/rolodex_remote_loader.go
View File

@@ -20,6 +20,8 @@ import (
"time" "time"
) )
type RemoteURLHandler = func(url string) (*http.Response, error)
type RemoteFS struct { type RemoteFS struct {
indexConfig *SpecIndexConfig indexConfig *SpecIndexConfig
rootURL string rootURL string
@@ -414,7 +416,9 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
filepath.Dir(remoteParsedURL.Path)) filepath.Dir(remoteParsedURL.Path))
newBaseURL, _ := url.Parse(newBase) newBaseURL, _ := url.Parse(newBase)
if newBaseURL != nil {
copiedCfg.BaseURL = newBaseURL copiedCfg.BaseURL = newBaseURL
}
copiedCfg.SpecAbsolutePath = remoteParsedURL.String() copiedCfg.SpecAbsolutePath = remoteParsedURL.String()
idx, idxError := remoteFile.Index(&copiedCfg) idx, idxError := remoteFile.Index(&copiedCfg)

108
index/rolodex_remote_loader_test.go
View File

@@ -8,6 +8,7 @@ import (
"io" "io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url"
"testing" "testing"
"time" "time"
) )
@@ -106,55 +107,58 @@ func TestNewRemoteFS_BasicCheck(t *testing.T) {
assert.Equal(t, "2015-10-21 07:28:00 +0000 GMT", lastMod.String()) assert.Equal(t, "2015-10-21 07:28:00 +0000 GMT", lastMod.String())
} }
// func TestNewRemoteFS_BasicCheck_Relative(t *testing.T) {
//func TestNewRemoteFS_BasicCheck_Relative(t *testing.T) {
// server := test_buildServer()
// server := test_buildServer() defer server.Close()
// defer server.Close()
// remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
// remoteFS, _ := NewRemoteFSWithRootURL(server.URL) remoteFS.RemoteHandlerFunc = test_httpClient.Get
// remoteFS.RemoteHandlerFunc = test_httpClient.Get
// file, err := remoteFS.Open("/deeper/file2.yaml")
// file, err := remoteFS.Open("/deeper/file2.yaml")
// assert.NoError(t, err)
// assert.NoError(t, err)
// bytes, rErr := io.ReadAll(file)
// bytes, rErr := io.ReadAll(file) assert.NoError(t, rErr)
// assert.NoError(t, rErr)
// assert.Len(t, bytes, 64)
// assert.Len(t, bytes, 64)
// stat, _ := file.Stat()
// stat, _ := file.Stat()
// assert.Equal(t, "/deeper/file2.yaml", stat.Name())
// assert.Equal(t, "/deeper/file2.yaml", stat.Name()) assert.Equal(t, int64(64), stat.Size())
// assert.Equal(t, int64(64), stat.Size())
// lastMod := stat.ModTime()
// lastMod := stat.ModTime() assert.Equal(t, "2015-10-21 08:28:00 +0000 GMT", lastMod.String())
// assert.Equal(t, "2015-10-21 08:28:00 +0000 GMT", lastMod.String()) }
//}
// func TestNewRemoteFS_BasicCheck_Relative_Deeper(t *testing.T) {
//func TestNewRemoteFS_BasicCheck_Relative_Deeper(t *testing.T) {
// server := test_buildServer()
// server := test_buildServer() defer server.Close()
// defer server.Close()
// cf := CreateOpenAPIIndexConfig()
// remoteFS, _ := NewRemoteFSWithRootURL(server.URL) u, _ := url.Parse(server.URL)
// remoteFS.RemoteHandlerFunc = test_httpClient.Get cf.BaseURL = u
//
// file, err := remoteFS.Open("/deeper/even_deeper/file3.yaml") remoteFS, _ := NewRemoteFSWithConfig(cf)
// remoteFS.RemoteHandlerFunc = test_httpClient.Get
// assert.NoError(t, err)
// file, err := remoteFS.Open("/deeper/even_deeper/file3.yaml")
// bytes, rErr := io.ReadAll(file)
// assert.NoError(t, rErr) assert.NoError(t, err)
//
// assert.Len(t, bytes, 47) bytes, rErr := io.ReadAll(file)
// assert.NoError(t, rErr)
// stat, _ := file.Stat()
// assert.Len(t, bytes, 47)
// assert.Equal(t, "/deeper/even_deeper/file3.yaml", stat.Name())
// assert.Equal(t, int64(47), stat.Size()) stat, _ := file.Stat()
//
// lastMod := stat.ModTime() assert.Equal(t, "/deeper/even_deeper/file3.yaml", stat.Name())
// assert.Equal(t, "2015-10-21 10:28:00 +0000 GMT", lastMod.String()) assert.Equal(t, int64(47), stat.Size())
//}
lastMod := stat.ModTime()
assert.Equal(t, "2015-10-21 10:28:00 +0000 GMT", lastMod.String())
}

8
index/search_index.go
View File

@@ -11,9 +11,9 @@ import (
func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *Reference { func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *Reference {
//if v, ok := index.cache.Load(fullRef); ok { if v, ok := index.cache.Load(fullRef); ok {
// return v.(*Reference) return v.(*Reference)
//} }
ref := fullRef.FullDefinition ref := fullRef.FullDefinition
refAlt := ref refAlt := ref
@@ -39,14 +39,12 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
} }
} else { } else {
//roloLookup = absPath // hang on a jiffy whiffy
if filepath.Ext(uri[1]) != "" { if filepath.Ext(uri[1]) != "" {
roloLookup = absPath roloLookup = absPath
} else { } else {
roloLookup = "" roloLookup = ""
} }
//ref = fmt.Sprintf("%s#/%s", absPath, uri[1]) this seems wrong
ref = fmt.Sprintf("#/%s", uri[1]) ref = fmt.Sprintf("#/%s", uri[1])
refAlt = fmt.Sprintf("%s#/%s", absPath, uri[1]) refAlt = fmt.Sprintf("%s#/%s", absPath, uri[1])

13
index/spec_index.go
View File

@@ -1241,16 +1241,3 @@ func (index *SpecIndex) GetAllDescriptionsCount() int {
func (index *SpecIndex) GetAllSummariesCount() int { func (index *SpecIndex) GetAllSummariesCount() int {
return len(index.allSummaries) return len(index.allSummaries)
} }
// CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it,
// to avoid making duplicate remote calls for document data.
//func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) {
// if index.config == nil || index.config.seenRemoteSources == nil {
// return false, nil
// }
// j, _ := index.config.seenRemoteSources.Load(url)
// if j != nil {
// return true, j.(*yaml.Node)
// }
// return false, nil
//}

86
index/spec_index_test.go
View File

@@ -837,51 +837,6 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) {
assert.Nil(t, index.lookupRolodex(nil)) assert.Nil(t, index.lookupRolodex(nil))
} }
//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) {
// index := new(SpecIndex)
// index.seenRemoteSources = make(map[string]*yaml.Node)
// index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
// _, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something")
// assert.Error(t, err)
//}
//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing.T) {
// index := new(SpecIndex)
// index.seenRemoteSources = make(map[string]*yaml.Node)
// index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
// a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
// assert.Error(t, err)
// assert.Nil(t, a)
// assert.Nil(t, b)
//}
// Discovered in issue https://github.com/pb33f/libopenapi/issues/37
//func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) {
// index := new(SpecIndex)
// index.seenRemoteSources = make(map[string]*yaml.Node)
// index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
// a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
// assert.NoError(t, err)
// assert.NotNil(t, a)
// assert.NotNil(t, b)
//}
// Discovered in issue https://github.com/daveshanley/vacuum/issues/225
//func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
// cwd, _ := os.Getwd()
// index := new(SpecIndex)
// index.config = &SpecIndexConfig{BasePath: cwd}
//
// _ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664)
// defer os.Remove("coffee-time.yaml")
//
// //index.seenRemoteSources = make(map[string]*yaml.Node)
// a, b, err := index.lookupFileReference("coffee-time.yaml")
// assert.NoError(t, err)
// assert.NotNil(t, a)
// assert.NotNil(t, b)
//}
func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) { func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) {
yml := `openapi: 3.1.0 yml := `openapi: 3.1.0
paths: paths:
@@ -947,47 +902,6 @@ paths:
assert.NotNil(t, index.GetAllParametersFromOperations()["/cakes"]["post"]["coffee-time.yaml"][0].Node) assert.NotNil(t, index.GetAllParametersFromOperations()["/cakes"]["post"]["coffee-time.yaml"][0].Node)
} }
//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) {
// index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{
// //AllowRemoteLookup: true,
// })
// index.seenRemoteSources = make(map[string]*yaml.Node)
// a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey")
// assert.Error(t, err)
// assert.Nil(t, a)
// assert.Nil(t, b)
//}
//func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) {
// index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
// _, _, err := index.lookupFileReference("not-a-reference")
// assert.Error(t, err)
//}
//
//func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) {
// index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
// index.seenRemoteSources = make(map[string]*yaml.Node)
// index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{}
// _, _, err := index.lookupFileReference("magic-money-file.json#something")
// assert.Error(t, err)
//}
//
//func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) {
// index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
// _, _, err := index.lookupFileReference("chickers.json#no-rice")
// assert.Error(t, err)
//}
//
//func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) {
// _ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664)
// defer os.Remove("chickers.yaml")
// var root yaml.Node
// index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig())
// _, _, err := index.lookupFileReference("chickers.yaml#no-rice")
// assert.Error(t, err)
//}
func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) { func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) {
embie := []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy") embie := []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy")