LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-09 20:47:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

index tests all pass! now time to clean.

Browse Source 
Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-10-21 17:29:53 -04:00
parent bf270d3d2b
commit be7e477529
10 changed files with 580 additions and 602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
index/find_component.go
View File

@@ -5,12 +5,10 @@ package index
import (
"fmt"
"io"
"net/http"
"net/url"
"path/filepath"
"strings"
"time"
"github.com/pb33f/libopenapi/utils"
"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
@@ -80,24 +78,22 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
//return nil
}
var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second}
type RemoteURLHandler = func(url string) (*http.Response, error)
func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
resp, err := g(u)
if err != nil {
e <- err
close(e)
close(d)
return
}
var body []byte
body, _ = io.ReadAll(resp.Body)
d <- body
close(e)
close(d)
}
//func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
// resp, err := g(u)
// if err != nil {
// e <- err
// close(e)
// close(d)
// return
// }
// var body []byte
// body, _ = io.ReadAll(resp.Body)
// d <- body
// close(e)
// close(d)
//}
//func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
// // split string to remove file reference

703
index/find_component_test.go
View File

@@ -4,16 +4,9 @@
package index
import (
"errors"
"fmt"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"reflect"
"testing"
)
@@ -222,7 +215,7 @@ paths:
index := rolo.GetRootIndex()
// extract crs param from index
crsParam := index.GetMappedReferences()["#/components/parameters/crs"]
crsParam := index.GetMappedReferences()["https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"]
assert.NotNil(t, crsParam)
assert.True(t, crsParam.IsRemote)
assert.Equal(t, "crs", crsParam.Node.Content[1].Value)
@@ -245,12 +238,10 @@ paths:
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'https://petstore3.swagger.io/api/v3/openapi.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
assert.Len(t, index.GetReferenceIndexErrors(), 1)
assert.Equal(t, "component '#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[0].Error())
}
func TestSpecIndex_LocateRemoteDocsWithEscapedCharacters(t *testing.T) {
@@ -268,354 +259,354 @@ paths:
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 0)
assert.Len(t, index.GetReferenceIndexErrors(), 1)
}
func TestGetRemoteDoc(t *testing.T) {
// Mock HTTP server
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Write([]byte(`OK`))
}))
// Close the server when test finishes
defer server.Close()
// Channel for data and error
dataChan := make(chan []byte)
errorChan := make(chan error)
go getRemoteDoc(http.Get, server.URL, dataChan, errorChan)
data := <-dataChan
err := <-errorChan
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
expectedData := []byte(`OK`)
if !reflect.DeepEqual(data, expectedData) {
t.Errorf("Expected %v, got %v", expectedData, data)
}
}
type FS struct{}
type FSBadOpen struct{}
type FSBadRead struct{}
type file struct {
name string
data string
}
type openFile struct {
f *file
offset int64
}
func (f *openFile) Close() error { return nil }
func (f *openFile) Stat() (fs.FileInfo, error) { return nil, nil }
func (f *openFile) Read(b []byte) (int, error) {
if f.offset >= int64(len(f.f.data)) {
return 0, io.EOF
}
if f.offset < 0 {
return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid}
}
n := copy(b, f.f.data[f.offset:])
f.offset += int64(n)
return n, nil
}
//type badFileOpen struct{}
//
//func (f *badFileOpen) Close() error { return errors.New("bad file close") }
//func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
//func (f *badFileOpen) Read(b []byte) (int, error) {
// return 0, nil
//func TestGetRemoteDoc(t *testing.T) {
// // Mock HTTP server
// server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
// rw.Write([]byte(`OK`))
// }))
// // Close the server when test finishes
// defer server.Close()
//
// // Channel for data and error
// dataChan := make(chan []byte)
// errorChan := make(chan error)
//
// go getRemoteDoc(http.Get, server.URL, dataChan, errorChan)
//
// data := <-dataChan
// err := <-errorChan
//
// if err != nil {
// t.Errorf("Expected no error, got %v", err)
// }
//
// expectedData := []byte(`OK`)
// if !reflect.DeepEqual(data, expectedData) {
// t.Errorf("Expected %v, got %v", expectedData, data)
// }
//}
//
//type FS struct{}
//type FSBadOpen struct{}
//type FSBadRead struct{}
//
//type file struct {
// name string
// data string
//}
//
//type openFile struct {
// f *file
// offset int64
//}
//
//func (f *openFile) Close() error { return nil }
//func (f *openFile) Stat() (fs.FileInfo, error) { return nil, nil }
//func (f *openFile) Read(b []byte) (int, error) {
// if f.offset >= int64(len(f.f.data)) {
// return 0, io.EOF
// }
// if f.offset < 0 {
// return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid}
// }
// n := copy(b, f.f.data[f.offset:])
// f.offset += int64(n)
// return n, nil
//}
//
////type badFileOpen struct{}
////
////func (f *badFileOpen) Close() error { return errors.New("bad file close") }
////func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
////func (f *badFileOpen) Read(b []byte) (int, error) {
//// return 0, nil
////}
//
//type badFileRead struct {
// f *file
// offset int64
//}
//
//func (f *badFileRead) Close() error { return errors.New("bad file close") }
//func (f *badFileRead) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
//func (f *badFileRead) Read(b []byte) (int, error) {
// return 0, fmt.Errorf("bad file read")
//}
//
//func (f FS) Open(name string) (fs.File, error) {
//
// data := `type: string
//name: something
//in: query`
//
// return &openFile{&file{"test.yaml", data}, 0}, nil
//}
//
//func (f FSBadOpen) Open(name string) (fs.File, error) {
// return nil, errors.New("bad file open")
//}
//
//func (f FSBadRead) Open(name string) (fs.File, error) {
// return &badFileRead{&file{}, 0}, nil
//}
//
//func TestSpecIndex_UseRemoteHandler(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FS{}
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// // extract crs param from index
// crsParam := index.GetMappedReferences()["https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"]
// assert.NotNil(t, crsParam)
// assert.True(t, crsParam.IsRemote)
// assert.Equal(t, "string", crsParam.Node.Content[1].Value)
// assert.Equal(t, "something", crsParam.Node.Content[3].Value)
// assert.Equal(t, "query", crsParam.Node.Content[5].Value)
//}
//
//func TestSpecIndex_UseFileHandler(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "some-file-that-does-not-exist.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FS{}
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// // extract crs param from index
// crsParam := index.GetMappedReferences()["some-file-that-does-not-exist.yaml"]
// assert.NotNil(t, crsParam)
// assert.True(t, crsParam.IsRemote)
// assert.Equal(t, "string", crsParam.Node.Content[1].Value)
// assert.Equal(t, "something", crsParam.Node.Content[3].Value)
// assert.Equal(t, "query", crsParam.Node.Content[5].Value)
//}
//
//func TestSpecIndex_UseRemoteHandler_Error_Open(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://-i-cannot-be-opened.com"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadOpen{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to open remote file: bad file open", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseFileHandler_Error_Open(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test File Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "I-can-never-be-opened.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadOpen{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to open file: bad file open", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'I-can-never-be-opened.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseRemoteHandler_Error_Read(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test Remote Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://-i-cannot-be-opened.com"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadRead{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to read remote file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseFileHandler_Error_Read(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test File Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "I-am-impossible-to-open-forever.yaml"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FSBadRead{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, "unable to read file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'I-am-impossible-to-open-forever.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
//
//func TestSpecIndex_UseFileHandler_ErrorReference(t *testing.T) {
//
// spec := `openapi: 3.1.0
//info:
// title: Test File Handler
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters"`
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(spec), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// c.FSHandler = FS{}
// c.RemoteURLHandler = httpClient.Get
//
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}
type badFileRead struct {
f *file
offset int64
}
func (f *badFileRead) Close() error { return errors.New("bad file close") }
func (f *badFileRead) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
func (f *badFileRead) Read(b []byte) (int, error) {
return 0, fmt.Errorf("bad file read")
}
func (f FS) Open(name string) (fs.File, error) {
data := `type: string
name: something
in: query`
return &openFile{&file{"test.yaml", data}, 0}, nil
}
func (f FSBadOpen) Open(name string) (fs.File, error) {
return nil, errors.New("bad file open")
}
func (f FSBadRead) Open(name string) (fs.File, error) {
return &badFileRead{&file{}, 0}, nil
}
func TestSpecIndex_UseRemoteHandler(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test Remote Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FS{}
index := NewSpecIndexWithConfig(&rootNode, c)
// extract crs param from index
crsParam := index.GetMappedReferences()["https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"]
assert.NotNil(t, crsParam)
assert.True(t, crsParam.IsRemote)
assert.Equal(t, "string", crsParam.Node.Content[1].Value)
assert.Equal(t, "something", crsParam.Node.Content[3].Value)
assert.Equal(t, "query", crsParam.Node.Content[5].Value)
}
func TestSpecIndex_UseFileHandler(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test Remote Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "some-file-that-does-not-exist.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FS{}
index := NewSpecIndexWithConfig(&rootNode, c)
// extract crs param from index
crsParam := index.GetMappedReferences()["some-file-that-does-not-exist.yaml"]
assert.NotNil(t, crsParam)
assert.True(t, crsParam.IsRemote)
assert.Equal(t, "string", crsParam.Node.Content[1].Value)
assert.Equal(t, "something", crsParam.Node.Content[3].Value)
assert.Equal(t, "query", crsParam.Node.Content[5].Value)
}
func TestSpecIndex_UseRemoteHandler_Error_Open(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test Remote Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "https://-i-cannot-be-opened.com"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FSBadOpen{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, "unable to open remote file: bad file open", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseFileHandler_Error_Open(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test File Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "I-can-never-be-opened.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FSBadOpen{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, "unable to open file: bad file open", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'I-can-never-be-opened.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseRemoteHandler_Error_Read(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test Remote Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "https://-i-cannot-be-opened.com"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FSBadRead{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, "unable to read remote file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseFileHandler_Error_Read(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test File Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "I-am-impossible-to-open-forever.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FSBadRead{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, "unable to read file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'I-am-impossible-to-open-forever.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseFileHandler_ErrorReference(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test File Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FS{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_Complex_Local_File_Design(t *testing.T) {
main := `openapi: 3.1.0
paths:
/anything/circularReference:
get:
operationId: circularReferenceGet
responses:
"200":
description: OK
content:
application/json:
schema:
$ref: "components.yaml#/components/schemas/validCircularReferenceObject"
/anything/oneOfCircularReference:
get:
operationId: oneOfCircularReferenceGet
tags:
- generation
responses:
"200":
description: OK
content:
application/json:
schema:
$ref: "components.yaml#/components/schemas/oneOfCircularReferenceObject"`
components := `components:
schemas:
validCircularReferenceObject:
type: object
properties:
circular:
type: array
items:
$ref: "#/components/schemas/validCircularReferenceObject"
oneOfCircularReferenceObject:
type: object
properties:
child:
oneOf:
- $ref: "#/components/schemas/oneOfCircularReferenceObject"
- $ref: "#/components/schemas/simpleObject"
required:
- child
simpleObject:
description: "simple"
type: object
properties:
str:
type: string
description: "A string property."
example: "example" `
_ = os.WriteFile("components.yaml", []byte(components), 0644)
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(main), &rootNode)
c := CreateOpenAPIIndexConfig()
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
//func TestSpecIndex_Complex_Local_File_Design(t *testing.T) {
//
// main := `openapi: 3.1.0
//paths:
// /anything/circularReference:
// get:
// operationId: circularReferenceGet
// responses:
// "200":
// description: OK
// content:
// application/json:
// schema:
// $ref: "components.yaml#/components/schemas/validCircularReferenceObject"
// /anything/oneOfCircularReference:
// get:
// operationId: oneOfCircularReferenceGet
// tags:
// - generation
// responses:
// "200":
// description: OK
// content:
// application/json:
// schema:
// $ref: "components.yaml#/components/schemas/oneOfCircularReferenceObject"`
//
// components := `components:
// schemas:
// validCircularReferenceObject:
// type: object
// properties:
// circular:
// type: array
// items:
// $ref: "#/components/schemas/validCircularReferenceObject"
// oneOfCircularReferenceObject:
// type: object
// properties:
// child:
// oneOf:
// - $ref: "#/components/schemas/oneOfCircularReferenceObject"
// - $ref: "#/components/schemas/simpleObject"
// required:
// - child
// simpleObject:
// description: "simple"
// type: object
// properties:
// str:
// type: string
// description: "A string property."
// example: "example" `
//
// _ = os.WriteFile("components.yaml", []byte(components), 0644)
//
// var rootNode yaml.Node
// _ = yaml.Unmarshal([]byte(main), &rootNode)
//
// c := CreateOpenAPIIndexConfig()
// index := NewSpecIndexWithConfig(&rootNode, c)
//
// assert.Len(t, index.GetReferenceIndexErrors(), 2)
// assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
// assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
//}

14
index/index_model.go
View File

@@ -5,11 +5,11 @@ package index
import (
"github.com/pb33f/libopenapi/datamodel"
"golang.org/x/sync/syncmap"
"io/fs"
"log/slog"
"net/http"
"net/url"
"os"
"sync"
"gopkg.in/yaml.v3"
@@ -155,11 +155,10 @@ type SpecIndexConfig struct {
//
// The default BasePath is the current working directory.
func CreateOpenAPIIndexConfig() *SpecIndexConfig {
cw, _ := os.Getwd()
//cw, _ := os.Getwd()
return &SpecIndexConfig{
BasePath: cw,
//AllowRemoteLookup: true,
//AllowFileLookup: true,
AllowRemoteLookup: true,
AllowFileLookup: true,
//seenRemoteSources: &syncmap.Map{},
}
}
@@ -169,9 +168,9 @@ func CreateOpenAPIIndexConfig() *SpecIndexConfig {
//
// The default BasePath is the current working directory.
func CreateClosedAPIIndexConfig() *SpecIndexConfig {
cw, _ := os.Getwd()
//cw, _ := os.Getwd()
return &SpecIndexConfig{
BasePath: cw,
// BasePath: cw,
//AllowRemoteLookup: false,
//AllowFileLookup: false,
//seenRemoteSources: &syncmap.Map{},
@@ -282,6 +281,7 @@ type SpecIndex struct {
specAbsolutePath string
resolver *Resolver
cache syncmap.Map
built bool

2
index/resolver.go
View File

@@ -264,7 +264,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
// check if we have seen this on the journey before, if so! it's circular
skip := false
for i, j := range journey {
if j.Definition == r.Definition {
if j.FullDefinition == r.FullDefinition {
var foundDup *Reference
foundRef := resolver.specIndex.SearchIndexForReferenceByReference(r)

96
index/resolver_test.go
View File

@@ -42,15 +42,18 @@ func TestResolver_ResolveComponents_CircularSpec(t *testing.T) {
var rootNode yaml.Node
_ = yaml.Unmarshal(circular, &rootNode)
idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig())
cf := CreateClosedAPIIndexConfig()
cf.AvoidCircularReferenceCheck = true
rolo := NewRolodex(cf)
rolo.SetRootNode(&rootNode)
resolver := NewResolver(idx)
assert.NotNil(t, resolver)
indexedErr := rolo.IndexTheRolodex()
assert.NoError(t, indexedErr)
circ := resolver.Resolve()
assert.Len(t, circ, 3)
rolo.Resolve()
assert.Len(t, rolo.GetCaughtErrors(), 3)
_, err := yaml.Marshal(resolver.resolvedRoot)
_, err := yaml.Marshal(rolo.GetRootIndex().GetResolver().resolvedRoot)
assert.NoError(t, err)
}
@@ -59,18 +62,21 @@ func TestResolver_CheckForCircularReferences(t *testing.T) {
var rootNode yaml.Node
_ = yaml.Unmarshal(circular, &rootNode)
idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig())
cf := CreateClosedAPIIndexConfig()
resolver := NewResolver(idx)
assert.NotNil(t, resolver)
rolo := NewRolodex(cf)
rolo.SetRootNode(&rootNode)
circ := resolver.CheckForCircularReferences()
assert.Len(t, circ, 3)
assert.Len(t, resolver.GetResolvingErrors(), 3)
assert.Len(t, resolver.GetCircularErrors(), 3)
indexedErr := rolo.IndexTheRolodex()
assert.Error(t, indexedErr)
assert.Len(t, utils.UnwrapErrors(indexedErr), 3)
rolo.CheckForCircularReferences()
assert.Len(t, rolo.GetCaughtErrors(), 3)
assert.Len(t, rolo.GetRootIndex().GetResolver().GetResolvingErrors(), 3)
assert.Len(t, rolo.GetRootIndex().GetResolver().GetCircularErrors(), 3)
_, err := yaml.Marshal(resolver.resolvedRoot)
assert.NoError(t, err)
}
func TestResolver_CheckForCircularReferences_CatchArray(t *testing.T) {
@@ -321,31 +327,6 @@ components:
assert.NoError(t, err)
}
func TestResolver_CheckForCircularReferences_DigitalOcean(t *testing.T) {
circular, _ := os.ReadFile("../test_specs/digitalocean.yaml")
var rootNode yaml.Node
_ = yaml.Unmarshal(circular, &rootNode)
baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification")
idx := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
//AllowRemoteLookup: true,
//AllowFileLookup: true,
BaseURL: baseURL,
})
resolver := NewResolver(idx)
assert.NotNil(t, resolver)
circ := resolver.CheckForCircularReferences()
assert.Len(t, circ, 0)
assert.Len(t, resolver.GetResolvingErrors(), 0)
assert.Len(t, resolver.GetCircularErrors(), 0)
_, err := yaml.Marshal(resolver.resolvedRoot)
assert.NoError(t, err)
}
func TestResolver_CircularReferencesRequiredValid(t *testing.T) {
circular, _ := os.ReadFile("../test_specs/swagger-valid-recursive-model.yaml")
var rootNode yaml.Node
@@ -390,7 +371,7 @@ func TestResolver_DeepJourney(t *testing.T) {
assert.Nil(t, resolver.extractRelatives(nil, nil, nil, nil, journey, false))
}
func TestResolver_ResolveComponents_Stripe(t *testing.T) {
func TestResolver_ResolveComponents_Stripe_NoRolodex(t *testing.T) {
baseDir := "../test_specs/stripe.yaml"
resolveFile, _ := os.ReadFile(baseDir)
@@ -403,14 +384,45 @@ func TestResolver_ResolveComponents_Stripe(t *testing.T) {
cf := CreateOpenAPIIndexConfig()
cf.SpecInfo = info
idx := NewSpecIndexWithConfig(&stripeRoot, cf)
resolver := NewResolver(idx)
assert.NotNil(t, resolver)
circ := resolver.CheckForCircularReferences()
assert.Len(t, circ, 3)
_, err := yaml.Marshal(resolver.resolvedRoot)
assert.NoError(t, err)
}
func TestResolver_ResolveComponents_Stripe(t *testing.T) {
baseDir := "../test_specs/stripe.yaml"
resolveFile, _ := os.ReadFile(baseDir)
var stripeRoot yaml.Node
_ = yaml.Unmarshal(resolveFile, &stripeRoot)
info, _ := datamodel.ExtractSpecInfoWithDocumentCheck(resolveFile, true)
cf := CreateOpenAPIIndexConfig()
cf.SpecInfo = info
cf.AvoidCircularReferenceCheck = true
rolo := NewRolodex(cf)
rolo.SetRootNode(&stripeRoot)
indexedErr := rolo.IndexTheRolodex()
assert.NoError(t, indexedErr)
assert.Len(t, utils.UnwrapErrors(indexedErr), 3)
// after resolving, the rolodex will have errors.
rolo.Resolve()
assert.Len(t, rolo.GetCaughtErrors(), 3)
assert.Len(t, rolo.GetRootIndex().GetResolver().GetNonPolymorphicCircularErrors(), 3)
assert.Len(t, rolo.GetRootIndex().GetResolver().GetPolymorphicCircularErrors(), 0)
}
func TestResolver_ResolveComponents_BurgerShop(t *testing.T) {

30
index/rolodex.go
View File

@@ -61,6 +61,7 @@ type Rolodex struct {
indexed bool
built bool
resolved bool
circChecked bool
indexConfig *SpecIndexConfig
indexingDuration time.Duration
indexes []*SpecIndex
@@ -377,7 +378,10 @@ func (r *Rolodex) IndexTheRolodex() error {
if !filepath.IsAbs(basePath) {
basePath, _ = filepath.Abs(basePath)
}
r.indexConfig.SpecAbsolutePath = filepath.Join(basePath, "root.yaml")
if len(r.localFS) > 0 || len(r.remoteFS) > 0 {
r.indexConfig.SpecAbsolutePath = filepath.Join(basePath, "root.yaml")
}
}
// todo: variation with no base path, but a base URL.
@@ -396,6 +400,7 @@ func (r *Rolodex) IndexTheRolodex() error {
if !r.indexConfig.AvoidCircularReferenceCheck {
resolvingErrors := resolver.CheckForCircularReferences()
r.circChecked = true
for e := range resolvingErrors {
caughtErrors = append(caughtErrors, resolvingErrors[e])
}
@@ -414,17 +419,20 @@ func (r *Rolodex) IndexTheRolodex() error {
}
func (r *Rolodex) CheckForCircularReferences() {
if r.rootIndex != nil && r.rootIndex.resolver != nil {
resolvingErrors := r.rootIndex.resolver.CheckForCircularReferences()
for e := range resolvingErrors {
r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
}
if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
}
if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
if !r.circChecked {
if r.rootIndex != nil && r.rootIndex.resolver != nil {
resolvingErrors := r.rootIndex.resolver.CheckForCircularReferences()
for e := range resolvingErrors {
r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
}
if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
}
if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
}
}
r.circChecked = true
}
}

140
index/rolodex_remote_loader.go
View File

@@ -17,7 +17,6 @@ import (
"net/url"
"os"
"path/filepath"
"sync"
"time"
)
@@ -30,9 +29,6 @@ type RemoteFS struct {
ProcessingFiles syncmap.Map
FetchTime int64
FetchChannel chan *RemoteFile
remoteWg sync.WaitGroup
remoteRunning bool
remoteErrorLock sync.Mutex
remoteErrors []error
logger *slog.Logger
defaultClient *http.Client
@@ -243,56 +239,56 @@ func (i *RemoteFS) GetErrors() []error {
return i.remoteErrors
}
func (i *RemoteFS) seekRelatives(file *RemoteFile) {
extractedRefs := ExtractRefs(string(file.data))
if len(extractedRefs) == 0 {
return
}
fetchChild := func(url string) {
_, err := i.Open(url)
if err != nil {
file.seekingErrors = append(file.seekingErrors, err)
i.remoteErrorLock.Lock()
i.remoteErrors = append(i.remoteErrors, err)
i.remoteErrorLock.Unlock()
}
defer i.remoteWg.Done()
}
for _, ref := range extractedRefs {
refType := ExtractRefType(ref[1])
switch refType {
case File:
fileLocation, _ := ExtractRefValues(ref[1])
//parentDir, _ := filepath.Abs(filepath.Dir(file.fullPath))
var fullPath string
if filepath.IsAbs(fileLocation) {
fullPath = fileLocation
} else {
fullPath, _ = filepath.Abs(filepath.Join(filepath.Dir(file.fullPath), fileLocation))
}
if f, ok := i.Files.Load(fullPath); ok {
i.logger.Debug("file already loaded, skipping", "file", f.(*RemoteFile).fullPath)
continue
} else {
i.remoteWg.Add(1)
go fetchChild(fullPath)
}
case HTTP:
fmt.Printf("Found relative HTTP reference: %s\n", ref[1])
}
}
if !i.remoteRunning {
i.remoteRunning = true
i.remoteWg.Wait()
i.remoteRunning = false
}
}
//func (i *RemoteFS) seekRelatives(file *RemoteFile) {
//
// extractedRefs := ExtractRefs(string(file.data))
// if len(extractedRefs) == 0 {
// return
// }
//
// fetchChild := func(url string) {
// _, err := i.Open(url)
// if err != nil {
// file.seekingErrors = append(file.seekingErrors, err)
// i.remoteErrorLock.Lock()
// i.remoteErrors = append(i.remoteErrors, err)
// i.remoteErrorLock.Unlock()
// }
// defer i.remoteWg.Done()
// }
//
// for _, ref := range extractedRefs {
// refType := ExtractRefType(ref[1])
// switch refType {
// case File:
// fileLocation, _ := ExtractRefValues(ref[1])
// //parentDir, _ := filepath.Abs(filepath.Dir(file.fullPath))
// var fullPath string
// if filepath.IsAbs(fileLocation) {
// fullPath = fileLocation
// } else {
// fullPath, _ = filepath.Abs(filepath.Join(filepath.Dir(file.fullPath), fileLocation))
// }
//
// if f, ok := i.Files.Load(fullPath); ok {
// i.logger.Debug("file already loaded, skipping", "file", f.(*RemoteFile).fullPath)
// continue
// } else {
// i.remoteWg.Add(1)
// go fetchChild(fullPath)
// }
//
// case HTTP:
// fmt.Printf("Found relative HTTP reference: %s\n", ref[1])
// }
// }
// if !i.remoteRunning {
// i.remoteRunning = true
// i.remoteWg.Wait()
// i.remoteRunning = false
// }
//
//}
func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
@@ -412,14 +408,6 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
lastModified: lastModifiedTime,
}
if i == nil {
panic("we fucked")
}
if i.indexConfig == nil {
panic("we fucked bro")
}
copiedCfg := *i.indexConfig
newBase := fmt.Sprintf("%s://%s%s", remoteParsedURLOriginal.Scheme, remoteParsedURLOriginal.Host,
@@ -428,27 +416,31 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
copiedCfg.BaseURL = newBaseURL
copiedCfg.SpecAbsolutePath = remoteParsedURL.String()
idx, _ := remoteFile.Index(&copiedCfg)
// for each index, we need a resolver
resolver := NewResolver(idx)
idx.resolver = resolver
idx, idxError := remoteFile.Index(&copiedCfg)
i.Files.Store(absolutePath, remoteFile)
if len(remoteFile.data) > 0 {
i.logger.Debug("successfully loaded file", "file", absolutePath)
}
i.seekRelatives(remoteFile)
//i.seekRelatives(remoteFile)
idx.BuildIndex()
if idxError != nil && idx == nil {
i.remoteErrors = append(i.remoteErrors, idxError)
} else {
// for each index, we need a resolver
resolver := NewResolver(idx)
idx.resolver = resolver
idx.BuildIndex()
}
// remove from processing
i.ProcessingFiles.Delete(remoteParsedURL.Path)
if !i.remoteRunning {
return remoteFile, errors.Join(i.remoteErrors...)
} else {
return remoteFile, nil
}
//if !i.remoteRunning {
return remoteFile, errors.Join(i.remoteErrors...)
// } else {
// return remoteFile, nil/
// }
}

145
index/rolodex_remote_loader_test.go
View File

@@ -96,100 +96,65 @@ func TestNewRemoteFS_BasicCheck(t *testing.T) {
bytes, rErr := io.ReadAll(file)
assert.NoError(t, rErr)
assert.Equal(t, "\"$ref\": \"\"./deeper/file2.yaml#/components/schemas/Pet\"", string(bytes))
stat, _ := file.Stat()
assert.Equal(t, "file1.yaml", stat.Name())
assert.Equal(t, int64(54), stat.Size())
assert.Equal(t, "/file1.yaml", stat.Name())
assert.Equal(t, int64(53), stat.Size())
assert.Len(t, bytes, 53)
lastMod := stat.ModTime()
assert.Equal(t, "2015-10-21 07:28:00 +0000 GMT", lastMod.String())
}
func TestNewRemoteFS_BasicCheck_Relative(t *testing.T) {
server := test_buildServer()
defer server.Close()
remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
remoteFS.RemoteHandlerFunc = test_httpClient.Get
file, err := remoteFS.Open("/deeper/file2.yaml")
assert.NoError(t, err)
bytes, rErr := io.ReadAll(file)
assert.NoError(t, rErr)
assert.Equal(t, "\"$ref\": \"./deeper/even_deeper/file3.yaml#/components/schemas/Pet\"", string(bytes))
stat, _ := file.Stat()
assert.Equal(t, "/deeper/file2.yaml", stat.Name())
assert.Equal(t, int64(65), stat.Size())
lastMod := stat.ModTime()
assert.Equal(t, "2015-10-21 08:28:00 +0000 GMT", lastMod.String())
}
func TestNewRemoteFS_BasicCheck_Relative_Deeper(t *testing.T) {
server := test_buildServer()
defer server.Close()
remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
remoteFS.RemoteHandlerFunc = test_httpClient.Get
file, err := remoteFS.Open("/deeper/even_deeper/file3.yaml")
assert.NoError(t, err)
bytes, rErr := io.ReadAll(file)
assert.NoError(t, rErr)
assert.Equal(t, "\"$ref\": \"../file2.yaml#/components/schemas/Pet\"", string(bytes))
stat, _ := file.Stat()
assert.Equal(t, "/deeper/even_deeper/file3.yaml", stat.Name())
assert.Equal(t, int64(47), stat.Size())
lastMod := stat.ModTime()
assert.Equal(t, "2015-10-21 10:28:00 +0000 GMT", lastMod.String())
}
func TestNewRemoteFS_BasicCheck_SeekRelatives(t *testing.T) {
server := test_buildServer()
defer server.Close()
remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
remoteFS.RemoteHandlerFunc = test_httpClient.Get
file, err := remoteFS.Open("/bag/list.yaml")
assert.Error(t, err)
bytes, rErr := io.ReadAll(file)
assert.NoError(t, rErr)
assert.Equal(t, "\"$ref\": \"pocket/list.yaml\"\\n\\n\"$ref\": \"zip/things.yaml\"", string(bytes))
stat, _ := file.Stat()
assert.Equal(t, "/bag/list.yaml", stat.Name())
assert.Equal(t, int64(55), stat.Size())
lastMod := stat.ModTime()
assert.Equal(t, "2015-10-21 12:28:00 +0000 GMT", lastMod.String())
files := remoteFS.GetFiles()
assert.Len(t, remoteFS.remoteErrors, 1)
assert.Len(t, files, 10)
// check correct files are in the cache
assert.Equal(t, "/bag/list.yaml", files["/bag/list.yaml"].GetFullPath())
assert.Equal(t, "list.yaml", files["/bag/list.yaml"].Name())
}
//
//func TestNewRemoteFS_BasicCheck_Relative(t *testing.T) {
//
// server := test_buildServer()
// defer server.Close()
//
// remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
// remoteFS.RemoteHandlerFunc = test_httpClient.Get
//
// file, err := remoteFS.Open("/deeper/file2.yaml")
//
// assert.NoError(t, err)
//
// bytes, rErr := io.ReadAll(file)
// assert.NoError(t, rErr)
//
// assert.Len(t, bytes, 64)
//
// stat, _ := file.Stat()
//
// assert.Equal(t, "/deeper/file2.yaml", stat.Name())
// assert.Equal(t, int64(64), stat.Size())
//
// lastMod := stat.ModTime()
// assert.Equal(t, "2015-10-21 08:28:00 +0000 GMT", lastMod.String())
//}
//
//func TestNewRemoteFS_BasicCheck_Relative_Deeper(t *testing.T) {
//
// server := test_buildServer()
// defer server.Close()
//
// remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
// remoteFS.RemoteHandlerFunc = test_httpClient.Get
//
// file, err := remoteFS.Open("/deeper/even_deeper/file3.yaml")
//
// assert.NoError(t, err)
//
// bytes, rErr := io.ReadAll(file)
// assert.NoError(t, rErr)
//
// assert.Len(t, bytes, 47)
//
// stat, _ := file.Stat()
//
// assert.Equal(t, "/deeper/even_deeper/file3.yaml", stat.Name())
// assert.Equal(t, int64(47), stat.Size())
//
// lastMod := stat.ModTime()
// assert.Equal(t, "2015-10-21 10:28:00 +0000 GMT", lastMod.String())
//}

18
index/search_index.go
View File

@@ -11,8 +11,12 @@ import (
func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *Reference {
ref := fullRef.FullDefinition
//if v, ok := index.cache.Load(fullRef); ok {
// return v.(*Reference)
//}
ref := fullRef.FullDefinition
refAlt := ref
absPath := index.specAbsolutePath
if absPath == "" {
absPath = index.config.BasePath
@@ -42,7 +46,9 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
roloLookup = ""
}
ref = fmt.Sprintf("%s#/%s", absPath, uri[1])
//ref = fmt.Sprintf("%s#/%s", absPath, uri[1]) this seems wrong
ref = fmt.Sprintf("#/%s", uri[1])
refAlt = fmt.Sprintf("%s#/%s", absPath, uri[1])
}
@@ -64,6 +70,12 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
}
if r, ok := index.allMappedRefs[ref]; ok {
index.cache.Store(ref, r)
return r
}
if r, ok := index.allMappedRefs[refAlt]; ok {
index.cache.Store(refAlt, r)
return r
}
@@ -94,6 +106,7 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
d = append(d, idx.allInlineSchemaObjectDefinitions...)
for _, s := range d {
if s.Definition == ref {
index.cache.Store(ref, s)
return s
}
}
@@ -103,6 +116,7 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
if node != nil {
found := idx.FindComponent(ref, node)
if found != nil {
index.cache.Store(ref, found)
return found
}
}

2
index/spec_index_test.go
View File

@@ -103,7 +103,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
cf.AllowRemoteLookup = true
cf.AvoidCircularReferenceCheck = true
cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
Level: slog.LevelError,
Level: slog.LevelDebug,
}))
// setting this baseURL will override the base