LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-09 04:20:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tuned some glitches with v3 doc creation.

Browse Source 
all covered now

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-10-31 15:31:19 -04:00
parent b37b9a2fb9
commit 8f3f568e5f
7 changed files with 334 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
datamodel/document_config.go
View File

@@ -4,9 +4,9 @@
package datamodel package datamodel
import ( import (
"github.com/pb33f/libopenapi/utils"
"io/fs" "io/fs"
"log/slog" "log/slog"
"net/http"
"net/url" "net/url"
"os" "os"
) )
@@ -28,7 +28,7 @@ type DocumentConfiguration struct {
// will not be used, as there will be nothing to use it against. // will not be used, as there will be nothing to use it against.
// //
// Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132 // Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132
RemoteURLHandler func(url string) (*http.Response, error) RemoteURLHandler utils.RemoteURLHandler
// If resolving locally, the BasePath will be the root from which relative references will be resolved from. // If resolving locally, the BasePath will be the root from which relative references will be resolved from.
// It's usually the location of the root specification. // It's usually the location of the root specification.
@@ -103,13 +103,3 @@ func NewDocumentConfiguration() *DocumentConfiguration {
})), })),
} }
} }
// Deprecated: use NewDocumentConfiguration instead.
func NewOpenDocumentConfiguration() *DocumentConfiguration {
return NewDocumentConfiguration()
}
// Deprecated: use NewDocumentConfiguration instead.
func NewClosedDocumentConfiguration() *DocumentConfiguration {
return NewDocumentConfiguration()
}

38
datamodel/low/v3/create_document.go
View File

@@ -51,12 +51,8 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
// If basePath is provided, add a local filesystem to the rolodex. // If basePath is provided, add a local filesystem to the rolodex.
if idxConfig.BasePath != "" { if idxConfig.BasePath != "" {
var absError error
var cwd string var cwd string
cwd, absError = filepath.Abs(config.BasePath) cwd, _ = filepath.Abs(config.BasePath)
if absError != nil {
return nil, absError
}
// if a supplied local filesystem is provided, add it to the rolodex. // if a supplied local filesystem is provided, add it to the rolodex.
if config.LocalFS != nil { if config.LocalFS != nil {
rolodex.AddLocalFS(cwd, config.LocalFS) rolodex.AddLocalFS(cwd, config.LocalFS)
@@ -68,42 +64,30 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
DirFS: os.DirFS(cwd), DirFS: os.DirFS(cwd),
FileFilters: config.FileFilter, FileFilters: config.FileFilter,
} }
fileFS, err := index.NewLocalFSWithConfig(&localFSConf) fileFS, err := index.NewLocalFSWithConfig(&localFSConf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
idxConfig.AllowFileLookup = true idxConfig.AllowFileLookup = true
// add the filesystem to the rolodex // add the filesystem to the rolodex
rolodex.AddLocalFS(cwd, fileFS) rolodex.AddLocalFS(cwd, fileFS)
} }
} }
// if base url is provided, add a remote filesystem to the rolodex. // if base url is provided, add a remote filesystem to the rolodex.
if idxConfig.BaseURL != nil { if idxConfig.BaseURL != nil {
// if a supplied remote filesystem is provided, add it to the rolodex. // create a remote filesystem
if config.RemoteFS != nil { remoteFS, _ := index.NewRemoteFSWithConfig(idxConfig)
if config.BaseURL == nil { if config.RemoteURLHandler != nil {
return nil, errors.New("cannot use remote filesystem without a BaseURL") remoteFS.RemoteHandlerFunc = config.RemoteURLHandler
}
rolodex.AddRemoteFS(config.BaseURL.String(), config.RemoteFS)
} else {
// create a remote filesystem
remoteFS, fsErr := index.NewRemoteFSWithConfig(idxConfig)
if fsErr != nil {
return nil, fsErr
}
if config.RemoteURLHandler != nil {
remoteFS.RemoteHandlerFunc = config.RemoteURLHandler
}
idxConfig.AllowRemoteLookup = true
// add to the rolodex
rolodex.AddRemoteFS(config.BaseURL.String(), remoteFS)
} }
idxConfig.AllowRemoteLookup = true
// add to the rolodex
rolodex.AddRemoteFS(config.BaseURL.String(), remoteFS)
} }
// index the rolodex // index the rolodex

191
datamodel/low/v3/create_document_test.go
View File

@@ -2,7 +2,10 @@ package v3
import ( import (
"fmt" "fmt"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils" "github.com/pb33f/libopenapi/utils"
"net/http"
"net/url"
"os" "os"
"testing" "testing"
@@ -20,7 +23,7 @@ func initTest() {
info, _ := datamodel.ExtractSpecInfo(data) info, _ := datamodel.ExtractSpecInfo(data)
var err error var err error
// deprecated function test. // deprecated function test.
doc, err = CreateDocument(info) doc, err = CreateDocumentFromConfig(info, datamodel.NewDocumentConfiguration())
if err != nil { if err != nil {
panic("broken something") panic("broken something")
} }
@@ -56,6 +59,123 @@ func TestCircularReferenceError(t *testing.T) {
assert.Len(t, utils.UnwrapErrors(err), 3) assert.Len(t, utils.UnwrapErrors(err), 3)
} }
func TestRolodexLocalFileSystem(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
cf := datamodel.NewDocumentConfiguration()
cf.BasePath = "../../../test_specs"
cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.NoError(t, err)
}
func TestRolodexLocalFileSystem_ProvideNonRolodexFS(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
baseDir := "../../../test_specs"
cf := datamodel.NewDocumentConfiguration()
cf.BasePath = baseDir
cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
cf.LocalFS = os.DirFS(baseDir)
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.Error(t, err)
}
func TestRolodexLocalFileSystem_ProvideRolodexFS(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
baseDir := "../../../test_specs"
cf := datamodel.NewDocumentConfiguration()
cf.BasePath = baseDir
cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
localFS, lErr := index.NewLocalFSWithConfig(&index.LocalFSConfig{
BaseDirectory: baseDir,
DirFS: os.DirFS(baseDir),
FileFilters: cf.FileFilter,
})
cf.LocalFS = localFS
assert.NoError(t, lErr)
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.NoError(t, err)
}
func TestRolodexLocalFileSystem_BadPath(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
cf := datamodel.NewDocumentConfiguration()
cf.BasePath = "/NOWHERE"
cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.Nil(t, lDoc)
assert.Error(t, err)
}
func TestRolodexRemoteFileSystem(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
cf := datamodel.NewDocumentConfiguration()
baseUrl := "https://raw.githubusercontent.com/pb33f/libopenapi/main/test_specs"
u, _ := url.Parse(baseUrl)
cf.BaseURL = u
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.NoError(t, err)
}
func TestRolodexRemoteFileSystem_BadBase(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
cf := datamodel.NewDocumentConfiguration()
baseUrl := "https://no-no-this-will-not-work-it-just-will-not-get-the-job-done-mate.com"
u, _ := url.Parse(baseUrl)
cf.BaseURL = u
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.Error(t, err)
}
func TestRolodexRemoteFileSystem_CustomRemote_NoBaseURL(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
cf := datamodel.NewDocumentConfiguration()
cf.RemoteFS, _ = index.NewRemoteFSWithConfig(&index.SpecIndexConfig{})
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.Error(t, err)
}
func TestRolodexRemoteFileSystem_CustomHttpHandler(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/first.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
cf := datamodel.NewDocumentConfiguration()
cf.RemoteURLHandler = http.Get
baseUrl := "https://no-no-this-will-not-work-it-just-will-not-get-the-job-done-mate.com"
u, _ := url.Parse(baseUrl)
cf.BaseURL = u
pizza := func(url string) (resp *http.Response, err error) {
return nil, nil
}
cf.RemoteURLHandler = pizza
lDoc, err := CreateDocumentFromConfig(info, cf)
assert.NotNil(t, lDoc)
assert.Error(t, err)
}
func TestCircularReference_IgnoreArray(t *testing.T) { func TestCircularReference_IgnoreArray(t *testing.T) {
spec := `openapi: 3.1.0 spec := `openapi: 3.1.0
components: components:
@@ -76,8 +196,6 @@ components:
info, _ := datamodel.ExtractSpecInfo([]byte(spec)) info, _ := datamodel.ExtractSpecInfo([]byte(spec))
circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
AllowFileReferences: false,
AllowRemoteReferences: false,
IgnoreArrayCircularReferences: true, IgnoreArrayCircularReferences: true,
}) })
assert.NotNil(t, circDoc) assert.NotNil(t, circDoc)
@@ -104,8 +222,6 @@ components:
info, _ := datamodel.ExtractSpecInfo([]byte(spec)) info, _ := datamodel.ExtractSpecInfo([]byte(spec))
circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
AllowFileReferences: false,
AllowRemoteReferences: false,
IgnorePolymorphicCircularReferences: true, IgnorePolymorphicCircularReferences: true,
}) })
assert.NotNil(t, circDoc) assert.NotNil(t, circDoc)
@@ -117,10 +233,7 @@ func BenchmarkCreateDocument_Stripe(b *testing.B) {
info, _ := datamodel.ExtractSpecInfo(data) info, _ := datamodel.ExtractSpecInfo(data)
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
if err != nil { if err != nil {
panic("this should not error") panic("this should not error")
} }
@@ -131,10 +244,7 @@ func BenchmarkCreateDocument_Petstore(b *testing.B) {
data, _ := os.ReadFile("../../../test_specs/petstorev3.json") data, _ := os.ReadFile("../../../test_specs/petstorev3.json")
info, _ := datamodel.ExtractSpecInfo(data) info, _ := datamodel.ExtractSpecInfo(data)
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
_, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
if err != nil { if err != nil {
panic("this should not error") panic("this should not error")
} }
@@ -144,10 +254,7 @@ func BenchmarkCreateDocument_Petstore(b *testing.B) {
func TestCreateDocumentStripe(t *testing.T) { func TestCreateDocumentStripe(t *testing.T) {
data, _ := os.ReadFile("../../../test_specs/stripe.yaml") data, _ := os.ReadFile("../../../test_specs/stripe.yaml")
info, _ := datamodel.ExtractSpecInfo(data) info, _ := datamodel.ExtractSpecInfo(data)
d, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ d, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Len(t, utils.UnwrapErrors(err), 3) assert.Len(t, utils.UnwrapErrors(err), 3)
assert.Equal(t, "3.0.0", d.Version.Value) assert.Equal(t, "3.0.0", d.Version.Value)
@@ -211,10 +318,7 @@ webhooks:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Len(t, utils.UnwrapErrors(err), 1) assert.Len(t, utils.UnwrapErrors(err), 1)
} }
@@ -590,10 +694,7 @@ components:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.NoError(t, err) assert.NoError(t, err)
ob := doc.Components.Value.FindSchema("bork").Value ob := doc.Components.Value.FindSchema("bork").Value
@@ -609,10 +710,7 @@ webhooks:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Equal(t, "flat map build failed: reference cannot be found: reference at line 4, column 5 is empty, it cannot be resolved", assert.Equal(t, "flat map build failed: reference cannot be found: reference at line 4, column 5 is empty, it cannot be resolved",
err.Error()) err.Error())
} }
@@ -626,10 +724,7 @@ components:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Equal(t, "reference at line 5, column 7 is empty, it cannot be resolved", err.Error()) assert.Equal(t, "reference at line 5, column 7 is empty, it cannot be resolved", err.Error())
} }
@@ -641,10 +736,7 @@ paths:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Equal(t, assert.Equal(t,
"path item build failed: cannot find reference: at line 4, col 10", err.Error()) "path item build failed: cannot find reference: at line 4, col 10", err.Error())
} }
@@ -656,10 +748,7 @@ tags:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Equal(t, assert.Equal(t,
"object extraction failed: reference at line 3, column 5 is empty, it cannot be resolved", err.Error()) "object extraction failed: reference at line 3, column 5 is empty, it cannot be resolved", err.Error())
} }
@@ -671,10 +760,7 @@ security:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Equal(t, assert.Equal(t,
"array build failed: reference cannot be found: reference at line 3, column 3 is empty, it cannot be resolved", "array build failed: reference cannot be found: reference at line 3, column 3 is empty, it cannot be resolved",
err.Error()) err.Error())
@@ -687,10 +773,7 @@ externalDocs:
info, _ := datamodel.ExtractSpecInfo([]byte(yml)) info, _ := datamodel.ExtractSpecInfo([]byte(yml))
var err error var err error
_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
assert.Equal(t, "object extraction failed: reference at line 3, column 3 is empty, it cannot be resolved", err.Error()) assert.Equal(t, "object extraction failed: reference at line 3, column 3 is empty, it cannot be resolved", err.Error())
} }
@@ -702,10 +785,7 @@ func TestCreateDocument_YamlAnchor(t *testing.T) {
info, _ := datamodel.ExtractSpecInfo(anchorDocument) info, _ := datamodel.ExtractSpecInfo(anchorDocument)
// build low-level document model // build low-level document model
document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
if err != nil { if err != nil {
fmt.Printf("error: %s\n", err.Error()) fmt.Printf("error: %s\n", err.Error())
@@ -755,10 +835,7 @@ func ExampleCreateDocument() {
info, _ := datamodel.ExtractSpecInfo(petstoreBytes) info, _ := datamodel.ExtractSpecInfo(petstoreBytes)
// build low-level document model // build low-level document model
document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
AllowFileReferences: false,
AllowRemoteReferences: false,
})
if err != nil { if err != nil {
fmt.Printf("error: %s\n", err.Error()) fmt.Printf("error: %s\n", err.Error())

2
index/extract_refs.go
View File

@@ -283,7 +283,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
if index.config.BaseURL != nil { if index.config.BaseURL != nil {
u := *index.config.BaseURL u := *index.config.BaseURL
abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0])) abs := filepath.Join(u.Path, uri[0])
u.Path = abs u.Path = abs
fullDefinitionPath = u.String() fullDefinitionPath = u.String()
componentName = uri[0] componentName = uri[0]

354
index/rolodex_file_loader.go
View File

@@ -4,277 +4,277 @@
package index package index
import ( import (
"fmt" "fmt"
"github.com/pb33f/libopenapi/datamodel" "github.com/pb33f/libopenapi/datamodel"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"io" "io"
"io/fs" "io/fs"
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
) )
type LocalFS struct { type LocalFS struct {
indexConfig *SpecIndexConfig indexConfig *SpecIndexConfig
entryPointDirectory string entryPointDirectory string
baseDirectory string baseDirectory string
Files map[string]RolodexFile Files map[string]RolodexFile
logger *slog.Logger logger *slog.Logger
readingErrors []error readingErrors []error
} }
func (l *LocalFS) GetFiles() map[string]RolodexFile { func (l *LocalFS) GetFiles() map[string]RolodexFile {
return l.Files return l.Files
} }
func (l *LocalFS) GetErrors() []error { func (l *LocalFS) GetErrors() []error {
return l.readingErrors return l.readingErrors
} }
func (l *LocalFS) Open(name string) (fs.File, error) { func (l *LocalFS) Open(name string) (fs.File, error) {
if l.indexConfig != nil && !l.indexConfig.AllowFileLookup { if l.indexConfig != nil && !l.indexConfig.AllowFileLookup {
return nil, &fs.PathError{Op: "open", Path: name, return nil, &fs.PathError{Op: "open", Path: name,
Err: fmt.Errorf("file lookup for '%s' not allowed, set the index configuration "+ Err: fmt.Errorf("file lookup for '%s' not allowed, set the index configuration "+
"to AllowFileLookup to be true", name)} "to AllowFileLookup to be true", name)}
} }
if !filepath.IsAbs(name) { if !filepath.IsAbs(name) {
name, _ = filepath.Abs(filepath.Join(l.baseDirectory, name)) name, _ = filepath.Abs(filepath.Join(l.baseDirectory, name))
} }
if f, ok := l.Files[name]; ok { if f, ok := l.Files[name]; ok {
return f.(*LocalFile), nil return f.(*LocalFile), nil
} else { } else {
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
} }
} }
type LocalFile struct { type LocalFile struct {
filename string filename string
name string name string
extension FileExtension extension FileExtension
data []byte data []byte
fullPath string fullPath string
lastModified time.Time lastModified time.Time
readingErrors []error readingErrors []error
index *SpecIndex index *SpecIndex
parsed *yaml.Node parsed *yaml.Node
offset int64 offset int64
} }
func (l *LocalFile) GetIndex() *SpecIndex { func (l *LocalFile) GetIndex() *SpecIndex {
return l.index return l.index
} }
func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
if l.index != nil { if l.index != nil {
return l.index, nil return l.index, nil
} }
content := l.data content := l.data
// first, we must parse the content of the file // first, we must parse the content of the file
info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, true) info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
index := NewSpecIndexWithConfig(info.RootNode, config) index := NewSpecIndexWithConfig(info.RootNode, config)
index.specAbsolutePath = l.fullPath index.specAbsolutePath = l.fullPath
l.index = index l.index = index
return index, nil return index, nil
} }
func (l *LocalFile) GetContent() string { func (l *LocalFile) GetContent() string {
return string(l.data) return string(l.data)
} }
func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) { func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) {
if l.parsed != nil { if l.parsed != nil {
return l.parsed, nil return l.parsed, nil
} }
if l.index != nil && l.index.root != nil { if l.index != nil && l.index.root != nil {
return l.index.root, nil return l.index.root, nil
} }
if l.data == nil { if l.data == nil {
return nil, fmt.Errorf("no data to parse for file: %s", l.fullPath) return nil, fmt.Errorf("no data to parse for file: %s", l.fullPath)
} }
var root yaml.Node var root yaml.Node
err := yaml.Unmarshal(l.data, &root) err := yaml.Unmarshal(l.data, &root)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if l.index != nil && l.index.root == nil { if l.index != nil && l.index.root == nil {
l.index.root = &root l.index.root = &root
} }
l.parsed = &root l.parsed = &root
return &root, nil return &root, nil
} }
func (l *LocalFile) GetFileExtension() FileExtension { func (l *LocalFile) GetFileExtension() FileExtension {
return l.extension return l.extension
} }
func (l *LocalFile) GetFullPath() string { func (l *LocalFile) GetFullPath() string {
return l.fullPath return l.fullPath
} }
func (l *LocalFile) GetErrors() []error { func (l *LocalFile) GetErrors() []error {
return l.readingErrors return l.readingErrors
} }
type LocalFSConfig struct { type LocalFSConfig struct {
// the base directory to index // the base directory to index
BaseDirectory string BaseDirectory string
Logger *slog.Logger Logger *slog.Logger
FileFilters []string FileFilters []string
DirFS fs.FS DirFS fs.FS
} }
func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) {
localFiles := make(map[string]RolodexFile) localFiles := make(map[string]RolodexFile)
var allErrors []error var allErrors []error
log := config.Logger log := config.Logger
if log == nil { if log == nil {
log = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ log = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
Level: slog.LevelError, Level: slog.LevelError,
})) }))
} }
// if the basedir is an absolute file, we're just going to index that file. // if the basedir is an absolute file, we're just going to index that file.
ext := filepath.Ext(config.BaseDirectory) ext := filepath.Ext(config.BaseDirectory)
file := filepath.Base(config.BaseDirectory) file := filepath.Base(config.BaseDirectory)
var absBaseDir string var absBaseDir string
absBaseDir, _ = filepath.Abs(config.BaseDirectory) absBaseDir, _ = filepath.Abs(config.BaseDirectory)
walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error { walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
// we don't care about directories, or errors, just read everything we can. // we don't care about directories, or errors, just read everything we can.
if d.IsDir() { if d.IsDir() {
return nil return nil
} }
if len(ext) > 2 && p != file { if len(ext) > 2 && p != file {
return nil return nil
} }
if strings.HasPrefix(p, ".") { if strings.HasPrefix(p, ".") {
return nil return nil
} }
if len(config.FileFilters) > 0 { if len(config.FileFilters) > 0 {
if !slices.Contains(config.FileFilters, p) { if !slices.Contains(config.FileFilters, p) {
return nil return nil
} }
} }
extension := ExtractFileType(p) extension := ExtractFileType(p)
var readingErrors []error var readingErrors []error
abs, _ := filepath.Abs(filepath.Join(config.BaseDirectory, p)) abs, _ := filepath.Abs(filepath.Join(config.BaseDirectory, p))
var fileData []byte var fileData []byte
switch extension { switch extension {
case YAML, JSON: case YAML, JSON:
dirFile, _ := config.DirFS.Open(p) dirFile, _ := config.DirFS.Open(p)
modTime := time.Now() modTime := time.Now()
stat, _ := dirFile.Stat() stat, _ := dirFile.Stat()
if stat != nil { if stat != nil {
modTime = stat.ModTime() modTime = stat.ModTime()
} }
fileData, _ = io.ReadAll(dirFile) fileData, _ = io.ReadAll(dirFile)
log.Debug("collecting JSON/YAML file", "file", abs) log.Debug("collecting JSON/YAML file", "file", abs)
localFiles[abs] = &LocalFile{ localFiles[abs] = &LocalFile{
filename: p, filename: p,
name: filepath.Base(p), name: filepath.Base(p),
extension: ExtractFileType(p), extension: ExtractFileType(p),
data: fileData, data: fileData,
fullPath: abs, fullPath: abs,
lastModified: modTime, lastModified: modTime,
readingErrors: readingErrors, readingErrors: readingErrors,
} }
case UNSUPPORTED: case UNSUPPORTED:
log.Debug("skipping non JSON/YAML file", "file", abs) log.Debug("skipping non JSON/YAML file", "file", abs)
} }
return nil return nil
}) })
if walkErr != nil { if walkErr != nil {
return nil, walkErr return nil, walkErr
} }
return &LocalFS{ return &LocalFS{
Files: localFiles, Files: localFiles,
logger: log, logger: log,
baseDirectory: absBaseDir, baseDirectory: absBaseDir,
entryPointDirectory: config.BaseDirectory, entryPointDirectory: config.BaseDirectory,
readingErrors: allErrors, readingErrors: allErrors,
}, nil }, nil
} }
func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) {
config := &LocalFSConfig{ config := &LocalFSConfig{
BaseDirectory: baseDir, BaseDirectory: baseDir,
DirFS: dirFS, DirFS: dirFS,
} }
return NewLocalFSWithConfig(config) return NewLocalFSWithConfig(config)
} }
func (l *LocalFile) FullPath() string { func (l *LocalFile) FullPath() string {
return l.fullPath return l.fullPath
} }
func (l *LocalFile) Name() string { func (l *LocalFile) Name() string {
return l.name return l.name
} }
func (l *LocalFile) Size() int64 { func (l *LocalFile) Size() int64 {
return int64(len(l.data)) return int64(len(l.data))
} }
func (l *LocalFile) Mode() fs.FileMode { func (l *LocalFile) Mode() fs.FileMode {
return fs.FileMode(0) return fs.FileMode(0)
} }
func (l *LocalFile) ModTime() time.Time { func (l *LocalFile) ModTime() time.Time {
return l.lastModified return l.lastModified
} }
func (l *LocalFile) IsDir() bool { func (l *LocalFile) IsDir() bool {
return false return false
} }
func (l *LocalFile) Sys() interface{} { func (l *LocalFile) Sys() interface{} {
return nil return nil
} }
func (l *LocalFile) Close() error { func (l *LocalFile) Close() error {
return nil return nil
} }
func (l *LocalFile) Stat() (fs.FileInfo, error) { func (l *LocalFile) Stat() (fs.FileInfo, error) {
return l, nil return l, nil
} }
func (l *LocalFile) Read(b []byte) (int, error) { func (l *LocalFile) Read(b []byte) (int, error) {
if l.offset >= int64(len(l.GetContent())) { if l.offset >= int64(len(l.GetContent())) {
return 0, io.EOF return 0, io.EOF
} }
if l.offset < 0 { if l.offset < 0 {
return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid} return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid}
} }
n := copy(b, l.GetContent()[l.offset:]) n := copy(b, l.GetContent()[l.offset:])
l.offset += int64(n) l.offset += int64(n)
return n, nil return n, nil
} }

10
index/rolodex_remote_loader.go
View File

@@ -7,6 +7,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"github.com/pb33f/libopenapi/datamodel" "github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/utils"
"log/slog" "log/slog"
"runtime" "runtime"
@@ -21,13 +22,11 @@ import (
"time" "time"
) )
type RemoteURLHandler = func(url string) (*http.Response, error)
type RemoteFS struct { type RemoteFS struct {
indexConfig *SpecIndexConfig indexConfig *SpecIndexConfig
rootURL string rootURL string
rootURLParsed *url.URL rootURLParsed *url.URL
RemoteHandlerFunc RemoteURLHandler RemoteHandlerFunc utils.RemoteURLHandler
Files syncmap.Map Files syncmap.Map
ProcessingFiles syncmap.Map ProcessingFiles syncmap.Map
FetchTime int64 FetchTime int64
@@ -176,6 +175,9 @@ const (
) )
func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) { func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) {
if specIndexConfig == nil {
return nil, errors.New("no spec index config provided")
}
remoteRootURL := specIndexConfig.BaseURL remoteRootURL := specIndexConfig.BaseURL
log := specIndexConfig.Logger log := specIndexConfig.Logger
if log == nil { if log == nil {
@@ -217,7 +219,7 @@ func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) {
return NewRemoteFSWithConfig(config) return NewRemoteFSWithConfig(config)
} }
func (i *RemoteFS) SetRemoteHandlerFunc(handlerFunc RemoteURLHandler) { func (i *RemoteFS) SetRemoteHandlerFunc(handlerFunc utils.RemoteURLHandler) {
i.RemoteHandlerFunc = handlerFunc i.RemoteHandlerFunc = handlerFunc
} }

3
utils/utils.go
View File

@@ -3,6 +3,7 @@ package utils
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/http"
"net/url" "net/url"
"regexp" "regexp"
"sort" "sort"
@@ -749,3 +750,5 @@ func CheckForMergeNodes(node *yaml.Node) {
} }
} }
} }
type RemoteURLHandler = func(url string) (*http.Response, error)