LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-06 04:20:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

More cleaning and added docs.

Browse Source 
We’re ready for review

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-11-01 16:14:11 -04:00
parent ce4a60baa8
commit 80b2b2d0b5
10 changed files with 305 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
index/circular_reference_result.go
View File

@@ -14,6 +14,7 @@ type CircularReferenceResult struct {
IsInfiniteLoop bool // if all the definitions in the reference loop are marked as required, this is an infinite circular reference, thus is not allowed.
}
// GenerateJourneyPath generates a string representation of the journey taken to find the circular reference.
func (c *CircularReferenceResult) GenerateJourneyPath() string {
buf := strings.Builder{}
for i, ref := range c.Journey {

8
index/extract_refs.go
View File

@@ -215,13 +215,10 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
if strings.HasPrefix(uri[0], "http") {
fullDefinitionPath = value
componentName = fmt.Sprintf("#/%s", uri[1])
} else {
if filepath.IsAbs(uri[0]) {
fullDefinitionPath = value
componentName = fmt.Sprintf("#/%s", uri[1])
} else {
// if the index has a base path, use that to resolve the path
@@ -235,7 +232,6 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
} else {
// if the index has a base URL, use that to resolve the path.
if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) {
u := *index.config.BaseURL
abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0]))
u.Path = abs
@@ -287,16 +283,12 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
u.Path = abs
fullDefinitionPath = u.String()
componentName = uri[0]
} else {
abs, _ := filepath.Abs(filepath.Join(defRoot, uri[0]))
fullDefinitionPath = abs
componentName = uri[0]
}
}
}
}
}

4
index/find_component.go
View File

@@ -102,7 +102,6 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
var absoluteFileLocation, fileName string
// is this a local or a remote file?
fileName = filepath.Base(file)
if filepath.IsAbs(file) || strings.HasPrefix(file, "http") {
absoluteFileLocation = file
@@ -119,14 +118,11 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
// if the absolute file location has no file ext, then get the rolodex root.
ext := filepath.Ext(absoluteFileLocation)
var parsedDocument *yaml.Node
var err error
idx := index
if ext != "" {
// extract the document from the rolodex.
rFile, rError := index.rolodex.Open(absoluteFileLocation)

1
index/index_model.go
View File

@@ -173,7 +173,6 @@ func CreateClosedAPIIndexConfig() *SpecIndexConfig {
// everything is pre-walked if you need it.
type SpecIndex struct {
specAbsolutePath string
AbsoluteFile string
rolodex *Rolodex // the rolodex is used to fetch remote and file based documents.
allRefs map[string]*Reference // all (deduplicated) refs
rawSequencedRefs []*Reference // all raw references in sequence as they are scanned, not deduped.

17
index/resolver.go
View File

@@ -208,9 +208,6 @@ func visitIndexWithoutDamagingIt(res *Resolver, idx *SpecIndex) {
res.VisitReference(schemaRef, seenReferences, journey, false)
}
}
//for _, c := range idx.GetChildren() {
// visitIndexWithoutDamagingIt(res, c)
//}
}
func visitIndex(res *Resolver, idx *SpecIndex) {
@@ -377,7 +374,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
value := node.Content[i+1].Value
var locatedRef *Reference
var fullDef string
var definition string
@@ -390,12 +386,9 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
if strings.HasPrefix(exp[0], "http") {
fullDef = value
} else {
if filepath.IsAbs(exp[0]) {
fullDef = value
} else {
if strings.HasPrefix(ref.FullDefinition, "http") {
// split the http URI into parts
@@ -420,7 +413,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
}
}
} else {
// local component, full def is based on passed in ref
if strings.HasPrefix(ref.FullDefinition, "http") {
@@ -434,13 +426,10 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1])
} else {
// split the full def into parts
fileDef := strings.Split(ref.FullDefinition, "#/")
fullDef = fmt.Sprintf("%s#/%s", fileDef[0], exp[1])
}
}
} else {
@@ -460,14 +449,12 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
// is the file def a http link?
if strings.HasPrefix(fileDef[0], "http") {
u, _ := url.Parse(fileDef[0])
path, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0]))
u.Path = path
fullDef = u.String()
} else {
fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0]))
}
}
@@ -529,17 +516,13 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
if exp[0] != "" {
if !strings.HasPrefix(exp[0], "http") {
if !filepath.IsAbs(exp[0]) {
if strings.HasPrefix(ref.FullDefinition, "http") {
u, _ := url.Parse(ref.FullDefinition)
p, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0]))
u.Path = p
u.Fragment = ""
def = fmt.Sprintf("%s#/%s", u.String(), exp[1])
} else {
fd := strings.Split(ref.FullDefinition, "#/")
abs, _ := filepath.Abs(filepath.Join(filepath.Dir(fd[0]), exp[0]))
def = fmt.Sprintf("%s#/%s", abs, exp[1])

168
index/rolodex.go
View File

@@ -6,7 +6,6 @@ package index
import (
"errors"
"fmt"
"github.com/pb33f/libopenapi/datamodel"
"gopkg.in/yaml.v3"
"io"
"io/fs"
@@ -46,6 +45,9 @@ type RolodexFS interface {
GetFiles() map[string]RolodexFile
}
// Rolodex is a file system abstraction that allows for the indexing of multiple file systems
// and the ability to resolve references across those file systems. It is used to hold references to external
// files, and the indexes they hold. The rolodex is the master lookup for all references.
type Rolodex struct {
localFS map[string]fs.FS
remoteFS map[string]fs.FS
@@ -63,148 +65,7 @@ type Rolodex struct {
ignoredCircularReferences []*CircularReferenceResult
}
type rolodexFile struct {
location string
rolodex *Rolodex
index *SpecIndex
localFile *LocalFile
remoteFile *RemoteFile
}
func (rf *rolodexFile) Name() string {
if rf.localFile != nil {
return rf.localFile.filename
}
if rf.remoteFile != nil {
return rf.remoteFile.filename
}
return ""
}
func (rf *rolodexFile) GetIndex() *SpecIndex {
if rf.localFile != nil {
return rf.localFile.GetIndex()
}
if rf.remoteFile != nil {
return rf.remoteFile.GetIndex()
}
return nil
}
func (rf *rolodexFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
if rf.index != nil {
return rf.index, nil
}
var content []byte
if rf.localFile != nil {
content = rf.localFile.data
}
if rf.remoteFile != nil {
content = rf.remoteFile.data
}
// first, we must parse the content of the file
info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, config.SkipDocumentCheck)
if err != nil {
return nil, err
}
// create a new index for this file and link it to this rolodex.
config.Rolodex = rf.rolodex
index := NewSpecIndexWithConfig(info.RootNode, config)
rf.index = index
return index, nil
}
func (rf *rolodexFile) GetContent() string {
if rf.localFile != nil {
return string(rf.localFile.data)
}
if rf.remoteFile != nil {
return string(rf.remoteFile.data)
}
return ""
}
func (rf *rolodexFile) GetContentAsYAMLNode() (*yaml.Node, error) {
if rf.localFile != nil {
return rf.localFile.GetContentAsYAMLNode()
}
if rf.remoteFile != nil {
return rf.remoteFile.GetContentAsYAMLNode()
}
return nil, nil
}
func (rf *rolodexFile) GetFileExtension() FileExtension {
if rf.localFile != nil {
return rf.localFile.extension
}
if rf.remoteFile != nil {
return rf.remoteFile.extension
}
return UNSUPPORTED
}
func (rf *rolodexFile) GetFullPath() string {
if rf.localFile != nil {
return rf.localFile.fullPath
}
if rf.remoteFile != nil {
return rf.remoteFile.fullPath
}
return ""
}
func (rf *rolodexFile) ModTime() time.Time {
if rf.localFile != nil {
return rf.localFile.lastModified
}
if rf.remoteFile != nil {
return rf.remoteFile.lastModified
}
return time.Now()
}
func (rf *rolodexFile) Size() int64 {
if rf.localFile != nil {
return rf.localFile.Size()
}
if rf.remoteFile != nil {
return rf.remoteFile.Size()
}
return 0
}
func (rf *rolodexFile) IsDir() bool {
// always false.
return false
}
func (rf *rolodexFile) Sys() interface{} {
// not implemented.
return nil
}
func (rf *rolodexFile) Mode() os.FileMode {
if rf.localFile != nil {
return rf.localFile.Mode()
}
if rf.remoteFile != nil {
return rf.remoteFile.Mode()
}
return os.FileMode(0)
}
func (rf *rolodexFile) GetErrors() []error {
if rf.localFile != nil {
return rf.localFile.readingErrors
}
if rf.remoteFile != nil {
return rf.remoteFile.seekingErrors
}
return nil
}
// NewRolodex creates a new rolodex with the provided index configuration.
func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex {
r := &Rolodex{
indexConfig: indexConfig,
@@ -215,6 +76,8 @@ func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex {
return r
}
// GetIgnoredCircularReferences returns a list of circular references that were ignored during the indexing process.
// These can be array or polymorphic references.
func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult {
debounced := make(map[string]*CircularReferenceResult)
for _, c := range r.ignoredCircularReferences {
@@ -229,35 +92,43 @@ func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult {
return debouncedResults
}
// GetIndexingDuration returns the duration it took to index the rolodex.
func (r *Rolodex) GetIndexingDuration() time.Duration {
return r.indexingDuration
}
// GetRootIndex returns the root index of the rolodex (the entry point, the main document)
func (r *Rolodex) GetRootIndex() *SpecIndex {
return r.rootIndex
}
// GetIndexes returns all the indexes in the rolodex.
func (r *Rolodex) GetIndexes() []*SpecIndex {
return r.indexes
}
// GetCaughtErrors returns all the errors that were caught during the indexing process.
func (r *Rolodex) GetCaughtErrors() []error {
return r.caughtErrors
}
// AddLocalFS adds a local file system to the rolodex.
func (r *Rolodex) AddLocalFS(baseDir string, fileSystem fs.FS) {
absBaseDir, _ := filepath.Abs(baseDir)
r.localFS[absBaseDir] = fileSystem
}
// SetRootNode sets the root node of the rolodex (the entry point, the main document)
func (r *Rolodex) SetRootNode(node *yaml.Node) {
r.rootNode = node
}
// AddRemoteFS adds a remote file system to the rolodex.
func (r *Rolodex) AddRemoteFS(baseURL string, fileSystem fs.FS) {
r.remoteFS[baseURL] = fileSystem
}
// IndexTheRolodex indexes the rolodex, building out the indexes for each file, and then building the root index.
func (r *Rolodex) IndexTheRolodex() error {
if r.indexed {
return nil
@@ -393,8 +264,6 @@ func (r *Rolodex) IndexTheRolodex() error {
}
}
// todo: variation with no base path, but a base URL.
index := NewSpecIndexWithConfig(r.rootNode, r.indexConfig)
resolver := NewResolver(index)
@@ -433,6 +302,7 @@ func (r *Rolodex) IndexTheRolodex() error {
}
// CheckForCircularReferences checks for circular references in the rolodex.
func (r *Rolodex) CheckForCircularReferences() {
if !r.circChecked {
if r.rootIndex != nil && r.rootIndex.resolver != nil {
@@ -451,6 +321,7 @@ func (r *Rolodex) CheckForCircularReferences() {
}
}
// Resolve resolves references in the rolodex.
func (r *Rolodex) Resolve() {
if r.rootIndex != nil && r.rootIndex.resolver != nil {
resolvingErrors := r.rootIndex.resolver.Resolve()
@@ -467,6 +338,7 @@ func (r *Rolodex) Resolve() {
r.resolved = true
}
// BuildIndexes builds the indexes in the rolodex, this is generally not required unless manually building a rolodex.
func (r *Rolodex) BuildIndexes() {
if r.manualBuilt {
return
@@ -480,6 +352,7 @@ func (r *Rolodex) BuildIndexes() {
r.manualBuilt = true
}
// Open opens a file in the rolodex, and returns a RolodexFile.
func (r *Rolodex) Open(location string) (RolodexFile, error) {
if r == nil {
return nil, fmt.Errorf("rolodex has not been initialized, cannot open file '%s'", location)
@@ -490,10 +363,8 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) {
}
var errorStack []error
var localFile *LocalFile
var remoteFile *RemoteFile
fileLookup := location
isUrl := false
u, _ := url.Parse(location)
@@ -502,18 +373,15 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) {
}
if !isUrl {
for k, v := range r.localFS {
// check if this is a URL or an abs/rel reference.
if !filepath.IsAbs(location) {
fileLookup, _ = filepath.Abs(filepath.Join(k, location))
}
f, err := v.Open(fileLookup)
if err != nil {
// try a lookup that is not absolute, but relative
f, err = v.Open(location)
if err != nil {

153
index/rolodex_file.go Normal file
View File

@@ -0,0 +1,153 @@
// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT
package index
import (
"github.com/pb33f/libopenapi/datamodel"
"gopkg.in/yaml.v3"
"os"
"time"
)
type rolodexFile struct {
location string
rolodex *Rolodex
index *SpecIndex
localFile *LocalFile
remoteFile *RemoteFile
}
func (rf *rolodexFile) Name() string {
if rf.localFile != nil {
return rf.localFile.filename
}
if rf.remoteFile != nil {
return rf.remoteFile.filename
}
return ""
}
func (rf *rolodexFile) GetIndex() *SpecIndex {
if rf.localFile != nil {
return rf.localFile.GetIndex()
}
if rf.remoteFile != nil {
return rf.remoteFile.GetIndex()
}
return nil
}
func (rf *rolodexFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
if rf.index != nil {
return rf.index, nil
}
var content []byte
if rf.localFile != nil {
content = rf.localFile.data
}
if rf.remoteFile != nil {
content = rf.remoteFile.data
}
// first, we must parse the content of the file
info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, config.SkipDocumentCheck)
if err != nil {
return nil, err
}
// create a new index for this file and link it to this rolodex.
config.Rolodex = rf.rolodex
index := NewSpecIndexWithConfig(info.RootNode, config)
rf.index = index
return index, nil
}
func (rf *rolodexFile) GetContent() string {
if rf.localFile != nil {
return string(rf.localFile.data)
}
if rf.remoteFile != nil {
return string(rf.remoteFile.data)
}
return ""
}
func (rf *rolodexFile) GetContentAsYAMLNode() (*yaml.Node, error) {
if rf.localFile != nil {
return rf.localFile.GetContentAsYAMLNode()
}
if rf.remoteFile != nil {
return rf.remoteFile.GetContentAsYAMLNode()
}
return nil, nil
}
func (rf *rolodexFile) GetFileExtension() FileExtension {
if rf.localFile != nil {
return rf.localFile.extension
}
if rf.remoteFile != nil {
return rf.remoteFile.extension
}
return UNSUPPORTED
}
func (rf *rolodexFile) GetFullPath() string {
if rf.localFile != nil {
return rf.localFile.fullPath
}
if rf.remoteFile != nil {
return rf.remoteFile.fullPath
}
return ""
}
func (rf *rolodexFile) ModTime() time.Time {
if rf.localFile != nil {
return rf.localFile.lastModified
}
if rf.remoteFile != nil {
return rf.remoteFile.lastModified
}
return time.Now()
}
func (rf *rolodexFile) Size() int64 {
if rf.localFile != nil {
return rf.localFile.Size()
}
if rf.remoteFile != nil {
return rf.remoteFile.Size()
}
return 0
}
func (rf *rolodexFile) IsDir() bool {
// always false.
return false
}
func (rf *rolodexFile) Sys() interface{} {
// not implemented.
return nil
}
func (rf *rolodexFile) Mode() os.FileMode {
if rf.localFile != nil {
return rf.localFile.Mode()
}
if rf.remoteFile != nil {
return rf.remoteFile.Mode()
}
return os.FileMode(0)
}
func (rf *rolodexFile) GetErrors() []error {
if rf.localFile != nil {
return rf.localFile.readingErrors
}
if rf.remoteFile != nil {
return rf.remoteFile.seekingErrors
}
return nil
}

139
index/rolodex_file_loader.go
View File

@@ -6,17 +6,18 @@ package index
import (
"fmt"
"github.com/pb33f/libopenapi/datamodel"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v3"
"io"
"io/fs"
"log/slog"
"os"
"path/filepath"
"slices"
"strings"
"time"
)
// LocalFS is a file system that indexes local files.
type LocalFS struct {
indexConfig *SpecIndexConfig
entryPointDirectory string
@@ -26,14 +27,17 @@ type LocalFS struct {
readingErrors []error
}
// GetFiles returns the files that have been indexed. A map of RolodexFile objects keyed by the full path of the file.
func (l *LocalFS) GetFiles() map[string]RolodexFile {
return l.Files
}
// GetErrors returns any errors that occurred during the indexing process.
func (l *LocalFS) GetErrors() []error {
return l.readingErrors
}
// Open opens a file, returning it or an error. If the file is not found, the error is of type *PathError.
func (l *LocalFS) Open(name string) (fs.File, error) {
if l.indexConfig != nil && !l.indexConfig.AllowFileLookup {
@@ -53,6 +57,7 @@ func (l *LocalFS) Open(name string) (fs.File, error) {
}
}
// LocalFile is a file that has been indexed by the LocalFS. It implements the RolodexFile interface.
type LocalFile struct {
filename string
name string
@@ -66,10 +71,12 @@ type LocalFile struct {
offset int64
}
// GetIndex returns the *SpecIndex for the file.
func (l *LocalFile) GetIndex() *SpecIndex {
return l.index
}
// Index returns the *SpecIndex for the file. If the index has not been created, it will be created (indexed)
func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
if l.index != nil {
return l.index, nil
@@ -90,10 +97,13 @@ func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
}
// GetContent returns the content of the file as a string.
func (l *LocalFile) GetContent() string {
return string(l.data)
}
// GetContentAsYAMLNode returns the content of the file as a *yaml.Node. If something went wrong
// then an error is returned.
func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) {
if l.parsed != nil {
return l.parsed, nil
@@ -116,26 +126,95 @@ func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) {
return &root, nil
}
// GetFileExtension returns the FileExtension of the file.
func (l *LocalFile) GetFileExtension() FileExtension {
return l.extension
}
// GetFullPath returns the full path of the file.
func (l *LocalFile) GetFullPath() string {
return l.fullPath
}
// GetErrors returns any errors that occurred during the indexing process.
func (l *LocalFile) GetErrors() []error {
return l.readingErrors
}
// FullPath returns the full path of the file.
func (l *LocalFile) FullPath() string {
return l.fullPath
}
// Name returns the name of the file.
func (l *LocalFile) Name() string {
return l.name
}
// Size returns the size of the file.
func (l *LocalFile) Size() int64 {
return int64(len(l.data))
}
// Mode returns the file mode bits for the file.
func (l *LocalFile) Mode() fs.FileMode {
return fs.FileMode(0)
}
// ModTime returns the modification time of the file.
func (l *LocalFile) ModTime() time.Time {
return l.lastModified
}
// IsDir returns true if the file is a directory, it always returns false
func (l *LocalFile) IsDir() bool {
return false
}
// Sys returns the underlying data source (always returns nil)
func (l *LocalFile) Sys() interface{} {
return nil
}
// Close closes the file (doesn't do anything, returns no error)
func (l *LocalFile) Close() error {
return nil
}
// Stat returns the FileInfo for the file.
func (l *LocalFile) Stat() (fs.FileInfo, error) {
return l, nil
}
// Read reads the file into a byte slice, makes it compatible with io.Reader.
func (l *LocalFile) Read(b []byte) (int, error) {
if l.offset >= int64(len(l.GetContent())) {
return 0, io.EOF
}
if l.offset < 0 {
return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid}
}
n := copy(b, l.GetContent()[l.offset:])
l.offset += int64(n)
return n, nil
}
// LocalFSConfig is the configuration for the LocalFS.
type LocalFSConfig struct {
// the base directory to index
BaseDirectory string
Logger *slog.Logger
FileFilters []string
DirFS fs.FS
// supply your own logger
Logger *slog.Logger
// supply a list of specific files to index only
FileFilters []string
// supply a custom fs.FS to use
DirFS fs.FS
}
// NewLocalFSWithConfig creates a new LocalFS with the supplied configuration.
func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) {
localFiles := make(map[string]RolodexFile)
var allErrors []error
@@ -163,15 +242,12 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) {
if d.IsDir() {
return nil
}
if len(ext) > 2 && p != file {
return nil
}
if strings.HasPrefix(p, ".") {
return nil
}
if len(config.FileFilters) > 0 {
if !slices.Contains(config.FileFilters, p) {
return nil
@@ -223,6 +299,7 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) {
}, nil
}
// NewLocalFS creates a new LocalFS with the supplied base directory.
func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) {
config := &LocalFSConfig{
BaseDirectory: baseDir,
@@ -230,51 +307,3 @@ func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) {
}
return NewLocalFSWithConfig(config)
}
func (l *LocalFile) FullPath() string {
return l.fullPath
}
func (l *LocalFile) Name() string {
return l.name
}
func (l *LocalFile) Size() int64 {
return int64(len(l.data))
}
func (l *LocalFile) Mode() fs.FileMode {
return fs.FileMode(0)
}
func (l *LocalFile) ModTime() time.Time {
return l.lastModified
}
func (l *LocalFile) IsDir() bool {
return false
}
func (l *LocalFile) Sys() interface{} {
return nil
}
func (l *LocalFile) Close() error {
return nil
}
func (l *LocalFile) Stat() (fs.FileInfo, error) {
return l, nil
}
func (l *LocalFile) Read(b []byte) (int, error) {
if l.offset >= int64(len(l.GetContent())) {
return 0, io.EOF
}
if l.offset < 0 {
return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid}
}
n := copy(b, l.GetContent()[l.offset:])
l.offset += int64(n)
return n, nil
}

7
index/rolodex_ref_extractor.go
View File

@@ -8,19 +8,20 @@ import (
"strings"
)
type RefType int
const (
Local RefType = iota
File
HTTP
)
type RefType int
type ExtractedRef struct {
Location string
Type RefType
}
// GetFile returns the file path of the reference.
func (r *ExtractedRef) GetFile() string {
switch r.Type {
case File, HTTP:
@@ -31,6 +32,7 @@ func (r *ExtractedRef) GetFile() string {
}
}
// GetReference returns the reference path of the reference.
func (r *ExtractedRef) GetReference() string {
switch r.Type {
case File, HTTP:
@@ -41,6 +43,7 @@ func (r *ExtractedRef) GetReference() string {
}
}
// ExtractFileType returns the file extension of the reference.
func ExtractFileType(ref string) FileExtension {
if strings.HasSuffix(ref, ".yaml") {
return YAML

56
index/rolodex_remote_loader.go
View File

@@ -21,6 +21,17 @@ import (
"time"
)
const (
YAML FileExtension = iota
JSON
UNSUPPORTED
)
// FileExtension is the type of file extension.
type FileExtension int
// RemoteFS is a file system that indexes remote files. It implements the fs.FS interface. Files are located remotely
// and served via HTTP.
type RemoteFS struct {
indexConfig *SpecIndexConfig
rootURL string
@@ -35,6 +46,7 @@ type RemoteFS struct {
extractedFiles map[string]RolodexFile
}
// RemoteFile is a file that has been indexed by the RemoteFS. It implements the RolodexFile interface.
type RemoteFile struct {
filename string
name string
@@ -49,14 +61,17 @@ type RemoteFile struct {
offset int64
}
// GetFileName returns the name of the file.
func (f *RemoteFile) GetFileName() string {
return f.filename
}
// GetContent returns the content of the file as a string.
func (f *RemoteFile) GetContent() string {
return string(f.data)
}
// GetContentAsYAMLNode returns the content of the file as a yaml.Node.
func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) {
if f.parsed != nil {
return f.parsed, nil
@@ -79,56 +94,71 @@ func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) {
return &root, nil
}
// GetFileExtension returns the file extension of the file.
func (f *RemoteFile) GetFileExtension() FileExtension {
return f.extension
}
// GetLastModified returns the last modified time of the file.
func (f *RemoteFile) GetLastModified() time.Time {
return f.lastModified
}
// GetErrors returns any errors that occurred while reading the file.
func (f *RemoteFile) GetErrors() []error {
return f.seekingErrors
}
// GetFullPath returns the full path of the file.
func (f *RemoteFile) GetFullPath() string {
return f.fullPath
}
// fs.FileInfo interfaces
// Name returns the name of the file.
func (f *RemoteFile) Name() string {
return f.name
}
// Size returns the size of the file.
func (f *RemoteFile) Size() int64 {
return int64(len(f.data))
}
// Mode returns the file mode bits for the file.
func (f *RemoteFile) Mode() fs.FileMode {
return fs.FileMode(0)
}
// ModTime returns the modification time of the file.
func (f *RemoteFile) ModTime() time.Time {
return f.lastModified
}
// IsDir returns true if the file is a directory.
func (f *RemoteFile) IsDir() bool {
return false
}
// fs.File interfaces
// Sys returns the underlying data source (always returns nil)
func (f *RemoteFile) Sys() interface{} {
return nil
}
// Close closes the file (doesn't do anything, returns no error)
func (f *RemoteFile) Close() error {
return nil
}
// Stat returns the FileInfo for the file.
func (f *RemoteFile) Stat() (fs.FileInfo, error) {
return f, nil
}
// Read reads the file. Makes it compatible with io.Reader.
func (f *RemoteFile) Read(b []byte) (int, error) {
if f.offset >= int64(len(f.data)) {
return 0, io.EOF
@@ -141,8 +171,8 @@ func (f *RemoteFile) Read(b []byte) (int, error) {
return n, nil
}
// Index indexes the file and returns a *SpecIndex, any errors are returned as well.
func (f *RemoteFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
if f.index != nil {
return f.index, nil
}
@@ -155,23 +185,17 @@ func (f *RemoteFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
}
index := NewSpecIndexWithConfig(info.RootNode, config)
index.specAbsolutePath = config.SpecAbsolutePath
f.index = index
return index, nil
}
// GetIndex returns the index for the file.
func (f *RemoteFile) GetIndex() *SpecIndex {
return f.index
}
type FileExtension int
const (
YAML FileExtension = iota
JSON
UNSUPPORTED
)
// NewRemoteFSWithConfig creates a new RemoteFS using the supplied SpecIndexConfig.
func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) {
if specIndexConfig == nil {
return nil, errors.New("no spec index config provided")
@@ -207,6 +231,7 @@ func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error)
return rfs, nil
}
// NewRemoteFSWithRootURL creates a new RemoteFS using the supplied root URL.
func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) {
remoteRootURL, err := url.Parse(rootURL)
if err != nil {
@@ -217,14 +242,17 @@ func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) {
return NewRemoteFSWithConfig(config)
}
// SetRemoteHandlerFunc sets the remote handler function.
func (i *RemoteFS) SetRemoteHandlerFunc(handlerFunc utils.RemoteURLHandler) {
i.RemoteHandlerFunc = handlerFunc
}
// SetIndexConfig sets the index configuration.
func (i *RemoteFS) SetIndexConfig(config *SpecIndexConfig) {
i.indexConfig = config
}
// GetFiles returns the files that have been indexed.
func (i *RemoteFS) GetFiles() map[string]RolodexFile {
files := make(map[string]RolodexFile)
i.Files.Range(func(key, value interface{}) bool {
@@ -235,10 +263,12 @@ func (i *RemoteFS) GetFiles() map[string]RolodexFile {
return files
}
// GetErrors returns any errors that occurred during the indexing process.
func (i *RemoteFS) GetErrors() []error {
return i.remoteErrors
}
// Open opens a file, returning it or an error. If the file is not found, the error is of type *PathError.
func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
if i.indexConfig != nil && !i.indexConfig.AllowRemoteLookup {
@@ -261,7 +291,8 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
// try path first
if _, ok := i.ProcessingFiles.Load(remoteParsedURL.Path); ok {
i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String())
i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL,
"remoteURL", remoteParsedURL.String())
// Create a context with a timeout of 50ms
ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Millisecond*50)
defer cancel()
@@ -277,7 +308,8 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
select {
case <-ctxTimeout.Done():
i.logger.Info("waiting for remote file timed out, trying again", "file", remoteURL, "remoteURL", remoteParsedURL.String())
i.logger.Info("waiting for remote file timed out, trying again", "file", remoteURL,
"remoteURL", remoteParsedURL.String())
case v := <-f:
return v, nil
}