From 80b2b2d0b55210b6bf042f14adf6d8821234404b Mon Sep 17 00:00:00 2001 From: quobix Date: Wed, 1 Nov 2023 16:14:11 -0400 Subject: [PATCH] More cleaning and added docs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’re ready for review Signed-off-by: quobix --- index/circular_reference_result.go | 1 + index/extract_refs.go | 8 -- index/find_component.go | 4 - index/index_model.go | 1 - index/resolver.go | 17 --- index/rolodex.go | 168 ++++------------------------- index/rolodex_file.go | 153 ++++++++++++++++++++++++++ index/rolodex_file_loader.go | 139 ++++++++++++++---------- index/rolodex_ref_extractor.go | 7 +- index/rolodex_remote_loader.go | 56 +++++++--- 10 files changed, 305 insertions(+), 249 deletions(-) create mode 100644 index/rolodex_file.go diff --git a/index/circular_reference_result.go b/index/circular_reference_result.go index 9d2ef03..6538b57 100644 --- a/index/circular_reference_result.go +++ b/index/circular_reference_result.go @@ -14,6 +14,7 @@ type CircularReferenceResult struct { IsInfiniteLoop bool // if all the definitions in the reference loop are marked as required, this is an infinite circular reference, thus is not allowed. } +// GenerateJourneyPath generates a string representation of the journey taken to find the circular reference. func (c *CircularReferenceResult) GenerateJourneyPath() string { buf := strings.Builder{} for i, ref := range c.Journey { diff --git a/index/extract_refs.go b/index/extract_refs.go index 6d9618b..6fe165a 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -215,13 +215,10 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, if strings.HasPrefix(uri[0], "http") { fullDefinitionPath = value componentName = fmt.Sprintf("#/%s", uri[1]) - } else { - if filepath.IsAbs(uri[0]) { fullDefinitionPath = value componentName = fmt.Sprintf("#/%s", uri[1]) - } else { // if the index has a base path, use that to resolve the path @@ -235,7 +232,6 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, } else { // if the index has a base URL, use that to resolve the path. if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) { - u := *index.config.BaseURL abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0])) u.Path = abs @@ -287,16 +283,12 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, u.Path = abs fullDefinitionPath = u.String() componentName = uri[0] - } else { - abs, _ := filepath.Abs(filepath.Join(defRoot, uri[0])) fullDefinitionPath = abs componentName = uri[0] - } } - } } } diff --git a/index/find_component.go b/index/find_component.go index 83e795f..1028ab8 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -102,7 +102,6 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { var absoluteFileLocation, fileName string // is this a local or a remote file? - fileName = filepath.Base(file) if filepath.IsAbs(file) || strings.HasPrefix(file, "http") { absoluteFileLocation = file @@ -119,14 +118,11 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { // if the absolute file location has no file ext, then get the rolodex root. ext := filepath.Ext(absoluteFileLocation) - var parsedDocument *yaml.Node var err error idx := index - if ext != "" { - // extract the document from the rolodex. rFile, rError := index.rolodex.Open(absoluteFileLocation) diff --git a/index/index_model.go b/index/index_model.go index 459a36d..a9e63d5 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -173,7 +173,6 @@ func CreateClosedAPIIndexConfig() *SpecIndexConfig { // everything is pre-walked if you need it. type SpecIndex struct { specAbsolutePath string - AbsoluteFile string rolodex *Rolodex // the rolodex is used to fetch remote and file based documents. allRefs map[string]*Reference // all (deduplicated) refs rawSequencedRefs []*Reference // all raw references in sequence as they are scanned, not deduped. diff --git a/index/resolver.go b/index/resolver.go index badd975..8af52c1 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -208,9 +208,6 @@ func visitIndexWithoutDamagingIt(res *Resolver, idx *SpecIndex) { res.VisitReference(schemaRef, seenReferences, journey, false) } } - //for _, c := range idx.GetChildren() { - // visitIndexWithoutDamagingIt(res, c) - //} } func visitIndex(res *Resolver, idx *SpecIndex) { @@ -377,7 +374,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No value := node.Content[i+1].Value var locatedRef *Reference - var fullDef string var definition string @@ -390,12 +386,9 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No if strings.HasPrefix(exp[0], "http") { fullDef = value } else { - if filepath.IsAbs(exp[0]) { fullDef = value - } else { - if strings.HasPrefix(ref.FullDefinition, "http") { // split the http URI into parts @@ -420,7 +413,6 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No } } } else { - // local component, full def is based on passed in ref if strings.HasPrefix(ref.FullDefinition, "http") { @@ -434,13 +426,10 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) } else { - // split the full def into parts fileDef := strings.Split(ref.FullDefinition, "#/") fullDef = fmt.Sprintf("%s#/%s", fileDef[0], exp[1]) - } - } } else { @@ -460,14 +449,12 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No // is the file def a http link? if strings.HasPrefix(fileDef[0], "http") { - u, _ := url.Parse(fileDef[0]) path, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0])) u.Path = path fullDef = u.String() } else { - fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0])) } } @@ -529,17 +516,13 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No if exp[0] != "" { if !strings.HasPrefix(exp[0], "http") { if !filepath.IsAbs(exp[0]) { - if strings.HasPrefix(ref.FullDefinition, "http") { - u, _ := url.Parse(ref.FullDefinition) p, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0])) u.Path = p u.Fragment = "" def = fmt.Sprintf("%s#/%s", u.String(), exp[1]) - } else { - fd := strings.Split(ref.FullDefinition, "#/") abs, _ := filepath.Abs(filepath.Join(filepath.Dir(fd[0]), exp[0])) def = fmt.Sprintf("%s#/%s", abs, exp[1]) diff --git a/index/rolodex.go b/index/rolodex.go index b204d69..1e0c3c6 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -6,7 +6,6 @@ package index import ( "errors" "fmt" - "github.com/pb33f/libopenapi/datamodel" "gopkg.in/yaml.v3" "io" "io/fs" @@ -46,6 +45,9 @@ type RolodexFS interface { GetFiles() map[string]RolodexFile } +// Rolodex is a file system abstraction that allows for the indexing of multiple file systems +// and the ability to resolve references across those file systems. It is used to hold references to external +// files, and the indexes they hold. The rolodex is the master lookup for all references. type Rolodex struct { localFS map[string]fs.FS remoteFS map[string]fs.FS @@ -63,148 +65,7 @@ type Rolodex struct { ignoredCircularReferences []*CircularReferenceResult } -type rolodexFile struct { - location string - rolodex *Rolodex - index *SpecIndex - localFile *LocalFile - remoteFile *RemoteFile -} - -func (rf *rolodexFile) Name() string { - if rf.localFile != nil { - return rf.localFile.filename - } - if rf.remoteFile != nil { - return rf.remoteFile.filename - } - return "" -} - -func (rf *rolodexFile) GetIndex() *SpecIndex { - if rf.localFile != nil { - return rf.localFile.GetIndex() - } - if rf.remoteFile != nil { - return rf.remoteFile.GetIndex() - } - return nil -} - -func (rf *rolodexFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { - if rf.index != nil { - return rf.index, nil - } - var content []byte - if rf.localFile != nil { - content = rf.localFile.data - } - if rf.remoteFile != nil { - content = rf.remoteFile.data - } - - // first, we must parse the content of the file - info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, config.SkipDocumentCheck) - if err != nil { - return nil, err - } - - // create a new index for this file and link it to this rolodex. - config.Rolodex = rf.rolodex - index := NewSpecIndexWithConfig(info.RootNode, config) - rf.index = index - return index, nil - -} - -func (rf *rolodexFile) GetContent() string { - if rf.localFile != nil { - return string(rf.localFile.data) - } - if rf.remoteFile != nil { - return string(rf.remoteFile.data) - } - return "" -} - -func (rf *rolodexFile) GetContentAsYAMLNode() (*yaml.Node, error) { - if rf.localFile != nil { - return rf.localFile.GetContentAsYAMLNode() - } - if rf.remoteFile != nil { - return rf.remoteFile.GetContentAsYAMLNode() - } - return nil, nil -} - -func (rf *rolodexFile) GetFileExtension() FileExtension { - if rf.localFile != nil { - return rf.localFile.extension - } - if rf.remoteFile != nil { - return rf.remoteFile.extension - } - return UNSUPPORTED -} -func (rf *rolodexFile) GetFullPath() string { - if rf.localFile != nil { - return rf.localFile.fullPath - } - if rf.remoteFile != nil { - return rf.remoteFile.fullPath - } - return "" -} -func (rf *rolodexFile) ModTime() time.Time { - if rf.localFile != nil { - return rf.localFile.lastModified - } - if rf.remoteFile != nil { - return rf.remoteFile.lastModified - } - return time.Now() -} - -func (rf *rolodexFile) Size() int64 { - if rf.localFile != nil { - return rf.localFile.Size() - } - if rf.remoteFile != nil { - return rf.remoteFile.Size() - } - return 0 -} - -func (rf *rolodexFile) IsDir() bool { - // always false. - return false -} - -func (rf *rolodexFile) Sys() interface{} { - // not implemented. - return nil -} - -func (rf *rolodexFile) Mode() os.FileMode { - if rf.localFile != nil { - return rf.localFile.Mode() - } - if rf.remoteFile != nil { - return rf.remoteFile.Mode() - } - return os.FileMode(0) -} - -func (rf *rolodexFile) GetErrors() []error { - if rf.localFile != nil { - return rf.localFile.readingErrors - } - if rf.remoteFile != nil { - return rf.remoteFile.seekingErrors - } - return nil -} - +// NewRolodex creates a new rolodex with the provided index configuration. func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex { r := &Rolodex{ indexConfig: indexConfig, @@ -215,6 +76,8 @@ func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex { return r } +// GetIgnoredCircularReferences returns a list of circular references that were ignored during the indexing process. +// These can be array or polymorphic references. func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult { debounced := make(map[string]*CircularReferenceResult) for _, c := range r.ignoredCircularReferences { @@ -229,35 +92,43 @@ func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult { return debouncedResults } +// GetIndexingDuration returns the duration it took to index the rolodex. func (r *Rolodex) GetIndexingDuration() time.Duration { return r.indexingDuration } +// GetRootIndex returns the root index of the rolodex (the entry point, the main document) func (r *Rolodex) GetRootIndex() *SpecIndex { return r.rootIndex } +// GetIndexes returns all the indexes in the rolodex. func (r *Rolodex) GetIndexes() []*SpecIndex { return r.indexes } +// GetCaughtErrors returns all the errors that were caught during the indexing process. func (r *Rolodex) GetCaughtErrors() []error { return r.caughtErrors } +// AddLocalFS adds a local file system to the rolodex. func (r *Rolodex) AddLocalFS(baseDir string, fileSystem fs.FS) { absBaseDir, _ := filepath.Abs(baseDir) r.localFS[absBaseDir] = fileSystem } +// SetRootNode sets the root node of the rolodex (the entry point, the main document) func (r *Rolodex) SetRootNode(node *yaml.Node) { r.rootNode = node } +// AddRemoteFS adds a remote file system to the rolodex. func (r *Rolodex) AddRemoteFS(baseURL string, fileSystem fs.FS) { r.remoteFS[baseURL] = fileSystem } +// IndexTheRolodex indexes the rolodex, building out the indexes for each file, and then building the root index. func (r *Rolodex) IndexTheRolodex() error { if r.indexed { return nil @@ -393,8 +264,6 @@ func (r *Rolodex) IndexTheRolodex() error { } } - // todo: variation with no base path, but a base URL. - index := NewSpecIndexWithConfig(r.rootNode, r.indexConfig) resolver := NewResolver(index) @@ -433,6 +302,7 @@ func (r *Rolodex) IndexTheRolodex() error { } +// CheckForCircularReferences checks for circular references in the rolodex. func (r *Rolodex) CheckForCircularReferences() { if !r.circChecked { if r.rootIndex != nil && r.rootIndex.resolver != nil { @@ -451,6 +321,7 @@ func (r *Rolodex) CheckForCircularReferences() { } } +// Resolve resolves references in the rolodex. func (r *Rolodex) Resolve() { if r.rootIndex != nil && r.rootIndex.resolver != nil { resolvingErrors := r.rootIndex.resolver.Resolve() @@ -467,6 +338,7 @@ func (r *Rolodex) Resolve() { r.resolved = true } +// BuildIndexes builds the indexes in the rolodex, this is generally not required unless manually building a rolodex. func (r *Rolodex) BuildIndexes() { if r.manualBuilt { return @@ -480,6 +352,7 @@ func (r *Rolodex) BuildIndexes() { r.manualBuilt = true } +// Open opens a file in the rolodex, and returns a RolodexFile. func (r *Rolodex) Open(location string) (RolodexFile, error) { if r == nil { return nil, fmt.Errorf("rolodex has not been initialized, cannot open file '%s'", location) @@ -490,10 +363,8 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { } var errorStack []error - var localFile *LocalFile var remoteFile *RemoteFile - fileLookup := location isUrl := false u, _ := url.Parse(location) @@ -502,18 +373,15 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { } if !isUrl { - for k, v := range r.localFS { // check if this is a URL or an abs/rel reference. - if !filepath.IsAbs(location) { fileLookup, _ = filepath.Abs(filepath.Join(k, location)) } f, err := v.Open(fileLookup) if err != nil { - // try a lookup that is not absolute, but relative f, err = v.Open(location) if err != nil { diff --git a/index/rolodex_file.go b/index/rolodex_file.go new file mode 100644 index 0000000..1268c7b --- /dev/null +++ b/index/rolodex_file.go @@ -0,0 +1,153 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package index + +import ( + "github.com/pb33f/libopenapi/datamodel" + "gopkg.in/yaml.v3" + "os" + "time" +) + +type rolodexFile struct { + location string + rolodex *Rolodex + index *SpecIndex + localFile *LocalFile + remoteFile *RemoteFile +} + +func (rf *rolodexFile) Name() string { + if rf.localFile != nil { + return rf.localFile.filename + } + if rf.remoteFile != nil { + return rf.remoteFile.filename + } + return "" +} + +func (rf *rolodexFile) GetIndex() *SpecIndex { + if rf.localFile != nil { + return rf.localFile.GetIndex() + } + if rf.remoteFile != nil { + return rf.remoteFile.GetIndex() + } + return nil +} + +func (rf *rolodexFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { + if rf.index != nil { + return rf.index, nil + } + var content []byte + if rf.localFile != nil { + content = rf.localFile.data + } + if rf.remoteFile != nil { + content = rf.remoteFile.data + } + + // first, we must parse the content of the file + info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, config.SkipDocumentCheck) + if err != nil { + return nil, err + } + + // create a new index for this file and link it to this rolodex. + config.Rolodex = rf.rolodex + index := NewSpecIndexWithConfig(info.RootNode, config) + rf.index = index + return index, nil + +} + +func (rf *rolodexFile) GetContent() string { + if rf.localFile != nil { + return string(rf.localFile.data) + } + if rf.remoteFile != nil { + return string(rf.remoteFile.data) + } + return "" +} + +func (rf *rolodexFile) GetContentAsYAMLNode() (*yaml.Node, error) { + if rf.localFile != nil { + return rf.localFile.GetContentAsYAMLNode() + } + if rf.remoteFile != nil { + return rf.remoteFile.GetContentAsYAMLNode() + } + return nil, nil +} + +func (rf *rolodexFile) GetFileExtension() FileExtension { + if rf.localFile != nil { + return rf.localFile.extension + } + if rf.remoteFile != nil { + return rf.remoteFile.extension + } + return UNSUPPORTED +} +func (rf *rolodexFile) GetFullPath() string { + if rf.localFile != nil { + return rf.localFile.fullPath + } + if rf.remoteFile != nil { + return rf.remoteFile.fullPath + } + return "" +} +func (rf *rolodexFile) ModTime() time.Time { + if rf.localFile != nil { + return rf.localFile.lastModified + } + if rf.remoteFile != nil { + return rf.remoteFile.lastModified + } + return time.Now() +} + +func (rf *rolodexFile) Size() int64 { + if rf.localFile != nil { + return rf.localFile.Size() + } + if rf.remoteFile != nil { + return rf.remoteFile.Size() + } + return 0 +} + +func (rf *rolodexFile) IsDir() bool { + // always false. + return false +} + +func (rf *rolodexFile) Sys() interface{} { + // not implemented. + return nil +} + +func (rf *rolodexFile) Mode() os.FileMode { + if rf.localFile != nil { + return rf.localFile.Mode() + } + if rf.remoteFile != nil { + return rf.remoteFile.Mode() + } + return os.FileMode(0) +} + +func (rf *rolodexFile) GetErrors() []error { + if rf.localFile != nil { + return rf.localFile.readingErrors + } + if rf.remoteFile != nil { + return rf.remoteFile.seekingErrors + } + return nil +} diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index c202a1e..5eddbfe 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -6,17 +6,18 @@ package index import ( "fmt" "github.com/pb33f/libopenapi/datamodel" - "golang.org/x/exp/slices" "gopkg.in/yaml.v3" "io" "io/fs" "log/slog" "os" "path/filepath" + "slices" "strings" "time" ) +// LocalFS is a file system that indexes local files. type LocalFS struct { indexConfig *SpecIndexConfig entryPointDirectory string @@ -26,14 +27,17 @@ type LocalFS struct { readingErrors []error } +// GetFiles returns the files that have been indexed. A map of RolodexFile objects keyed by the full path of the file. func (l *LocalFS) GetFiles() map[string]RolodexFile { return l.Files } +// GetErrors returns any errors that occurred during the indexing process. func (l *LocalFS) GetErrors() []error { return l.readingErrors } +// Open opens a file, returning it or an error. If the file is not found, the error is of type *PathError. func (l *LocalFS) Open(name string) (fs.File, error) { if l.indexConfig != nil && !l.indexConfig.AllowFileLookup { @@ -53,6 +57,7 @@ func (l *LocalFS) Open(name string) (fs.File, error) { } } +// LocalFile is a file that has been indexed by the LocalFS. It implements the RolodexFile interface. type LocalFile struct { filename string name string @@ -66,10 +71,12 @@ type LocalFile struct { offset int64 } +// GetIndex returns the *SpecIndex for the file. func (l *LocalFile) GetIndex() *SpecIndex { return l.index } +// Index returns the *SpecIndex for the file. If the index has not been created, it will be created (indexed) func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { if l.index != nil { return l.index, nil @@ -90,10 +97,13 @@ func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { } +// GetContent returns the content of the file as a string. func (l *LocalFile) GetContent() string { return string(l.data) } +// GetContentAsYAMLNode returns the content of the file as a *yaml.Node. If something went wrong +// then an error is returned. func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) { if l.parsed != nil { return l.parsed, nil @@ -116,26 +126,95 @@ func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) { return &root, nil } +// GetFileExtension returns the FileExtension of the file. func (l *LocalFile) GetFileExtension() FileExtension { return l.extension } +// GetFullPath returns the full path of the file. func (l *LocalFile) GetFullPath() string { return l.fullPath } +// GetErrors returns any errors that occurred during the indexing process. func (l *LocalFile) GetErrors() []error { return l.readingErrors } +// FullPath returns the full path of the file. +func (l *LocalFile) FullPath() string { + return l.fullPath +} + +// Name returns the name of the file. +func (l *LocalFile) Name() string { + return l.name +} + +// Size returns the size of the file. +func (l *LocalFile) Size() int64 { + return int64(len(l.data)) +} + +// Mode returns the file mode bits for the file. +func (l *LocalFile) Mode() fs.FileMode { + return fs.FileMode(0) +} + +// ModTime returns the modification time of the file. +func (l *LocalFile) ModTime() time.Time { + return l.lastModified +} + +// IsDir returns true if the file is a directory, it always returns false +func (l *LocalFile) IsDir() bool { + return false +} + +// Sys returns the underlying data source (always returns nil) +func (l *LocalFile) Sys() interface{} { + return nil +} + +// Close closes the file (doesn't do anything, returns no error) +func (l *LocalFile) Close() error { + return nil +} + +// Stat returns the FileInfo for the file. +func (l *LocalFile) Stat() (fs.FileInfo, error) { + return l, nil +} + +// Read reads the file into a byte slice, makes it compatible with io.Reader. +func (l *LocalFile) Read(b []byte) (int, error) { + if l.offset >= int64(len(l.GetContent())) { + return 0, io.EOF + } + if l.offset < 0 { + return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid} + } + n := copy(b, l.GetContent()[l.offset:]) + l.offset += int64(n) + return n, nil +} + +// LocalFSConfig is the configuration for the LocalFS. type LocalFSConfig struct { // the base directory to index BaseDirectory string - Logger *slog.Logger - FileFilters []string - DirFS fs.FS + + // supply your own logger + Logger *slog.Logger + + // supply a list of specific files to index only + FileFilters []string + + // supply a custom fs.FS to use + DirFS fs.FS } +// NewLocalFSWithConfig creates a new LocalFS with the supplied configuration. func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { localFiles := make(map[string]RolodexFile) var allErrors []error @@ -163,15 +242,12 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { if d.IsDir() { return nil } - if len(ext) > 2 && p != file { return nil } - if strings.HasPrefix(p, ".") { return nil } - if len(config.FileFilters) > 0 { if !slices.Contains(config.FileFilters, p) { return nil @@ -223,6 +299,7 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { }, nil } +// NewLocalFS creates a new LocalFS with the supplied base directory. func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { config := &LocalFSConfig{ BaseDirectory: baseDir, @@ -230,51 +307,3 @@ func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { } return NewLocalFSWithConfig(config) } - -func (l *LocalFile) FullPath() string { - return l.fullPath -} - -func (l *LocalFile) Name() string { - return l.name -} - -func (l *LocalFile) Size() int64 { - return int64(len(l.data)) -} - -func (l *LocalFile) Mode() fs.FileMode { - return fs.FileMode(0) -} - -func (l *LocalFile) ModTime() time.Time { - return l.lastModified -} - -func (l *LocalFile) IsDir() bool { - return false -} - -func (l *LocalFile) Sys() interface{} { - return nil -} - -func (l *LocalFile) Close() error { - return nil -} - -func (l *LocalFile) Stat() (fs.FileInfo, error) { - return l, nil -} - -func (l *LocalFile) Read(b []byte) (int, error) { - if l.offset >= int64(len(l.GetContent())) { - return 0, io.EOF - } - if l.offset < 0 { - return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid} - } - n := copy(b, l.GetContent()[l.offset:]) - l.offset += int64(n) - return n, nil -} diff --git a/index/rolodex_ref_extractor.go b/index/rolodex_ref_extractor.go index d60202a..a795fb8 100644 --- a/index/rolodex_ref_extractor.go +++ b/index/rolodex_ref_extractor.go @@ -8,19 +8,20 @@ import ( "strings" ) -type RefType int - const ( Local RefType = iota File HTTP ) +type RefType int + type ExtractedRef struct { Location string Type RefType } +// GetFile returns the file path of the reference. func (r *ExtractedRef) GetFile() string { switch r.Type { case File, HTTP: @@ -31,6 +32,7 @@ func (r *ExtractedRef) GetFile() string { } } +// GetReference returns the reference path of the reference. func (r *ExtractedRef) GetReference() string { switch r.Type { case File, HTTP: @@ -41,6 +43,7 @@ func (r *ExtractedRef) GetReference() string { } } +// ExtractFileType returns the file extension of the reference. func ExtractFileType(ref string) FileExtension { if strings.HasSuffix(ref, ".yaml") { return YAML diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index c419a3b..731990a 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -21,6 +21,17 @@ import ( "time" ) +const ( + YAML FileExtension = iota + JSON + UNSUPPORTED +) + +// FileExtension is the type of file extension. +type FileExtension int + +// RemoteFS is a file system that indexes remote files. It implements the fs.FS interface. Files are located remotely +// and served via HTTP. type RemoteFS struct { indexConfig *SpecIndexConfig rootURL string @@ -35,6 +46,7 @@ type RemoteFS struct { extractedFiles map[string]RolodexFile } +// RemoteFile is a file that has been indexed by the RemoteFS. It implements the RolodexFile interface. type RemoteFile struct { filename string name string @@ -49,14 +61,17 @@ type RemoteFile struct { offset int64 } +// GetFileName returns the name of the file. func (f *RemoteFile) GetFileName() string { return f.filename } +// GetContent returns the content of the file as a string. func (f *RemoteFile) GetContent() string { return string(f.data) } +// GetContentAsYAMLNode returns the content of the file as a yaml.Node. func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) { if f.parsed != nil { return f.parsed, nil @@ -79,56 +94,71 @@ func (f *RemoteFile) GetContentAsYAMLNode() (*yaml.Node, error) { return &root, nil } +// GetFileExtension returns the file extension of the file. func (f *RemoteFile) GetFileExtension() FileExtension { return f.extension } +// GetLastModified returns the last modified time of the file. func (f *RemoteFile) GetLastModified() time.Time { return f.lastModified } +// GetErrors returns any errors that occurred while reading the file. func (f *RemoteFile) GetErrors() []error { return f.seekingErrors } +// GetFullPath returns the full path of the file. func (f *RemoteFile) GetFullPath() string { return f.fullPath } // fs.FileInfo interfaces +// Name returns the name of the file. func (f *RemoteFile) Name() string { return f.name } +// Size returns the size of the file. func (f *RemoteFile) Size() int64 { return int64(len(f.data)) } +// Mode returns the file mode bits for the file. func (f *RemoteFile) Mode() fs.FileMode { return fs.FileMode(0) } +// ModTime returns the modification time of the file. func (f *RemoteFile) ModTime() time.Time { return f.lastModified } +// IsDir returns true if the file is a directory. func (f *RemoteFile) IsDir() bool { return false } // fs.File interfaces +// Sys returns the underlying data source (always returns nil) func (f *RemoteFile) Sys() interface{} { return nil } +// Close closes the file (doesn't do anything, returns no error) func (f *RemoteFile) Close() error { return nil } + +// Stat returns the FileInfo for the file. func (f *RemoteFile) Stat() (fs.FileInfo, error) { return f, nil } + +// Read reads the file. Makes it compatible with io.Reader. func (f *RemoteFile) Read(b []byte) (int, error) { if f.offset >= int64(len(f.data)) { return 0, io.EOF @@ -141,8 +171,8 @@ func (f *RemoteFile) Read(b []byte) (int, error) { return n, nil } +// Index indexes the file and returns a *SpecIndex, any errors are returned as well. func (f *RemoteFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { - if f.index != nil { return f.index, nil } @@ -155,23 +185,17 @@ func (f *RemoteFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { } index := NewSpecIndexWithConfig(info.RootNode, config) - index.specAbsolutePath = config.SpecAbsolutePath f.index = index return index, nil } + +// GetIndex returns the index for the file. func (f *RemoteFile) GetIndex() *SpecIndex { return f.index } -type FileExtension int - -const ( - YAML FileExtension = iota - JSON - UNSUPPORTED -) - +// NewRemoteFSWithConfig creates a new RemoteFS using the supplied SpecIndexConfig. func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) { if specIndexConfig == nil { return nil, errors.New("no spec index config provided") @@ -207,6 +231,7 @@ func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) return rfs, nil } +// NewRemoteFSWithRootURL creates a new RemoteFS using the supplied root URL. func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) { remoteRootURL, err := url.Parse(rootURL) if err != nil { @@ -217,14 +242,17 @@ func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) { return NewRemoteFSWithConfig(config) } +// SetRemoteHandlerFunc sets the remote handler function. func (i *RemoteFS) SetRemoteHandlerFunc(handlerFunc utils.RemoteURLHandler) { i.RemoteHandlerFunc = handlerFunc } +// SetIndexConfig sets the index configuration. func (i *RemoteFS) SetIndexConfig(config *SpecIndexConfig) { i.indexConfig = config } +// GetFiles returns the files that have been indexed. func (i *RemoteFS) GetFiles() map[string]RolodexFile { files := make(map[string]RolodexFile) i.Files.Range(func(key, value interface{}) bool { @@ -235,10 +263,12 @@ func (i *RemoteFS) GetFiles() map[string]RolodexFile { return files } +// GetErrors returns any errors that occurred during the indexing process. func (i *RemoteFS) GetErrors() []error { return i.remoteErrors } +// Open opens a file, returning it or an error. If the file is not found, the error is of type *PathError. func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { if i.indexConfig != nil && !i.indexConfig.AllowRemoteLookup { @@ -261,7 +291,8 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { // try path first if _, ok := i.ProcessingFiles.Load(remoteParsedURL.Path); ok { - i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String()) + i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, + "remoteURL", remoteParsedURL.String()) // Create a context with a timeout of 50ms ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Millisecond*50) defer cancel() @@ -277,7 +308,8 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { select { case <-ctxTimeout.Done(): - i.logger.Info("waiting for remote file timed out, trying again", "file", remoteURL, "remoteURL", remoteParsedURL.String()) + i.logger.Info("waiting for remote file timed out, trying again", "file", remoteURL, + "remoteURL", remoteParsedURL.String()) case v := <-f: return v, nil }