From 0b08a63e63dc510f4568a9f9047debff04ab63b6 Mon Sep 17 00:00:00 2001 From: quobix Date: Tue, 31 Oct 2023 13:58:58 -0400 Subject: [PATCH] more coverage bumps Signed-off-by: quobix --- index/rolodex.go | 8 +- index/rolodex_file_loader.go | 393 ++++++++++++++---------------- index/rolodex_file_loader_test.go | 183 ++++++++++++-- index/rolodex_test.go | 4 +- 4 files changed, 351 insertions(+), 237 deletions(-) diff --git a/index/rolodex.go b/index/rolodex.go index 5badefe..b204d69 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -522,11 +522,9 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { } } // check if this is a native rolodex FS, then the work is done. - if lrf, ok := interface{}(f).(*localRolodexFile); ok { - if lf, ko := interface{}(lrf.f).(*LocalFile); ko { - localFile = lf - break - } + if lf, ko := interface{}(f).(*LocalFile); ko { + localFile = lf + break } else { // not a native FS, so we need to read the file and create a local file. bytes, rErr := io.ReadAll(f) diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 13dab1e..c5b926f 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -4,310 +4,277 @@ package index import ( - "fmt" - "github.com/pb33f/libopenapi/datamodel" - "golang.org/x/exp/slices" - "gopkg.in/yaml.v3" - "io" - "io/fs" - "log/slog" - "os" - "path/filepath" - "strings" - "time" + "fmt" + "github.com/pb33f/libopenapi/datamodel" + "golang.org/x/exp/slices" + "gopkg.in/yaml.v3" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "time" ) type LocalFS struct { - indexConfig *SpecIndexConfig - entryPointDirectory string - baseDirectory string - Files map[string]RolodexFile - logger *slog.Logger - readingErrors []error + indexConfig *SpecIndexConfig + entryPointDirectory string + baseDirectory string + Files map[string]RolodexFile + logger *slog.Logger + readingErrors []error } func (l *LocalFS) GetFiles() map[string]RolodexFile { - return l.Files + return l.Files } func (l *LocalFS) GetErrors() []error { - return l.readingErrors + return l.readingErrors } func (l *LocalFS) Open(name string) (fs.File, error) { - if l.indexConfig != nil && !l.indexConfig.AllowFileLookup { - return nil, &fs.PathError{Op: "open", Path: name, - Err: fmt.Errorf("file lookup for '%s' not allowed, set the index configuration "+ - "to AllowFileLookup to be true", name)} - } + if l.indexConfig != nil && !l.indexConfig.AllowFileLookup { + return nil, &fs.PathError{Op: "open", Path: name, + Err: fmt.Errorf("file lookup for '%s' not allowed, set the index configuration "+ + "to AllowFileLookup to be true", name)} + } - if !filepath.IsAbs(name) { - var absErr error - name, absErr = filepath.Abs(filepath.Join(l.baseDirectory, name)) - if absErr != nil { - return nil, absErr - } - } + if !filepath.IsAbs(name) { + name, _ = filepath.Abs(filepath.Join(l.baseDirectory, name)) + } - if f, ok := l.Files[name]; ok { - return &localRolodexFile{f: f}, nil - } else { - return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} - } + if f, ok := l.Files[name]; ok { + return f.(*LocalFile), nil + } else { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } } type LocalFile struct { - filename string - name string - extension FileExtension - data []byte - fullPath string - lastModified time.Time - readingErrors []error - index *SpecIndex - parsed *yaml.Node + filename string + name string + extension FileExtension + data []byte + fullPath string + lastModified time.Time + readingErrors []error + index *SpecIndex + parsed *yaml.Node + offset int64 } func (l *LocalFile) GetIndex() *SpecIndex { - return l.index + return l.index } func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { - if l.index != nil { - return l.index, nil - } - content := l.data + if l.index != nil { + return l.index, nil + } + content := l.data - // first, we must parse the content of the file - info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, true) - if err != nil { - return nil, err - } + // first, we must parse the content of the file + info, err := datamodel.ExtractSpecInfoWithDocumentCheck(content, true) + if err != nil { + return nil, err + } - index := NewSpecIndexWithConfig(info.RootNode, config) - index.specAbsolutePath = l.fullPath + index := NewSpecIndexWithConfig(info.RootNode, config) + index.specAbsolutePath = l.fullPath - l.index = index - return index, nil + l.index = index + return index, nil } func (l *LocalFile) GetContent() string { - return string(l.data) + return string(l.data) } func (l *LocalFile) GetContentAsYAMLNode() (*yaml.Node, error) { - if l.parsed != nil { - return l.parsed, nil - } - if l.index != nil && l.index.root != nil { - return l.index.root, nil - } - if l.data == nil { - return nil, fmt.Errorf("no data to parse for file: %s", l.fullPath) - } - var root yaml.Node - err := yaml.Unmarshal(l.data, &root) - if err != nil { - return nil, err - } - if l.index != nil && l.index.root == nil { - l.index.root = &root - } - l.parsed = &root - return &root, nil + if l.parsed != nil { + return l.parsed, nil + } + if l.index != nil && l.index.root != nil { + return l.index.root, nil + } + if l.data == nil { + return nil, fmt.Errorf("no data to parse for file: %s", l.fullPath) + } + var root yaml.Node + err := yaml.Unmarshal(l.data, &root) + if err != nil { + return nil, err + } + if l.index != nil && l.index.root == nil { + l.index.root = &root + } + l.parsed = &root + return &root, nil } func (l *LocalFile) GetFileExtension() FileExtension { - return l.extension + return l.extension } func (l *LocalFile) GetFullPath() string { - return l.fullPath + return l.fullPath } func (l *LocalFile) GetErrors() []error { - return l.readingErrors + return l.readingErrors } type LocalFSConfig struct { - // the base directory to index - BaseDirectory string - Logger *slog.Logger - FileFilters []string - DirFS fs.FS + // the base directory to index + BaseDirectory string + Logger *slog.Logger + FileFilters []string + DirFS fs.FS } func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { - localFiles := make(map[string]RolodexFile) - var allErrors []error + localFiles := make(map[string]RolodexFile) + var allErrors []error - log := config.Logger - if log == nil { - log = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ - Level: slog.LevelError, - })) - } + log := config.Logger + if log == nil { + log = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + } - // if the basedir is an absolute file, we're just going to index that file. - ext := filepath.Ext(config.BaseDirectory) - file := filepath.Base(config.BaseDirectory) + // if the basedir is an absolute file, we're just going to index that file. + ext := filepath.Ext(config.BaseDirectory) + file := filepath.Base(config.BaseDirectory) - var absBaseDir string - var absBaseErr error + var absBaseDir string + absBaseDir, _ = filepath.Abs(config.BaseDirectory) - absBaseDir, absBaseErr = filepath.Abs(config.BaseDirectory) + walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } - if absBaseErr != nil { - return nil, absBaseErr - } + // we don't care about directories, or errors, just read everything we can. + if d.IsDir() { + return nil + } - walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error { - if err != nil { - return err - } + if len(ext) > 2 && p != file { + return nil + } - // we don't care about directories, or errors, just read everything we can. - if d == nil || d.IsDir() { - return nil - } + if strings.HasPrefix(p, ".") { + return nil + } - if len(ext) > 2 && p != file { - return nil - } + if len(config.FileFilters) > 0 { + if !slices.Contains(config.FileFilters, p) { + return nil + } + } - if strings.HasPrefix(p, ".") { - return nil - } + extension := ExtractFileType(p) + var readingErrors []error + abs, _ := filepath.Abs(filepath.Join(config.BaseDirectory, p)) - if len(config.FileFilters) > 0 { - if !slices.Contains(config.FileFilters, p) { - return nil - } - } + var fileData []byte - extension := ExtractFileType(p) - var readingErrors []error - abs, absErr := filepath.Abs(filepath.Join(config.BaseDirectory, p)) - if absErr != nil { - readingErrors = append(readingErrors, absErr) - log.Error("cannot create absolute path for file: ", "file", p, "error", absErr.Error()) - } + switch extension { + case YAML, JSON: - var fileData []byte + dirFile, _ := config.DirFS.Open(p) + modTime := time.Now() + stat, _ := dirFile.Stat() + if stat != nil { + modTime = stat.ModTime() + } + fileData, _ = io.ReadAll(dirFile) + log.Debug("collecting JSON/YAML file", "file", abs) + localFiles[abs] = &LocalFile{ + filename: p, + name: filepath.Base(p), + extension: ExtractFileType(p), + data: fileData, + fullPath: abs, + lastModified: modTime, + readingErrors: readingErrors, + } + case UNSUPPORTED: + log.Debug("skipping non JSON/YAML file", "file", abs) + } + return nil + }) - switch extension { - case YAML, JSON: + if walkErr != nil { + return nil, walkErr + } - dirFile, readErr := config.DirFS.Open(p) - modTime := time.Now() - if readErr != nil { - allErrors = append(allErrors, readErr) - log.Error("[rolodex] cannot open file: ", "file", abs, "error", readErr.Error()) - return nil - } - stat, statErr := dirFile.Stat() - if statErr != nil { - allErrors = append(allErrors, statErr) - log.Error("[rolodex] cannot stat file: ", "file", abs, "error", statErr.Error()) - } - if stat != nil { - modTime = stat.ModTime() - } - fileData, readErr = io.ReadAll(dirFile) - if readErr != nil { - allErrors = append(allErrors, readErr) - log.Error("cannot read file data: ", "file", abs, "error", readErr.Error()) - return nil - } - - log.Debug("collecting JSON/YAML file", "file", abs) - localFiles[abs] = &LocalFile{ - filename: p, - name: filepath.Base(p), - extension: ExtractFileType(p), - data: fileData, - fullPath: abs, - lastModified: modTime, - readingErrors: readingErrors, - } - case UNSUPPORTED: - log.Debug("skipping non JSON/YAML file", "file", abs) - } - return nil - }) - - if walkErr != nil { - return nil, walkErr - } - - return &LocalFS{ - Files: localFiles, - logger: log, - baseDirectory: absBaseDir, - entryPointDirectory: config.BaseDirectory, - readingErrors: allErrors, - }, nil + return &LocalFS{ + Files: localFiles, + logger: log, + baseDirectory: absBaseDir, + entryPointDirectory: config.BaseDirectory, + readingErrors: allErrors, + }, nil } func NewLocalFS(baseDir string, dirFS fs.FS) (*LocalFS, error) { - config := &LocalFSConfig{ - BaseDirectory: baseDir, - DirFS: dirFS, - } - return NewLocalFSWithConfig(config) + config := &LocalFSConfig{ + BaseDirectory: baseDir, + DirFS: dirFS, + } + return NewLocalFSWithConfig(config) } func (l *LocalFile) FullPath() string { - return l.fullPath + return l.fullPath } func (l *LocalFile) Name() string { - return l.name + return l.name } func (l *LocalFile) Size() int64 { - return int64(len(l.data)) + return int64(len(l.data)) } func (l *LocalFile) Mode() fs.FileMode { - return fs.FileMode(0) + return fs.FileMode(0) } func (l *LocalFile) ModTime() time.Time { - return l.lastModified + return l.lastModified } func (l *LocalFile) IsDir() bool { - return false + return false } func (l *LocalFile) Sys() interface{} { - return nil + return nil } -type localRolodexFile struct { - f RolodexFile - offset int64 +func (l *LocalFile) Close() error { + return nil } -func (r *localRolodexFile) Close() error { - return nil +func (l *LocalFile) Stat() (fs.FileInfo, error) { + return l, nil } -func (r *localRolodexFile) Stat() (fs.FileInfo, error) { - return r.f, nil -} - -func (r *localRolodexFile) Read(b []byte) (int, error) { - if r.offset >= int64(len(r.f.GetContent())) { - return 0, io.EOF - } - if r.offset < 0 { - return 0, &fs.PathError{Op: "read", Path: r.f.GetFullPath(), Err: fs.ErrInvalid} - } - n := copy(b, r.f.GetContent()[r.offset:]) - r.offset += int64(n) - return n, nil +func (l *LocalFile) Read(b []byte) (int, error) { + if l.offset >= int64(len(l.GetContent())) { + return 0, io.EOF + } + if l.offset < 0 { + return 0, &fs.PathError{Op: "read", Path: l.GetFullPath(), Err: fs.ErrInvalid} + } + n := copy(b, l.GetContent()[l.offset:]) + l.offset += int64(n) + return n, nil } diff --git a/index/rolodex_file_loader_test.go b/index/rolodex_file_loader_test.go index e4387ed..9989731 100644 --- a/index/rolodex_file_loader_test.go +++ b/index/rolodex_file_loader_test.go @@ -4,26 +4,175 @@ package index import ( - "github.com/stretchr/testify/assert" - "testing" - "testing/fstest" - "time" + "github.com/stretchr/testify/assert" + "io" + "io/fs" + "path/filepath" + "testing" + "testing/fstest" + "time" ) func TestRolodexLoadsFilesCorrectly_NoErrors(t *testing.T) { - t.Parallel() - testFS := fstest.MapFS{ - "spec.yaml": {Data: []byte("hip"), ModTime: time.Now()}, - "subfolder/spec1.json": {Data: []byte("hop"), ModTime: time.Now()}, - "subfolder2/spec2.yaml": {Data: []byte("chop"), ModTime: time.Now()}, - "subfolder2/hello.jpg": {Data: []byte("shop"), ModTime: time.Now()}, - } + t.Parallel() + testFS := fstest.MapFS{ + "spec.yaml": {Data: []byte("hip"), ModTime: time.Now()}, + "spock.yaml": {Data: []byte("hip: : hello: :\n:hw"), ModTime: time.Now()}, + "subfolder/spec1.json": {Data: []byte("hop"), ModTime: time.Now()}, + "subfolder2/spec2.yaml": {Data: []byte("chop"), ModTime: time.Now()}, + "subfolder2/hello.jpg": {Data: []byte("shop"), ModTime: time.Now()}, + } - fileFS, err := NewLocalFS(".", testFS) - if err != nil { - t.Fatal(err) - } + fileFS, err := NewLocalFS(".", testFS) + if err != nil { + t.Fatal(err) + } + + files := fileFS.GetFiles() + assert.Len(t, files, 4) + assert.Len(t, fileFS.GetErrors(), 0) + + key, _ := filepath.Abs(filepath.Join(fileFS.baseDirectory, "spec.yaml")) + + localFile := files[key] + assert.NotNil(t, localFile) + assert.Nil(t, localFile.GetIndex()) + + lf := localFile.(*LocalFile) + idx, ierr := lf.Index(CreateOpenAPIIndexConfig()) + assert.NoError(t, ierr) + assert.NotNil(t, idx) + assert.NotNil(t, localFile.GetContent()) + + d, e := localFile.GetContentAsYAMLNode() + assert.NoError(t, e) + assert.NotNil(t, d) + assert.NotNil(t, localFile.GetIndex()) + assert.Equal(t, YAML, localFile.GetFileExtension()) + assert.Equal(t, key, localFile.GetFullPath()) + assert.Equal(t, "spec.yaml", lf.Name()) + assert.Equal(t, int64(3), lf.Size()) + assert.Equal(t, fs.FileMode(0), lf.Mode()) + assert.False(t, lf.IsDir()) + assert.Equal(t, time.Now().Unix(), lf.ModTime().Unix()) + assert.Nil(t, lf.Sys()) + assert.Nil(t, lf.Close()) + q, w := lf.Stat() + assert.NotNil(t, q) + assert.NoError(t, w) + + b, x := io.ReadAll(lf) + assert.Len(t, b, 3) + assert.NoError(t, x) + + assert.Equal(t, key, lf.FullPath()) + assert.Len(t, localFile.GetErrors(), 0) + + // try and reindex + idx, ierr = lf.Index(CreateOpenAPIIndexConfig()) + assert.NoError(t, ierr) + assert.NotNil(t, idx) + + key, _ = filepath.Abs(filepath.Join(fileFS.baseDirectory, "spock.yaml")) + + localFile = files[key] + assert.NotNil(t, localFile) + assert.Nil(t, localFile.GetIndex()) + + lf = localFile.(*LocalFile) + idx, ierr = lf.Index(CreateOpenAPIIndexConfig()) + assert.Error(t, ierr) + assert.Nil(t, idx) + assert.NotNil(t, localFile.GetContent()) + assert.Nil(t, localFile.GetIndex()) + +} + +func TestRolodexLocalFS_NoConfig(t *testing.T) { + + lfs := &LocalFS{} + f, e := lfs.Open("test.yaml") + assert.Nil(t, f) + assert.Error(t, e) +} + +func TestRolodexLocalFS_NoLookup(t *testing.T) { + + cf := CreateClosedAPIIndexConfig() + lfs := &LocalFS{indexConfig: cf} + f, e := lfs.Open("test.yaml") + assert.Nil(t, f) + assert.Error(t, e) +} + +func TestRolodexLocalFS_BadAbsFile(t *testing.T) { + + cf := CreateOpenAPIIndexConfig() + lfs := &LocalFS{indexConfig: cf} + f, e := lfs.Open("/test.yaml") + assert.Nil(t, f) + assert.Error(t, e) +} + +func TestRolodexLocalFile_BadParse(t *testing.T) { + + lf := &LocalFile{} + n, e := lf.GetContentAsYAMLNode() + assert.Nil(t, n) + assert.Error(t, e) + assert.Equal(t, "no data to parse for file: ", e.Error()) +} + +func TestRolodexLocalFile_NoIndexRoot(t *testing.T) { + + lf := &LocalFile{data: []byte("burders"), index: &SpecIndex{}} + n, e := lf.GetContentAsYAMLNode() + assert.NotNil(t, n) + assert.NoError(t, e) + +} + +func TestRolodexLocalFile_IndexSingleFile(t *testing.T) { + + testFS := fstest.MapFS{ + "spec.yaml": {Data: []byte("hip"), ModTime: time.Now()}, + "spock.yaml": {Data: []byte("hop"), ModTime: time.Now()}, + "i-am-a-dir": {Mode: fs.FileMode(fs.ModeDir), ModTime: time.Now()}, + } + + fileFS, _ := NewLocalFS("spec.yaml", testFS) + files := fileFS.GetFiles() + assert.Len(t, files, 1) + +} + +func TestRolodexLocalFile_TestFilters(t *testing.T) { + + testFS := fstest.MapFS{ + "spec.yaml": {Data: []byte("hip"), ModTime: time.Now()}, + "spock.yaml": {Data: []byte("pip"), ModTime: time.Now()}, + "jam.jpg": {Data: []byte("sip"), ModTime: time.Now()}, + } + + fileFS, _ := NewLocalFSWithConfig(&LocalFSConfig{ + BaseDirectory: ".", + FileFilters: []string{"spec.yaml", "spock.yaml", "jam.jpg"}, + DirFS: testFS, + }) + files := fileFS.GetFiles() + assert.Len(t, files, 2) + +} + +func TestRolodexLocalFile_TestBasFS(t *testing.T) { + + testFS := test_badfs{} + + fileFS, err := NewLocalFSWithConfig(&LocalFSConfig{ + BaseDirectory: ".", + DirFS: &testFS, + }) + assert.Error(t, err) + assert.Nil(t, fileFS) - assert.Len(t, fileFS.Files, 3) - assert.Len(t, fileFS.readingErrors, 0) } diff --git a/index/rolodex_test.go b/index/rolodex_test.go index f109668..fd84a5b 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -96,10 +96,10 @@ type test_badfs struct { func (t *test_badfs) Open(v string) (fs.File, error) { ok := false - if v != "/" && v != "http://localhost/test.yaml" { + if v != "/" && v != "." && v != "http://localhost/test.yaml" { ok = true } - if v == "http://localhost/goodstat.yaml" || v == "goodstat.yaml" { + if v == "http://localhost/goodstat.yaml" || strings.HasSuffix(v, "goodstat.yaml") { ok = true t.goodstat = true }