diff --git a/datamodel/low/v2/swagger.go b/datamodel/low/v2/swagger.go index 95eff28..d6f3c91 100644 --- a/datamodel/low/v2/swagger.go +++ b/datamodel/low/v2/swagger.go @@ -19,7 +19,6 @@ import ( "github.com/pb33f/libopenapi/datamodel/low/base" "github.com/pb33f/libopenapi/index" "gopkg.in/yaml.v3" - "os" "path/filepath" ) @@ -163,7 +162,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur // create a local filesystem localFSConf := index.LocalFSConfig{ BaseDirectory: cwd, - DirFS: os.DirFS(cwd), + IndexConfig: idxConfig, FileFilters: config.FileFilter, } fileFS, err := index.NewLocalFSWithConfig(&localFSConf) diff --git a/datamodel/low/v2/swagger_test.go b/datamodel/low/v2/swagger_test.go index af114bc..970c1e0 100644 --- a/datamodel/low/v2/swagger_test.go +++ b/datamodel/low/v2/swagger_test.go @@ -406,7 +406,7 @@ func TestRolodexLocalFileSystem_BadPath(t *testing.T) { cf.BasePath = "/NOWHERE" cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"} lDoc, err := CreateDocumentFromConfig(info, cf) - assert.Nil(t, lDoc) + assert.NotNil(t, lDoc) assert.Error(t, err) } diff --git a/datamodel/low/v3/create_document.go b/datamodel/low/v3/create_document.go index 9f32262..fefb144 100644 --- a/datamodel/low/v3/create_document.go +++ b/datamodel/low/v3/create_document.go @@ -3,7 +3,6 @@ package v3 import ( "context" "errors" - "os" "path/filepath" "sync" @@ -61,7 +60,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur // create a local filesystem localFSConf := index.LocalFSConfig{ BaseDirectory: cwd, - DirFS: os.DirFS(cwd), + IndexConfig: idxConfig, FileFilters: config.FileFilter, } diff --git a/datamodel/low/v3/create_document_test.go b/datamodel/low/v3/create_document_test.go index fc91222..81c73bb 100644 --- a/datamodel/low/v3/create_document_test.go +++ b/datamodel/low/v3/create_document_test.go @@ -114,7 +114,7 @@ func TestRolodexLocalFileSystem_BadPath(t *testing.T) { cf.BasePath = "/NOWHERE" cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"} lDoc, err := CreateDocumentFromConfig(info, cf) - assert.Nil(t, lDoc) + assert.NotNil(t, lDoc) assert.Error(t, err) } diff --git a/index/find_component_test.go b/index/find_component_test.go index c16cff5..919c675 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -71,6 +71,49 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) { assert.Nil(t, c) } +func TestSpecIndex_CheckCircularIndex_NoDirFS(t *testing.T) { + + cFile := "../test_specs/first.yaml" + yml, _ := os.ReadFile(cFile) + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + cf := CreateOpenAPIIndexConfig() + cf.AvoidCircularReferenceCheck = true + cf.BasePath = "../test_specs" + + rolo := NewRolodex(cf) + rolo.SetRootNode(&rootNode) + cf.Rolodex = rolo + + fsCfg := LocalFSConfig{ + BaseDirectory: cf.BasePath, + IndexConfig: cf, + } + + fileFS, err := NewLocalFSWithConfig(&fsCfg) + + assert.NoError(t, err) + rolo.AddLocalFS(cf.BasePath, fileFS) + + indexedErr := rolo.IndexTheRolodex() + rolo.BuildIndexes() + + assert.NoError(t, indexedErr) + + index := rolo.GetRootIndex() + + assert.Nil(t, index.uri) + + a, _ := index.SearchIndexForReference("second.yaml#/properties/property2") + b, _ := index.SearchIndexForReference("second.yaml") + c, _ := index.SearchIndexForReference("fourth.yaml") + + assert.NotNil(t, a) + assert.NotNil(t, b) + assert.Nil(t, c) +} + func TestFindComponent_RolodexFileParseError(t *testing.T) { badData := "I cannot be parsed: \"I am not a YAML file or a JSON file" diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index 5eddbfe..38fec9b 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -6,6 +6,7 @@ package index import ( "fmt" "github.com/pb33f/libopenapi/datamodel" + "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" "io" "io/fs" @@ -14,22 +15,32 @@ import ( "path/filepath" "slices" "strings" + "sync" "time" ) // LocalFS is a file system that indexes local files. type LocalFS struct { + fsConfig *LocalFSConfig indexConfig *SpecIndexConfig entryPointDirectory string baseDirectory string - Files map[string]RolodexFile + Files syncmap.Map + extractedFiles map[string]RolodexFile logger *slog.Logger + fileLock sync.Mutex readingErrors []error } // GetFiles returns the files that have been indexed. A map of RolodexFile objects keyed by the full path of the file. func (l *LocalFS) GetFiles() map[string]RolodexFile { - return l.Files + files := make(map[string]RolodexFile) + l.Files.Range(func(key, value interface{}) bool { + files[key.(string)] = value.(*LocalFile) + return true + }) + l.extractedFiles = files + return files } // GetErrors returns any errors that occurred during the indexing process. @@ -50,11 +61,47 @@ func (l *LocalFS) Open(name string) (fs.File, error) { name, _ = filepath.Abs(filepath.Join(l.baseDirectory, name)) } - if f, ok := l.Files[name]; ok { + if f, ok := l.Files.Load(name); ok { return f.(*LocalFile), nil } else { - return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + + if l.fsConfig != nil && l.fsConfig.DirFS == nil { + var extractedFile *LocalFile + var extErr error + // attempt to open the file from the local filesystem + extractedFile, extErr = l.extractFile(name) + if extErr != nil { + return nil, extErr + } + if extractedFile != nil { + + // in this mode, we need the index config to be set. + if l.indexConfig != nil { + copiedCfg := *l.indexConfig + copiedCfg.SpecAbsolutePath = name + copiedCfg.AvoidBuildIndex = true + + idx, idxError := extractedFile.Index(&copiedCfg) + + if idxError != nil && idx == nil { + l.readingErrors = append(l.readingErrors, idxError) + } else { + + // for each index, we need a resolver + resolver := NewResolver(idx) + idx.resolver = resolver + idx.BuildIndex() + } + + if len(extractedFile.data) > 0 { + l.logger.Debug("successfully loaded and indexed file", "file", name) + } + return extractedFile, nil + } + } + } } + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} } // LocalFile is a file that has been indexed by the LocalFS. It implements the RolodexFile interface. @@ -212,11 +259,13 @@ type LocalFSConfig struct { // supply a custom fs.FS to use DirFS fs.FS + + // supply an index configuration to use + IndexConfig *SpecIndexConfig } // NewLocalFSWithConfig creates a new LocalFS with the supplied configuration. func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { - localFiles := make(map[string]RolodexFile) var allErrors []error log := config.Logger @@ -233,70 +282,107 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) { var absBaseDir string absBaseDir, _ = filepath.Abs(config.BaseDirectory) - walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - // we don't care about directories, or errors, just read everything we can. - if d.IsDir() { - return nil - } - if len(ext) > 2 && p != file { - return nil - } - if strings.HasPrefix(p, ".") { - return nil - } - if len(config.FileFilters) > 0 { - if !slices.Contains(config.FileFilters, p) { - return nil - } - } - - extension := ExtractFileType(p) - var readingErrors []error - abs, _ := filepath.Abs(filepath.Join(config.BaseDirectory, p)) - - var fileData []byte - - switch extension { - case YAML, JSON: - - dirFile, _ := config.DirFS.Open(p) - modTime := time.Now() - stat, _ := dirFile.Stat() - if stat != nil { - modTime = stat.ModTime() - } - fileData, _ = io.ReadAll(dirFile) - log.Debug("collecting JSON/YAML file", "file", abs) - localFiles[abs] = &LocalFile{ - filename: p, - name: filepath.Base(p), - extension: ExtractFileType(p), - data: fileData, - fullPath: abs, - lastModified: modTime, - readingErrors: readingErrors, - } - case UNSUPPORTED: - log.Debug("skipping non JSON/YAML file", "file", abs) - } - return nil - }) - - if walkErr != nil { - return nil, walkErr - } - - return &LocalFS{ - Files: localFiles, + localFS := &LocalFS{ + indexConfig: config.IndexConfig, + fsConfig: config, logger: log, baseDirectory: absBaseDir, entryPointDirectory: config.BaseDirectory, - readingErrors: allErrors, - }, nil + } + + // if a directory filesystem is supplied, use that to walk the directory and pick up everything it finds. + if config.DirFS != nil { + walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // we don't care about directories, or errors, just read everything we can. + if d.IsDir() { + return nil + } + if len(ext) > 2 && p != file { + return nil + } + if strings.HasPrefix(p, ".") { + return nil + } + if len(config.FileFilters) > 0 { + if !slices.Contains(config.FileFilters, p) { + return nil + } + } + _, fErr := localFS.extractFile(p) + return fErr + }) + + if walkErr != nil { + return nil, walkErr + } + } + + localFS.readingErrors = allErrors + return localFS, nil +} + +func (l *LocalFS) extractFile(p string) (*LocalFile, error) { + extension := ExtractFileType(p) + var readingErrors []error + abs := p + config := l.fsConfig + if !filepath.IsAbs(p) { + if config != nil && config.BaseDirectory != "" { + abs, _ = filepath.Abs(filepath.Join(config.BaseDirectory, p)) + } else { + abs, _ = filepath.Abs(p) + } + } + var fileData []byte + + switch extension { + case YAML, JSON: + var file fs.File + var fileError error + if config != nil && config.DirFS != nil { + file, _ = config.DirFS.Open(p) + } else { + file, fileError = os.Open(abs) + } + + // if reading without a directory FS, error out on any error, do not continue. + if fileError != nil { + readingErrors = append(readingErrors, fileError) + return nil, fileError + } + + modTime := time.Now() + stat, _ := file.Stat() + if stat != nil { + modTime = stat.ModTime() + } + fileData, _ = io.ReadAll(file) + if config != nil && config.DirFS != nil { + l.logger.Debug("collecting JSON/YAML file", "file", abs) + } else { + l.logger.Debug("parsing file", "file", abs) + } + lf := &LocalFile{ + filename: p, + name: filepath.Base(p), + extension: ExtractFileType(p), + data: fileData, + fullPath: abs, + lastModified: modTime, + readingErrors: readingErrors, + } + l.Files.Store(abs, lf) + return lf, nil + case UNSUPPORTED: + if config != nil && config.DirFS != nil { + l.logger.Debug("skipping non JSON/YAML file", "file", abs) + } + } + return nil, nil } // NewLocalFS creates a new LocalFS with the supplied base directory. diff --git a/index/rolodex_test.go b/index/rolodex_test.go index 14cff4f..132f80a 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -540,6 +540,98 @@ func test_rolodexDeepRefServer(a, b, c, d, e []byte) *httptest.Server { })) } +func TestRolodex_IndexCircularLookup_PolyItems_LocalLoop_WithFiles_RecursiveLookup(t *testing.T) { + + fourth := `type: "object" +properties: + name: + type: "string" + children: + type: "object"` + + third := `type: "object" +properties: + name: + $ref: "http://the-space-race-is-all-about-space-and-time-dot.com/fourth.yaml"` + + second := `openapi: 3.1.0 +components: + schemas: + CircleTest: + type: "object" + properties: + bing: + $ref: "not_found.yaml" + name: + type: "string" + children: + type: "object" + anyOf: + - $ref: "third.yaml" + required: + - "name" + - "children"` + + first := `openapi: 3.1.0 +components: + schemas: + StartTest: + type: object + required: + - muffins + properties: + muffins: + $ref: "second_n.yaml#/components/schemas/CircleTest"` + + cwd, _ := os.Getwd() + + _ = os.WriteFile("third_n.yaml", []byte(strings.ReplaceAll(third, "$PWD", cwd)), 0644) + _ = os.WriteFile("second_n.yaml", []byte(second), 0644) + _ = os.WriteFile("first_n.yaml", []byte(first), 0644) + _ = os.WriteFile("fourth_n.yaml", []byte(fourth), 0644) + defer os.Remove("first_n.yaml") + defer os.Remove("second_n.yaml") + defer os.Remove("third_n.yaml") + defer os.Remove("fourth_n.yaml") + + baseDir := "." + cf := CreateOpenAPIIndexConfig() + cf.BasePath = baseDir + cf.IgnorePolymorphicCircularReferences = true + + fsCfg := &LocalFSConfig{ + BaseDirectory: baseDir, + IndexConfig: cf, + } + + fileFS, err := NewLocalFSWithConfig(fsCfg) + if err != nil { + t.Fatal(err) + } + + rolodex := NewRolodex(cf) + rolodex.AddLocalFS(baseDir, fileFS) + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(first), &rootNode) + rolodex.SetRootNode(&rootNode) + + srv := test_rolodexDeepRefServer([]byte(first), []byte(second), + []byte(strings.ReplaceAll(third, "$PWD", cwd)), []byte(fourth), nil) + defer srv.Close() + + u, _ := url.Parse(srv.URL) + cf.BaseURL = u + remoteFS, rErr := NewRemoteFSWithConfig(cf) + assert.NoError(t, rErr) + + rolodex.AddRemoteFS(srv.URL, remoteFS) + + err = rolodex.IndexTheRolodex() + assert.Error(t, err) + assert.Len(t, rolodex.GetCaughtErrors(), 2) +} + func TestRolodex_IndexCircularLookup_PolyItems_LocalLoop_WithFiles(t *testing.T) { first := `openapi: 3.1.0 diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 6536998..47859c5 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -261,6 +261,76 @@ func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) { } +func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve_RecursiveLookup(t *testing.T) { + // this is a full checkout of the digitalocean API repo. + tmp, _ := os.MkdirTemp("", "openapi") + cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp) + defer os.RemoveAll(filepath.Join(tmp, "openapi")) + + err := cmd.Run() + if err != nil { + log.Fatalf("cmd.Run() failed with %s\n", err) + } + + spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml")) + doLocal, _ := os.ReadFile(spec) + + var rootNode yaml.Node + _ = yaml.Unmarshal(doLocal, &rootNode) + + basePath := filepath.Join(tmp, "specification") + + // create a new config that allows local and remote to be mixed up. + cf := CreateOpenAPIIndexConfig() + cf.AllowRemoteLookup = true + cf.AvoidCircularReferenceCheck = true + cf.BasePath = basePath + cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelError, + })) + + // create a new rolodex + rolo := NewRolodex(cf) + + // set the rolodex root node to the root node of the spec. + rolo.SetRootNode(&rootNode) + + // configure the local filesystem. + fsCfg := LocalFSConfig{ + BaseDirectory: cf.BasePath, + IndexConfig: cf, + Logger: cf.Logger, + } + + // create a new local filesystem. + fileFS, fsErr := NewLocalFSWithConfig(&fsCfg) + assert.NoError(t, fsErr) + + rolo.AddLocalFS(basePath, fileFS) + + rErr := rolo.IndexTheRolodex() + files := fileFS.GetFiles() + fileLen := len(files) + + assert.Equal(t, 1677, fileLen) + + assert.NoError(t, rErr) + + index := rolo.GetRootIndex() + + assert.NotNil(t, index) + + assert.Len(t, index.GetMappedReferencesSequenced(), 299) + assert.Len(t, index.GetMappedReferences(), 299) + assert.Len(t, fileFS.GetErrors(), 0) + + // check circular references + rolo.CheckForCircularReferences() + assert.Len(t, rolo.GetCaughtErrors(), 0) + assert.Len(t, rolo.GetIgnoredCircularReferences(), 0) + +} + func TestSpecIndex_DigitalOcean_LookupsNotAllowed(t *testing.T) { do, _ := os.ReadFile("../test_specs/digitalocean.yaml") var rootNode yaml.Node