Defaulting localFS to be recursive

currently it pre-indexes everything in the root. This behavior is undesirable out of the box, so it now looks recursively by default.

Signed-off-by: quobix <dave@quobix.com>

datamodel/low/v2/swagger.go

@@ -19,7 +19,6 @@ import (
 	"github.com/pb33f/libopenapi/datamodel/low/base"
 	"github.com/pb33f/libopenapi/index"
 	"gopkg.in/yaml.v3"
-	"os"
 	"path/filepath"
 )
 
@@ -163,7 +162,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 			// create a local filesystem
 			localFSConf := index.LocalFSConfig{
 				BaseDirectory: cwd,
-				DirFS:         os.DirFS(cwd),
+				IndexConfig:   idxConfig,
 				FileFilters:   config.FileFilter,
 			}
 			fileFS, err := index.NewLocalFSWithConfig(&localFSConf)

datamodel/low/v2/swagger_test.go

@@ -406,7 +406,7 @@ func TestRolodexLocalFileSystem_BadPath(t *testing.T) {
 	cf.BasePath = "/NOWHERE"
 	cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
 	lDoc, err := CreateDocumentFromConfig(info, cf)
-	assert.Nil(t, lDoc)
+	assert.NotNil(t, lDoc)
 	assert.Error(t, err)
 }
 

datamodel/low/v3/create_document.go

@@ -3,7 +3,6 @@ package v3
 import (
 	"context"
 	"errors"
-	"os"
 	"path/filepath"
 	"sync"
 
@@ -61,7 +60,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 			// create a local filesystem
 			localFSConf := index.LocalFSConfig{
 				BaseDirectory: cwd,
-				DirFS:         os.DirFS(cwd),
+				IndexConfig:   idxConfig,
 				FileFilters:   config.FileFilter,
 			}
 

datamodel/low/v3/create_document_test.go

@@ -114,7 +114,7 @@ func TestRolodexLocalFileSystem_BadPath(t *testing.T) {
 	cf.BasePath = "/NOWHERE"
 	cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
 	lDoc, err := CreateDocumentFromConfig(info, cf)
-	assert.Nil(t, lDoc)
+	assert.NotNil(t, lDoc)
 	assert.Error(t, err)
 }
 

index/find_component_test.go

@@ -71,6 +71,49 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) {
 	assert.Nil(t, c)
 }
 
+func TestSpecIndex_CheckCircularIndex_NoDirFS(t *testing.T) {
+
+	cFile := "../test_specs/first.yaml"
+	yml, _ := os.ReadFile(cFile)
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(yml), &rootNode)
+
+	cf := CreateOpenAPIIndexConfig()
+	cf.AvoidCircularReferenceCheck = true
+	cf.BasePath = "../test_specs"
+
+	rolo := NewRolodex(cf)
+	rolo.SetRootNode(&rootNode)
+	cf.Rolodex = rolo
+
+	fsCfg := LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		IndexConfig:   cf,
+	}
+
+	fileFS, err := NewLocalFSWithConfig(&fsCfg)
+
+	assert.NoError(t, err)
+	rolo.AddLocalFS(cf.BasePath, fileFS)
+
+	indexedErr := rolo.IndexTheRolodex()
+	rolo.BuildIndexes()
+
+	assert.NoError(t, indexedErr)
+
+	index := rolo.GetRootIndex()
+
+	assert.Nil(t, index.uri)
+
+	a, _ := index.SearchIndexForReference("second.yaml#/properties/property2")
+	b, _ := index.SearchIndexForReference("second.yaml")
+	c, _ := index.SearchIndexForReference("fourth.yaml")
+
+	assert.NotNil(t, a)
+	assert.NotNil(t, b)
+	assert.Nil(t, c)
+}
+
 func TestFindComponent_RolodexFileParseError(t *testing.T) {
 
 	badData := "I cannot be parsed: \"I am not a YAML file or a JSON file"

index/rolodex_file_loader.go

@@ -6,6 +6,7 @@ package index
 import (
 	"fmt"
 	"github.com/pb33f/libopenapi/datamodel"
+	"golang.org/x/sync/syncmap"
 	"gopkg.in/yaml.v3"
 	"io"
 	"io/fs"
@@ -14,22 +15,32 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 	"time"
 )
 
 // LocalFS is a file system that indexes local files.
 type LocalFS struct {
+	fsConfig            *LocalFSConfig
 	indexConfig         *SpecIndexConfig
 	entryPointDirectory string
 	baseDirectory       string
-	Files               map[string]RolodexFile
+	Files               syncmap.Map
+	extractedFiles      map[string]RolodexFile
 	logger              *slog.Logger
+	fileLock            sync.Mutex
 	readingErrors       []error
 }
 
 // GetFiles returns the files that have been indexed. A map of RolodexFile objects keyed by the full path of the file.
 func (l *LocalFS) GetFiles() map[string]RolodexFile {
-	return l.Files
+	files := make(map[string]RolodexFile)
+	l.Files.Range(func(key, value interface{}) bool {
+		files[key.(string)] = value.(*LocalFile)
+		return true
+	})
+	l.extractedFiles = files
+	return files
 }
 
 // GetErrors returns any errors that occurred during the indexing process.
@@ -50,11 +61,47 @@ func (l *LocalFS) Open(name string) (fs.File, error) {
 		name, _ = filepath.Abs(filepath.Join(l.baseDirectory, name))
 	}
 
-	if f, ok := l.Files[name]; ok {
+	if f, ok := l.Files.Load(name); ok {
 		return f.(*LocalFile), nil
 	} else {
-		return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
+
+		if l.fsConfig != nil && l.fsConfig.DirFS == nil {
+			var extractedFile *LocalFile
+			var extErr error
+			// attempt to open the file from the local filesystem
+			extractedFile, extErr = l.extractFile(name)
+			if extErr != nil {
+				return nil, extErr
+			}
+			if extractedFile != nil {
+
+				// in this mode, we need the index config to be set.
+				if l.indexConfig != nil {
+					copiedCfg := *l.indexConfig
+					copiedCfg.SpecAbsolutePath = name
+					copiedCfg.AvoidBuildIndex = true
+
+					idx, idxError := extractedFile.Index(&copiedCfg)
+
+					if idxError != nil && idx == nil {
+						l.readingErrors = append(l.readingErrors, idxError)
+					} else {
+
+						// for each index, we need a resolver
+						resolver := NewResolver(idx)
+						idx.resolver = resolver
+						idx.BuildIndex()
+					}
+
+					if len(extractedFile.data) > 0 {
+						l.logger.Debug("successfully loaded and indexed file", "file", name)
+					}
+					return extractedFile, nil
+				}
+			}
+		}
 	}
+	return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
 }
 
 // LocalFile is a file that has been indexed by the LocalFS. It implements the RolodexFile interface.
@@ -212,11 +259,13 @@ type LocalFSConfig struct {
 
 	// supply a custom fs.FS to use
 	DirFS fs.FS
+
+	// supply an index configuration to use
+	IndexConfig *SpecIndexConfig
 }
 
 // NewLocalFSWithConfig creates a new LocalFS with the supplied configuration.
 func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) {
-	localFiles := make(map[string]RolodexFile)
 	var allErrors []error
 
 	log := config.Logger
@@ -233,70 +282,107 @@ func NewLocalFSWithConfig(config *LocalFSConfig) (*LocalFS, error) {
 	var absBaseDir string
 	absBaseDir, _ = filepath.Abs(config.BaseDirectory)
 
-	walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// we don't care about directories, or errors, just read everything we can.
-		if d.IsDir() {
-			return nil
-		}
-		if len(ext) > 2 && p != file {
-			return nil
-		}
-		if strings.HasPrefix(p, ".") {
-			return nil
-		}
-		if len(config.FileFilters) > 0 {
-			if !slices.Contains(config.FileFilters, p) {
-				return nil
-			}
-		}
-
-		extension := ExtractFileType(p)
-		var readingErrors []error
-		abs, _ := filepath.Abs(filepath.Join(config.BaseDirectory, p))
-
-		var fileData []byte
-
-		switch extension {
-		case YAML, JSON:
-
-			dirFile, _ := config.DirFS.Open(p)
-			modTime := time.Now()
-			stat, _ := dirFile.Stat()
-			if stat != nil {
-				modTime = stat.ModTime()
-			}
-			fileData, _ = io.ReadAll(dirFile)
-			log.Debug("collecting JSON/YAML file", "file", abs)
-			localFiles[abs] = &LocalFile{
-				filename:      p,
-				name:          filepath.Base(p),
-				extension:     ExtractFileType(p),
-				data:          fileData,
-				fullPath:      abs,
-				lastModified:  modTime,
-				readingErrors: readingErrors,
-			}
-		case UNSUPPORTED:
-			log.Debug("skipping non JSON/YAML file", "file", abs)
-		}
-		return nil
-	})
-
-	if walkErr != nil {
-		return nil, walkErr
-	}
-
-	return &LocalFS{
-		Files:               localFiles,
+	localFS := &LocalFS{
+		indexConfig:         config.IndexConfig,
+		fsConfig:            config,
 		logger:              log,
 		baseDirectory:       absBaseDir,
 		entryPointDirectory: config.BaseDirectory,
-		readingErrors:       allErrors,
-	}, nil
+	}
+
+	// if a directory filesystem is supplied, use that to walk the directory and pick up everything it finds.
+	if config.DirFS != nil {
+		walkErr := fs.WalkDir(config.DirFS, ".", func(p string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			// we don't care about directories, or errors, just read everything we can.
+			if d.IsDir() {
+				return nil
+			}
+			if len(ext) > 2 && p != file {
+				return nil
+			}
+			if strings.HasPrefix(p, ".") {
+				return nil
+			}
+			if len(config.FileFilters) > 0 {
+				if !slices.Contains(config.FileFilters, p) {
+					return nil
+				}
+			}
+			_, fErr := localFS.extractFile(p)
+			return fErr
+		})
+
+		if walkErr != nil {
+			return nil, walkErr
+		}
+	}
+
+	localFS.readingErrors = allErrors
+	return localFS, nil
+}
+
+func (l *LocalFS) extractFile(p string) (*LocalFile, error) {
+	extension := ExtractFileType(p)
+	var readingErrors []error
+	abs := p
+	config := l.fsConfig
+	if !filepath.IsAbs(p) {
+		if config != nil && config.BaseDirectory != "" {
+			abs, _ = filepath.Abs(filepath.Join(config.BaseDirectory, p))
+		} else {
+			abs, _ = filepath.Abs(p)
+		}
+	}
+	var fileData []byte
+
+	switch extension {
+	case YAML, JSON:
+		var file fs.File
+		var fileError error
+		if config != nil && config.DirFS != nil {
+			file, _ = config.DirFS.Open(p)
+		} else {
+			file, fileError = os.Open(abs)
+		}
+
+		// if reading without a directory FS, error out on any error, do not continue.
+		if fileError != nil {
+			readingErrors = append(readingErrors, fileError)
+			return nil, fileError
+		}
+
+		modTime := time.Now()
+		stat, _ := file.Stat()
+		if stat != nil {
+			modTime = stat.ModTime()
+		}
+		fileData, _ = io.ReadAll(file)
+		if config != nil && config.DirFS != nil {
+			l.logger.Debug("collecting JSON/YAML file", "file", abs)
+		} else {
+			l.logger.Debug("parsing file", "file", abs)
+		}
+		lf := &LocalFile{
+			filename:      p,
+			name:          filepath.Base(p),
+			extension:     ExtractFileType(p),
+			data:          fileData,
+			fullPath:      abs,
+			lastModified:  modTime,
+			readingErrors: readingErrors,
+		}
+		l.Files.Store(abs, lf)
+		return lf, nil
+	case UNSUPPORTED:
+		if config != nil && config.DirFS != nil {
+			l.logger.Debug("skipping non JSON/YAML file", "file", abs)
+		}
+	}
+	return nil, nil
 }
 
 // NewLocalFS creates a new LocalFS with the supplied base directory.

index/rolodex_test.go

@@ -540,6 +540,98 @@ func test_rolodexDeepRefServer(a, b, c, d, e []byte) *httptest.Server {
 	}))
 }
 
+func TestRolodex_IndexCircularLookup_PolyItems_LocalLoop_WithFiles_RecursiveLookup(t *testing.T) {
+
+	fourth := `type: "object"
+properties:
+  name:
+    type: "string"
+  children:
+    type: "object"`
+
+	third := `type: "object"
+properties:
+  name:
+    $ref: "http://the-space-race-is-all-about-space-and-time-dot.com/fourth.yaml"`
+
+	second := `openapi: 3.1.0
+components:
+  schemas:
+    CircleTest:
+      type: "object"
+      properties:
+        bing:
+          $ref: "not_found.yaml"
+        name:
+          type: "string"
+        children:
+          type: "object"
+          anyOf:
+            - $ref: "third.yaml"
+      required:
+        - "name"
+        - "children"`
+
+	first := `openapi: 3.1.0
+components:
+  schemas:
+    StartTest:
+      type: object
+      required:
+        - muffins
+      properties:
+        muffins:
+         $ref: "second_n.yaml#/components/schemas/CircleTest"`
+
+	cwd, _ := os.Getwd()
+
+	_ = os.WriteFile("third_n.yaml", []byte(strings.ReplaceAll(third, "$PWD", cwd)), 0644)
+	_ = os.WriteFile("second_n.yaml", []byte(second), 0644)
+	_ = os.WriteFile("first_n.yaml", []byte(first), 0644)
+	_ = os.WriteFile("fourth_n.yaml", []byte(fourth), 0644)
+	defer os.Remove("first_n.yaml")
+	defer os.Remove("second_n.yaml")
+	defer os.Remove("third_n.yaml")
+	defer os.Remove("fourth_n.yaml")
+
+	baseDir := "."
+	cf := CreateOpenAPIIndexConfig()
+	cf.BasePath = baseDir
+	cf.IgnorePolymorphicCircularReferences = true
+
+	fsCfg := &LocalFSConfig{
+		BaseDirectory: baseDir,
+		IndexConfig:   cf,
+	}
+
+	fileFS, err := NewLocalFSWithConfig(fsCfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rolodex := NewRolodex(cf)
+	rolodex.AddLocalFS(baseDir, fileFS)
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(first), &rootNode)
+	rolodex.SetRootNode(&rootNode)
+
+	srv := test_rolodexDeepRefServer([]byte(first), []byte(second),
+		[]byte(strings.ReplaceAll(third, "$PWD", cwd)), []byte(fourth), nil)
+	defer srv.Close()
+
+	u, _ := url.Parse(srv.URL)
+	cf.BaseURL = u
+	remoteFS, rErr := NewRemoteFSWithConfig(cf)
+	assert.NoError(t, rErr)
+
+	rolodex.AddRemoteFS(srv.URL, remoteFS)
+
+	err = rolodex.IndexTheRolodex()
+	assert.Error(t, err)
+	assert.Len(t, rolodex.GetCaughtErrors(), 2)
+}
+
 func TestRolodex_IndexCircularLookup_PolyItems_LocalLoop_WithFiles(t *testing.T) {
 
 	first := `openapi: 3.1.0

index/spec_index_test.go

@@ -261,6 +261,76 @@ func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) {
 
 }
 
+func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve_RecursiveLookup(t *testing.T) {
+	// this is a full checkout of the digitalocean API repo.
+	tmp, _ := os.MkdirTemp("", "openapi")
+	cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp)
+	defer os.RemoveAll(filepath.Join(tmp, "openapi"))
+
+	err := cmd.Run()
+	if err != nil {
+		log.Fatalf("cmd.Run() failed with %s\n", err)
+	}
+
+	spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml"))
+	doLocal, _ := os.ReadFile(spec)
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal(doLocal, &rootNode)
+
+	basePath := filepath.Join(tmp, "specification")
+
+	// create a new config that allows local and remote to be mixed up.
+	cf := CreateOpenAPIIndexConfig()
+	cf.AllowRemoteLookup = true
+	cf.AvoidCircularReferenceCheck = true
+	cf.BasePath = basePath
+	cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelError,
+	}))
+
+	// create a new rolodex
+	rolo := NewRolodex(cf)
+
+	// set the rolodex root node to the root node of the spec.
+	rolo.SetRootNode(&rootNode)
+
+	// configure the local filesystem.
+	fsCfg := LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		IndexConfig:   cf,
+		Logger:        cf.Logger,
+	}
+
+	// create a new local filesystem.
+	fileFS, fsErr := NewLocalFSWithConfig(&fsCfg)
+	assert.NoError(t, fsErr)
+
+	rolo.AddLocalFS(basePath, fileFS)
+
+	rErr := rolo.IndexTheRolodex()
+	files := fileFS.GetFiles()
+	fileLen := len(files)
+
+	assert.Equal(t, 1677, fileLen)
+
+	assert.NoError(t, rErr)
+
+	index := rolo.GetRootIndex()
+
+	assert.NotNil(t, index)
+
+	assert.Len(t, index.GetMappedReferencesSequenced(), 299)
+	assert.Len(t, index.GetMappedReferences(), 299)
+	assert.Len(t, fileFS.GetErrors(), 0)
+
+	// check circular references
+	rolo.CheckForCircularReferences()
+	assert.Len(t, rolo.GetCaughtErrors(), 0)
+	assert.Len(t, rolo.GetIgnoredCircularReferences(), 0)
+
+}
+
 func TestSpecIndex_DigitalOcean_LookupsNotAllowed(t *testing.T) {
 	do, _ := os.ReadFile("../test_specs/digitalocean.yaml")
 	var rootNode yaml.Node