bashing through usecases and updating tests as we go.

so many things that can go wrong. have to catch them all.

Signed-off-by: quobix <dave@quobix.com>

index/extract_refs.go

@@ -189,6 +189,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
 				var p string
 				uri := strings.Split(value, "#/")
+
 				if strings.HasPrefix(value, "http") || filepath.IsAbs(value) {
 					if len(uri) == 2 {
 						_, p = utils.ConvertComponentIdIntoFriendlyPathSearch(fmt.Sprintf("#/%s", uri[1]))

index/find_component.go

@@ -62,7 +62,16 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
 		}
 	} else {
 		if !strings.Contains(componentId, "#") {
-			return index.lookupRolodex(uri)
+
+			// does it contain a file extension?
+			fileExt := filepath.Ext(componentId)
+			if fileExt != "" {
+				return index.lookupRolodex(uri)
+			}
+
+			// root search
+			return index.FindComponentInRoot(componentId)
+
 		}
 		return index.FindComponentInRoot(fmt.Sprintf("#/%s", uri[0]))
 	}
@@ -368,6 +377,7 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
 			if index.specAbsolutePath != "" {
 				if index.config.BaseURL != nil {
 
+					// consider the file remote.
 					//if strings.Contains(file, "../../") {
 
 					// extract the base path from the specAbsolutePath for this index.
@@ -384,8 +394,11 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference {
 					//absoluteFileLocation = loc
 
 				} else {
-					panic("nooooooo")
-					absoluteFileLocation, _ = filepath.Abs(filepath.Join(index.config.BaseURL.Path, file))
+
+					// consider the file local
+
+					dir := filepath.Dir(index.config.SpecAbsolutePath)
+					absoluteFileLocation, _ = filepath.Abs(filepath.Join(dir, file))
 				}
 			} else {
 				absoluteFileLocation = file

index/find_component_test.go

@@ -43,7 +43,7 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) {
 	_ = yaml.Unmarshal([]byte(yml), &rootNode)
 
 	cf := CreateOpenAPIIndexConfig()
-	cf.AvoidBuildIndex = true
+	cf.AvoidCircularReferenceCheck = true
 	cf.BasePath = "../test_specs"
 
 	rolo := NewRolodex(cf)

index/index_model.go

@@ -268,17 +268,17 @@ type SpecIndex struct {
 	enumCount                           int
 	descriptionCount                    int
 	summaryCount                        int
-	seenRemoteSources                   map[string]*yaml.Node
-	seenLocalSources                    map[string]*yaml.Node
-	refLock                             sync.Mutex
-	componentLock                       sync.RWMutex
-	errorLock                           sync.RWMutex
-	circularReferences                  []*CircularReferenceResult // only available when the resolver has been used.
-	allowCircularReferences             bool                       // decide if you want to error out, or allow circular references, default is false.
-	config                              *SpecIndexConfig           // configuration for the index
-	httpClient                          *http.Client
-	componentIndexChan                  chan bool
-	polyComponentIndexChan              chan bool
+	//seenRemoteSources                   map[string]*yaml.Node
+	//seenLocalSources                    map[string]*yaml.Node
+	refLock                 sync.Mutex
+	componentLock           sync.RWMutex
+	errorLock               sync.RWMutex
+	circularReferences      []*CircularReferenceResult // only available when the resolver has been used.
+	allowCircularReferences bool                       // decide if you want to error out, or allow circular references, default is false.
+	config                  *SpecIndexConfig           // configuration for the index
+	httpClient              *http.Client
+	componentIndexChan      chan bool
+	polyComponentIndexChan  chan bool
 
 	specAbsolutePath string
 	resolver         *Resolver

index/index_utils.go

@@ -82,8 +82,8 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) {
 	index.securityRequirementRefs = make(map[string]map[string][]*Reference)
 	index.polymorphicRefs = make(map[string]*Reference)
 	index.refsWithSiblings = make(map[string]Reference)
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	index.seenLocalSources = make(map[string]*yaml.Node)
+	//index.seenRemoteSources = make(map[string]*yaml.Node)
+	//index.seenLocalSources = make(map[string]*yaml.Node)
 	index.opServersRefs = make(map[string]map[string][]*Reference)
 	index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second}
 	index.componentIndexChan = make(chan bool)

index/rolodex.go

@@ -258,10 +258,6 @@ func (r *Rolodex) IndexTheRolodex() error {
 		return nil
 	}
 
-	// disable index building, it will need to be run after the rolodex indexed
-	// at a high level.
-	r.indexConfig.AvoidBuildIndex = true
-
 	var caughtErrors []error
 
 	var indexBuildQueue []*SpecIndex
@@ -373,6 +369,19 @@ func (r *Rolodex) IndexTheRolodex() error {
 
 	if r.rootNode != nil {
 
+		// if there is a base path, then we need to set the root spec config to point to a theoretical root.yaml
+		// which does not exist, but is used to formulate the absolute path to root references correctly.
+		if r.indexConfig.BasePath != "" && r.indexConfig.BaseURL == nil {
+
+			basePath := r.indexConfig.BasePath
+			if !filepath.IsAbs(basePath) {
+				basePath, _ = filepath.Abs(basePath)
+			}
+			r.indexConfig.SpecAbsolutePath = filepath.Join(basePath, "root.yaml")
+		}
+
+		// todo: variation with no base path, but a base URL.
+
 		index := NewSpecIndexWithConfig(r.rootNode, r.indexConfig)
 		resolver := NewResolver(index)
 		if r.indexConfig.IgnoreArrayCircularReferences {
@@ -382,9 +391,7 @@ func (r *Rolodex) IndexTheRolodex() error {
 			resolver.IgnorePolymorphicCircularReferences()
 		}
 
-		if !r.indexConfig.AvoidBuildIndex {
-			index.BuildIndex()
-		}
+		index.BuildIndex()
 
 		if !r.indexConfig.AvoidCircularReferenceCheck {
 			resolvingErrors := resolver.CheckForCircularReferences()
@@ -393,10 +400,14 @@ func (r *Rolodex) IndexTheRolodex() error {
 			}
 		}
 		r.rootIndex = index
+		if len(index.refErrors) > 0 {
+			caughtErrors = append(caughtErrors, index.refErrors...)
+		}
 	}
 	r.indexingDuration = time.Since(started)
 	r.indexed = true
 	r.caughtErrors = caughtErrors
+	r.built = true
 	return errors.Join(caughtErrors...)
 
 }

index/rolodex_file_loader.go

@@ -17,6 +17,7 @@ import (
 )
 
 type LocalFS struct {
+	indexConfig         *SpecIndexConfig
 	entryPointDirectory string
 	baseDirectory       string
 	Files               map[string]RolodexFile
@@ -28,7 +29,18 @@ func (l *LocalFS) GetFiles() map[string]RolodexFile {
 	return l.Files
 }
 
+func (l *LocalFS) GetErrors() []error {
+	return l.readingErrors
+}
+
 func (l *LocalFS) Open(name string) (fs.File, error) {
+
+	if l.indexConfig != nil && !l.indexConfig.AllowFileLookup {
+		return nil, &fs.PathError{Op: "open", Path: name,
+			Err: fmt.Errorf("file lookup for '%s' not allowed, set the index configuration "+
+				"to AllowFileLookup to be true", name)}
+	}
+
 	if !filepath.IsAbs(name) {
 		var absErr error
 		name, absErr = filepath.Abs(filepath.Join(l.baseDirectory, name))

index/rolodex_remote_loader.go

@@ -27,6 +27,7 @@ type RemoteFS struct {
 	rootURLParsed     *url.URL
 	RemoteHandlerFunc RemoteURLHandler
 	Files             syncmap.Map
+	ProcessingFiles   syncmap.Map
 	FetchTime         int64
 	FetchChannel      chan *RemoteFile
 	remoteWg          sync.WaitGroup
@@ -235,6 +236,10 @@ func (i *RemoteFS) GetFiles() map[string]RolodexFile {
 	return files
 }
 
+func (i *RemoteFS) GetErrors() []error {
+	return i.remoteErrors
+}
+
 func (i *RemoteFS) seekRelatives(file *RemoteFile) {
 
 	extractedRefs := ExtractRefs(string(file.data))
@@ -288,6 +293,11 @@ func (i *RemoteFS) seekRelatives(file *RemoteFile) {
 
 func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 
+	if i.indexConfig != nil && !i.indexConfig.AllowRemoteLookup {
+		return nil, fmt.Errorf("remote lookup for '%s' is not allowed, please set "+
+			"AllowRemoteLookup to true as part of the index configuration", remoteURL)
+	}
+
 	remoteParsedURL, err := url.Parse(remoteURL)
 	if err != nil {
 		return nil, err
@@ -298,6 +308,20 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 		return r.(*RemoteFile), nil
 	}
 
+	// if we're processing, we need to block and wait for the file to be processed
+	// try path first
+	if _, ok := i.ProcessingFiles.Load(remoteParsedURL.Path); ok {
+		i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String())
+		for {
+			if wf, ko := i.Files.Load(remoteParsedURL.Path); ko {
+				return wf.(*RemoteFile), nil
+			}
+		}
+	}
+
+	// add to processing
+	i.ProcessingFiles.Store(remoteParsedURL.Path, true)
+
 	fileExt := ExtractFileType(remoteParsedURL.Path)
 
 	if fileExt == UNSUPPORTED {
@@ -314,7 +338,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 		}
 	}
 
-	i.logger.Debug("Loading remote file", "file", remoteURL, "remoteURL", remoteParsedURL.String())
+	i.logger.Debug("loading remote file", "file", remoteURL, "remoteURL", remoteParsedURL.String())
 
 	// no handler func? use the default client.
 	if i.RemoteHandlerFunc == nil {
@@ -323,20 +347,32 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 
 	response, clientErr := i.RemoteHandlerFunc(remoteParsedURL.String())
 	if clientErr != nil {
+
+		i.remoteErrors = append(i.remoteErrors, clientErr)
+		// remove from processing
+		i.ProcessingFiles.Delete(remoteParsedURL.Path)
 		if response != nil {
 			i.logger.Error("client error", "error", clientErr, "status", response.StatusCode)
 		} else {
-			i.logger.Error("no response for request", "error", clientErr.Error())
+			i.logger.Error("client error, empty body", "error", clientErr.Error())
 		}
 		return nil, clientErr
 	}
 
 	responseBytes, readError := io.ReadAll(response.Body)
 	if readError != nil {
+
+		// remove from processing
+		i.ProcessingFiles.Delete(remoteParsedURL.Path)
+
 		return nil, readError
 	}
 
 	if response.StatusCode >= 400 {
+
+		// remove from processing
+		i.ProcessingFiles.Delete(remoteParsedURL.Path)
+
 		i.logger.Error("Unable to fetch remote document",
 			"file", remoteParsedURL.Path, "status", response.StatusCode, "resp", string(responseBytes))
 		return nil, fmt.Errorf("unable to fetch remote document: %s", string(responseBytes))
@@ -344,6 +380,8 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 
 	absolutePath, pathErr := filepath.Abs(remoteParsedURL.Path)
 	if pathErr != nil {
+		// remove from processing
+		i.ProcessingFiles.Delete(remoteParsedURL.Path)
 		return nil, pathErr
 	}
 
@@ -394,11 +432,16 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 
 	i.Files.Store(absolutePath, remoteFile)
 
-	i.logger.Debug("successfully loaded file", "file", absolutePath)
+	if len(remoteFile.data) > 0 {
+		i.logger.Debug("successfully loaded file", "file", absolutePath)
+	}
 	i.seekRelatives(remoteFile)
 
 	idx.BuildIndex()
 
+	// remove from processing
+	i.ProcessingFiles.Delete(remoteParsedURL.Path)
+
 	if !i.remoteRunning {
 		return remoteFile, errors.Join(i.remoteErrors...)
 	} else {

index/search_index.go

@@ -24,12 +24,27 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
 			if strings.HasPrefix(uri[0], "http") {
 				roloLookup = fullRef.FullDefinition
 			} else {
-				roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0]))
+				if filepath.IsAbs(uri[0]) {
+					roloLookup = uri[0]
+				} else {
+					if filepath.Ext(absPath) != "" {
+						absPath = filepath.Dir(absPath)
+					}
+					roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0]))
+				}
 			}
 		}
 		ref = fmt.Sprintf("#/%s", uri[1])
 	} else {
-		roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0]))
+		if filepath.IsAbs(uri[0]) {
+			roloLookup = uri[0]
+		} else {
+			if filepath.Ext(absPath) != "" {
+				absPath = filepath.Dir(absPath)
+			}
+			roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0]))
+		}
+
 		ref = uri[0]
 	}
 
@@ -65,6 +80,15 @@ func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *
 					return s
 				}
 			}
+
+			// does component exist in the root?
+			node, _ := rFile.GetContentAsYAMLNode()
+			if node != nil {
+				found := idx.FindComponent(ref, node)
+				if found != nil {
+					return found
+				}
+			}
 		}
 	}
 

index/spec_index.go

@@ -430,10 +430,10 @@ func (index *SpecIndex) GetAllOperationsServers() map[string]map[string][]*Refer
 	return index.opServersRefs
 }
 
-// GetAllExternalIndexes will return all indexes for external documents
-func (index *SpecIndex) GetAllExternalIndexes() map[string]*SpecIndex {
-	return index.externalSpecIndex
-}
+//// GetAllExternalIndexes will return all indexes for external documents
+//func (index *SpecIndex) GetAllExternalIndexes() map[string]*SpecIndex {
+//	return index.externalSpecIndex
+//}
 
 // SetAllowCircularReferenceResolving will flip a bit that can be used by any consumers to determine if they want
 // to allow or disallow circular references to be resolved or visited

index/spec_index_test.go

@@ -5,6 +5,7 @@ package index
 
 import (
 	"fmt"
+	"github.com/pb33f/libopenapi/utils"
 	"log"
 	"log/slog"
 	"net/http"
@@ -64,7 +65,6 @@ func TestSpecIndex_ExtractRefsStripe(t *testing.T) {
 	assert.Len(t, index.GetRefsByLine(), 537)
 	assert.Len(t, index.GetLinesWithReferences(), 1972)
 	assert.Len(t, index.GetAllExternalDocuments(), 0)
-	assert.Len(t, index.GetAllExternalIndexes(), 0)
 }
 
 func TestSpecIndex_Asana(t *testing.T) {
@@ -97,7 +97,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
 	location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification"
 	baseURL, _ := url.Parse(location)
 
-	// create a new config that allows local and remote to be mixed up.
+	// create a new config that allows remote lookups.
 	cf := &SpecIndexConfig{}
 	cf.AvoidBuildIndex = true
 	cf.AllowRemoteLookup = true
@@ -121,7 +121,6 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
 	// create a handler that uses an env variable to capture any GITHUB_TOKEN in the OS ENV
 	// and inject it into the request header, so this does not fail when running lots of local tests.
 	if os.Getenv("GITHUB_TOKEN") != "" {
-
 		client := &http.Client{
 			Timeout: time.Second * 60,
 		}
@@ -130,7 +129,6 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
 			request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", os.Getenv("GITHUB_TOKEN")))
 			return client.Do(request)
 		})
-
 	}
 
 	// add remote filesystem
@@ -143,19 +141,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
 	files := remoteFS.GetFiles()
 	fileLen := len(files)
 	assert.Equal(t, 1646, fileLen)
-
-	//
-	//
-	//
-	//
-	//
-	//assert.NoError(t, indexedErr)
-	//
-	//index := rolo.GetRootIndex()
-	//rolo.CheckForCircularReferences()
-	//
-	//assert.Len(t, index.GetAllExternalIndexes(), 291)
-	//assert.NotNil(t, index)
+	assert.Len(t, remoteFS.GetErrors(), 0)
 }
 
 func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) {
@@ -163,69 +149,153 @@ func TestSpecIndex_DigitalOcean_FullCheckoutLocalResolve(t *testing.T) {
 	tmp, _ := os.MkdirTemp("", "openapi")
 	cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp)
 	defer os.RemoveAll(filepath.Join(tmp, "openapi"))
+
 	err := cmd.Run()
 	if err != nil {
 		log.Fatalf("cmd.Run() failed with %s\n", err)
 	}
+
 	spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml"))
 	doLocal, _ := os.ReadFile(spec)
+
 	var rootNode yaml.Node
 	_ = yaml.Unmarshal(doLocal, &rootNode)
 
-	config := CreateOpenAPIIndexConfig()
-	config.BasePath = filepath.Join(tmp, "specification")
+	basePath := filepath.Join(tmp, "specification")
 
-	index := NewSpecIndexWithConfig(&rootNode, config)
+	// create a new config that allows local and remote to be mixed up.
+	cf := CreateOpenAPIIndexConfig()
+	cf.AvoidBuildIndex = true
+	cf.AllowRemoteLookup = true
+	cf.AvoidCircularReferenceCheck = true
+	cf.BasePath = basePath
+
+	// create a new rolodex
+	rolo := NewRolodex(cf)
+
+	// set the rolodex root node to the root node of the spec.
+	rolo.SetRootNode(&rootNode)
+
+	// configure the local filesystem.
+	fsCfg := LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		DirFS:         os.DirFS(cf.BasePath),
+	}
+
+	// create a new local filesystem.
+	fileFS, fsErr := NewLocalFSWithConfig(&fsCfg)
+	assert.NoError(t, fsErr)
+
+	files := fileFS.GetFiles()
+	fileLen := len(files)
+
+	assert.Equal(t, 1684, fileLen)
+
+	rolo.AddLocalFS(basePath, fileFS)
+
+	rErr := rolo.IndexTheRolodex()
+
+	assert.NoError(t, rErr)
+
+	index := rolo.GetRootIndex()
 
 	assert.NotNil(t, index)
-	assert.Len(t, index.GetAllExternalIndexes(), 296)
 
-	ref := index.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml")
-	assert.NotNil(t, ref)
-	assert.Equal(t, "operationId", ref.Node.Content[0].Value)
-
-	ref = index.SearchIndexForReference("examples/ruby/domains_create.yml")
-	assert.NotNil(t, ref)
-	assert.Equal(t, "lang", ref.Node.Content[0].Value)
-
-	ref = index.SearchIndexForReference("../../shared/responses/server_error.yml")
-	assert.NotNil(t, ref)
-	assert.Equal(t, "description", ref.Node.Content[0].Value)
-
-	ref = index.SearchIndexForReference("../models/options.yml")
-	assert.NotNil(t, ref)
+	assert.Len(t, index.GetMappedReferencesSequenced(), 296)
+	assert.Len(t, index.GetMappedReferences(), 296)
+	assert.Len(t, fileFS.GetErrors(), 0)
 }
 
 func TestSpecIndex_DigitalOcean_LookupsNotAllowed(t *testing.T) {
-	asana, _ := os.ReadFile("../test_specs/digitalocean.yaml")
+	do, _ := os.ReadFile("../test_specs/digitalocean.yaml")
 	var rootNode yaml.Node
-	_ = yaml.Unmarshal(asana, &rootNode)
+	_ = yaml.Unmarshal(do, &rootNode)
 
-	baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification")
-	index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
-		BaseURL: baseURL,
-	})
+	location := "https://raw.githubusercontent.com/digitalocean/openapi/main/specification"
+	baseURL, _ := url.Parse(location)
+
+	// create a new config that does not allow remote lookups.
+	cf := &SpecIndexConfig{}
+	cf.AvoidBuildIndex = true
+	cf.AvoidCircularReferenceCheck = true
+	cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelError,
+	}))
+
+	// setting this baseURL will override the base
+	cf.BaseURL = baseURL
+
+	// create a new rolodex
+	rolo := NewRolodex(cf)
+
+	// set the rolodex root node to the root node of the spec.
+	rolo.SetRootNode(&rootNode)
+
+	// create a new remote fs and set the config for indexing.
+	remoteFS, _ := NewRemoteFSWithConfig(cf)
+
+	// add remote filesystem
+	rolo.AddRemoteFS(location, remoteFS)
+
+	// index the rolodex.
+	indexedErr := rolo.IndexTheRolodex()
+	assert.Error(t, indexedErr)
+	assert.Len(t, utils.UnwrapErrors(indexedErr), 291)
+
+	index := rolo.GetRootIndex()
+
+	files := remoteFS.GetFiles()
+	fileLen := len(files)
+	assert.Equal(t, 0, fileLen)
+	assert.Len(t, remoteFS.GetErrors(), 0)
 
 	// no lookups allowed, bits have not been set, so there should just be a bunch of errors.
-	assert.Len(t, index.GetAllExternalIndexes(), 0)
 	assert.True(t, len(index.GetReferenceIndexErrors()) > 0)
 }
 
 func TestSpecIndex_BaseURLError(t *testing.T) {
-	asana, _ := os.ReadFile("../test_specs/digitalocean.yaml")
+
+	do, _ := os.ReadFile("../test_specs/digitalocean.yaml")
 	var rootNode yaml.Node
-	_ = yaml.Unmarshal(asana, &rootNode)
+	_ = yaml.Unmarshal(do, &rootNode)
 
-	// this should fail because the base url is not a valid url and digital ocean won't be able to resolve
-	// anything.
-	baseURL, _ := url.Parse("https://githerbs.com/fresh/herbs/for/you")
-	index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
-		BaseURL: baseURL,
-		//AllowRemoteLookup: true,
-		//AllowFileLookup:   true,
-	})
+	location := "https://githerbsandcoffeeandcode.com/fresh/herbs/for/you" // not gonna work bro.
+	baseURL, _ := url.Parse(location)
+
+	// create a new config that allows remote lookups.
+	cf := &SpecIndexConfig{}
+	cf.AvoidBuildIndex = true
+	cf.AllowRemoteLookup = true
+	cf.AvoidCircularReferenceCheck = true
+	cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelError,
+	}))
+
+	// setting this baseURL will override the base
+	cf.BaseURL = baseURL
+
+	// create a new rolodex
+	rolo := NewRolodex(cf)
+
+	// set the rolodex root node to the root node of the spec.
+	rolo.SetRootNode(&rootNode)
+
+	// create a new remote fs and set the config for indexing.
+	remoteFS, _ := NewRemoteFSWithConfig(cf)
+
+	// add remote filesystem
+	rolo.AddRemoteFS(location, remoteFS)
+
+	// index the rolodex.
+	indexedErr := rolo.IndexTheRolodex()
+	assert.Error(t, indexedErr)
+	assert.Len(t, utils.UnwrapErrors(indexedErr), 291)
+
+	files := remoteFS.GetFiles()
+	fileLen := len(files)
+	assert.Equal(t, 0, fileLen)
+	assert.GreaterOrEqual(t, len(remoteFS.GetErrors()), 200)
 
-	assert.Len(t, index.GetAllExternalIndexes(), 0)
 }
 
 func TestSpecIndex_k8s(t *testing.T) {
@@ -494,13 +564,9 @@ func test_buildMixedRefServer() *httptest.Server {
 
 	bs, _ := os.ReadFile("../test_specs/burgershop.openapi.yaml")
 	return httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		if req.URL.String() == "/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml" {
-			rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT")
-			_, _ = rw.Write(bs)
-			return
-		}
+		rw.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT")
+		_, _ = rw.Write(bs)
 
-		_, _ = rw.Write([]byte(`OK`))
 	}))
 }
 
@@ -516,6 +582,9 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
 	cf.AllowRemoteLookup = true
 	cf.AvoidCircularReferenceCheck = true
 	cf.BasePath = "../test_specs"
+	cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+		Level: slog.LevelError,
+	}))
 
 	// setting this baseURL will override the base
 	cf.BaseURL, _ = url.Parse(server.URL)
@@ -720,7 +789,7 @@ func TestSpecIndex_FindComponent_WithACrazyAssPath(t *testing.T) {
 		index.FindComponent("#/paths/~1crazy~1ass~1references/get/parameters/0", nil).Node.Content[1].Value)
 }
 
-func TestSpecIndex_FindComponenth(t *testing.T) {
+func TestSpecIndex_FindComponent(t *testing.T) {
 	yml := `components:
   schemas:
     pizza:
@@ -754,34 +823,34 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) {
 	assert.Nil(t, index.lookupRolodex(nil))
 }
 
-func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) {
-	index := new(SpecIndex)
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
-	_, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something")
-	assert.Error(t, err)
-}
+//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) {
+//	index := new(SpecIndex)
+//	index.seenRemoteSources = make(map[string]*yaml.Node)
+//	index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
+//	_, _, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/$.....#[;]something")
+//	assert.Error(t, err)
+//}
 
-func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing.T) {
-	index := new(SpecIndex)
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
-	a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
-	assert.Error(t, err)
-	assert.Nil(t, a)
-	assert.Nil(t, b)
-}
+//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing.T) {
+//	index := new(SpecIndex)
+//	index.seenRemoteSources = make(map[string]*yaml.Node)
+//	index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
+//	a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
+//	assert.Error(t, err)
+//	assert.Nil(t, a)
+//	assert.Nil(t, b)
+//}
 
 // Discovered in issue https://github.com/pb33f/libopenapi/issues/37
-func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) {
-	index := new(SpecIndex)
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
-	a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
-	assert.NoError(t, err)
-	assert.NotNil(t, a)
-	assert.NotNil(t, b)
-}
+//func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) {
+//	index := new(SpecIndex)
+//	index.seenRemoteSources = make(map[string]*yaml.Node)
+//	index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
+//	a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
+//	assert.NoError(t, err)
+//	assert.NotNil(t, a)
+//	assert.NotNil(t, b)
+//}
 
 // Discovered in issue https://github.com/daveshanley/vacuum/issues/225
 func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
@@ -792,29 +861,13 @@ func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
 	_ = os.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664)
 	defer os.Remove("coffee-time.yaml")
 
-	index.seenRemoteSources = make(map[string]*yaml.Node)
+	//index.seenRemoteSources = make(map[string]*yaml.Node)
 	a, b, err := index.lookupFileReference("coffee-time.yaml")
 	assert.NoError(t, err)
 	assert.NotNil(t, a)
 	assert.NotNil(t, b)
 }
 
-func TestSpecIndex_CheckBadURLRef(t *testing.T) {
-	yml := `openapi: 3.1.0
-paths:
-  /cakes:
-    post:
-      parameters:
-        - $ref: 'httpsss://badurl'`
-
-	var rootNode yaml.Node
-	_ = yaml.Unmarshal([]byte(yml), &rootNode)
-
-	index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig())
-
-	assert.Len(t, index.refErrors, 2)
-}
-
 func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) {
 	yml := `openapi: 3.1.0
 paths:
@@ -829,15 +882,34 @@ paths:
 	c := CreateClosedAPIIndexConfig()
 	idx := NewSpecIndexWithConfig(&rootNode, c)
 
-	assert.Len(t, idx.refErrors, 2)
-	assert.Equal(t, "remote lookups are not permitted, "+
-		"please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error())
+	assert.Len(t, idx.refErrors, 1)
 }
 
 func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) {
-	_ = os.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664)
+	c := []byte("name: time for coffee")
+
+	_ = os.WriteFile("coffee-time.yaml", c, 0o664)
 	defer os.Remove("coffee-time.yaml")
 
+	// create a new config that allows local and remote to be mixed up.
+	cf := CreateOpenAPIIndexConfig()
+	cf.AvoidCircularReferenceCheck = true
+	cf.BasePath = "."
+
+	// create a new rolodex
+	rolo := NewRolodex(cf)
+
+	// configure the local filesystem.
+	fsCfg := LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		FileFilters:   []string{"coffee-time.yaml"},
+		DirFS:         os.DirFS(cf.BasePath),
+	}
+
+	// create a new local filesystem.
+	fileFS, err := NewLocalFSWithConfig(&fsCfg)
+	assert.NoError(t, err)
+
 	yml := `openapi: 3.0.3
 paths:
   /cakes:
@@ -845,24 +917,32 @@ paths:
       parameters:
         - $ref: 'coffee-time.yaml'`
 
-	var rootNode yaml.Node
-	_ = yaml.Unmarshal([]byte(yml), &rootNode)
+	var coffee yaml.Node
+	_ = yaml.Unmarshal([]byte(yml), &coffee)
 
-	index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig())
+	// set the rolodex root node to the root node of the spec.
+	rolo.SetRootNode(&coffee)
+
+	rolo.AddLocalFS(cf.BasePath, fileFS)
+	rErr := rolo.IndexTheRolodex()
+
+	assert.NoError(t, rErr)
+
+	index := rolo.GetRootIndex()
 
 	assert.NotNil(t, index.GetAllParametersFromOperations()["/cakes"]["post"]["coffee-time.yaml"][0].Node)
 }
 
-func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) {
-	index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{
-		//AllowRemoteLookup: true,
-	})
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey")
-	assert.Error(t, err)
-	assert.Nil(t, a)
-	assert.Nil(t, b)
-}
+//func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) {
+//	index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{
+//		//AllowRemoteLookup: true,
+//	})
+//	index.seenRemoteSources = make(map[string]*yaml.Node)
+//	a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey")
+//	assert.Error(t, err)
+//	assert.Nil(t, a)
+//	assert.Nil(t, b)
+//}
 
 func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) {
 	index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
@@ -870,39 +950,79 @@ func TestSpecIndex_lookupFileReference_BadFileName(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) {
-	index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{}
-	_, _, err := index.lookupFileReference("magic-money-file.json#something")
-	assert.Error(t, err)
-}
-
-func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) {
-	index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
-	_, _, err := index.lookupFileReference("chickers.json#no-rice")
-	assert.Error(t, err)
-}
-
-func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) {
-	_ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664)
-	defer os.Remove("chickers.yaml")
-	var root yaml.Node
-	index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig())
-	_, _, err := index.lookupFileReference("chickers.yaml#no-rice")
-	assert.Error(t, err)
-}
+//
+//func TestSpecIndex_lookupFileReference_SeenSourceSimulation_Error(t *testing.T) {
+//	index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
+//	index.seenRemoteSources = make(map[string]*yaml.Node)
+//	index.seenRemoteSources["magic-money-file.json"] = &yaml.Node{}
+//	_, _, err := index.lookupFileReference("magic-money-file.json#something")
+//	assert.Error(t, err)
+//}
+//
+//func TestSpecIndex_lookupFileReference_BadFile(t *testing.T) {
+//	index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
+//	_, _, err := index.lookupFileReference("chickers.json#no-rice")
+//	assert.Error(t, err)
+//}
+//
+//func TestSpecIndex_lookupFileReference_BadFileDataRead(t *testing.T) {
+//	_ = os.WriteFile("chickers.yaml", []byte("broke: the: thing: [again]"), 0o664)
+//	defer os.Remove("chickers.yaml")
+//	var root yaml.Node
+//	index := NewSpecIndexWithConfig(&root, CreateOpenAPIIndexConfig())
+//	_, _, err := index.lookupFileReference("chickers.yaml#no-rice")
+//	assert.Error(t, err)
+//}
 
 func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) {
-	_ = os.WriteFile("embie.yaml", []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy"), 0o664)
+
+	embie := []byte("naughty:\n - puppy: dog\n - puppy: naughty\npuppy:\n - naughty: puppy")
+
+	_ = os.WriteFile("embie.yaml", embie, 0o664)
 	defer os.Remove("embie.yaml")
 
-	index := NewSpecIndexWithConfig(nil, CreateOpenAPIIndexConfig())
-	index.seenRemoteSources = make(map[string]*yaml.Node)
-	k, doc, err := index.lookupFileReference("embie.yaml#/.naughty")
+	// create a new config that allows local and remote to be mixed up.
+	cf := CreateOpenAPIIndexConfig()
+	cf.AvoidBuildIndex = true
+	cf.AvoidCircularReferenceCheck = true
+	cf.BasePath = "."
+
+	// create a new rolodex
+	rolo := NewRolodex(cf)
+
+	var myPuppy yaml.Node
+	_ = yaml.Unmarshal(embie, &myPuppy)
+
+	// set the rolodex root node to the root node of the spec.
+	rolo.SetRootNode(&myPuppy)
+
+	// configure the local filesystem.
+	fsCfg := LocalFSConfig{
+		BaseDirectory: cf.BasePath,
+		FileFilters:   []string{"embie.yaml"},
+		DirFS:         os.DirFS(cf.BasePath),
+	}
+
+	// create a new local filesystem.
+	fileFS, err := NewLocalFSWithConfig(&fsCfg)
 	assert.NoError(t, err)
-	assert.NotNil(t, doc)
-	assert.Nil(t, k)
+
+	rolo.AddLocalFS(cf.BasePath, fileFS)
+	rErr := rolo.IndexTheRolodex()
+
+	assert.NoError(t, rErr)
+
+	embieRoloFile, fErr := rolo.Open("embie.yaml")
+
+	assert.NoError(t, fErr)
+	assert.NotNil(t, embieRoloFile)
+
+	index := rolo.GetRootIndex()
+	//index.seenRemoteSources = make(map[string]*yaml.Node)
+	absoluteRef, _ := filepath.Abs("embie.yaml#/naughty")
+	fRef := index.SearchIndexForReference(absoluteRef)
+	assert.NotNil(t, fRef)
+
 }
 
 func TestSpecIndex_lookupFileReference(t *testing.T) {
@@ -918,7 +1038,6 @@ func TestSpecIndex_lookupFileReference(t *testing.T) {
 	// create a new config that allows local and remote to be mixed up.
 	cf := CreateOpenAPIIndexConfig()
 	cf.AvoidBuildIndex = true
-	cf.AllowRemoteLookup = true
 	cf.AvoidCircularReferenceCheck = true
 	cf.BasePath = "."