A clean sweep around resolving during model builds.

Really hammering on this model using all available examples of crazy references used in crazy ways. The fixes mean I can delete the recent patch added to handle lost indexes.

I spent two days on this commit.

Signed-off-by: quobix <dave@quobix.com>

datamodel/high/v3/document_test.go

@@ -491,7 +491,7 @@ func TestDigitalOceanAsDocViaCheckout(t *testing.T) {
 		AllowRemoteReferences: true,
 		BasePath:              basePath,
 		Logger: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
-			Level: slog.LevelDebug,
+			Level: slog.LevelError,
 		})),
 	}
 

datamodel/low/base/schema.go

@@ -1020,8 +1020,9 @@ func buildPropertyMap(ctx context.Context, root *yaml.Node, idx *index.SpecIndex
 			refString := ""
 			var refNode *yaml.Node
 			if h, _, l := utils.IsNodeRefValue(prop); h {
-				ref, fIdx, _, fctx := low.LocateRefNodeWithContext(ctx, prop, idx)
+				ref, fIdx, _, fctx := low.LocateRefNodeWithContext(foundCtx, prop, foundIdx)
 				if ref != nil {
+
 					refNode = prop
 					prop = ref
 					refString = l
@@ -1087,9 +1088,9 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label
 		syncChan := make(chan buildResult)
 
 		// build out a SchemaProxy for every sub-schema.
-		build := func(pctx context.Context, kn, vn *yaml.Node, rf *yaml.Node, schemaIdx int, c chan buildResult,
+		build := func(pctx context.Context, fIdx *index.SpecIndex, kn, vn *yaml.Node, rf *yaml.Node, schemaIdx int, c chan buildResult,
 			isRef bool, refLocation string,
-		) {
+		) buildResult {
 			// a proxy design works best here. polymorphism, pretty much guarantees that a sub-schema can
 			// take on circular references through polymorphism. Like the resolver, if we try and follow these
 			// journey's through hyperspace, we will end up creating endless amounts of threads, spinning off
@@ -1099,7 +1100,7 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label
 			sp := new(SchemaProxy)
 			sp.kn = kn
 			sp.vn = vn
-			sp.idx = idx
+			sp.idx = fIdx
 			sp.ctx = pctx
 			if isRef {
 				sp.SetReference(refLocation, rf)
@@ -1108,25 +1109,28 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label
 				Value:     sp,
 				ValueNode: vn,
 			}
-			c <- buildResult{
+			return buildResult{
 				res: res,
 				idx: schemaIdx,
 			}
+
 		}
 
 		isRef := false
 		refLocation := ""
 		var refNode *yaml.Node
 		foundCtx := ctx
+		foundIdx := idx
 		if utils.IsNodeMap(valueNode) {
 			h := false
 			if h, _, refLocation = utils.IsNodeRefValue(valueNode); h {
 				isRef = true
-				ref, _, _, fctx := low.LocateRefNodeWithContext(ctx, valueNode, idx)
+				ref, fIdx, _, fctx := low.LocateRefNodeWithContext(foundCtx, valueNode, foundIdx)
 				if ref != nil {
 					refNode = valueNode
 					valueNode = ref
 					foundCtx = fctx
+					foundIdx = fIdx
 				} else {
 					errors <- fmt.Errorf("build schema failed: reference cannot be found: %s, line %d, col %d",
 						valueNode.Content[1].Value, valueNode.Content[1].Line, valueNode.Content[1].Column)
@@ -1135,8 +1139,7 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label
 
 			// this only runs once, however to keep things consistent, it makes sense to use the same async method
 			// that arrays will use.
-			go build(foundCtx, labelNode, valueNode, refNode, -1, syncChan, isRef, refLocation)
+			r := build(foundCtx, foundIdx, labelNode, valueNode, refNode, -1, syncChan, isRef, refLocation)
-			r := <-syncChan
 			schemas <- schemaProxyBuildResult{
 				k: low.KeyReference[string]{
 					KeyNode: labelNode,
@@ -1151,13 +1154,16 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label
 			for i, vn := range valueNode.Content {
 				isRef = false
 				h := false
+				foundIdx = idx
+				foundCtx = ctx
 				if h, _, refLocation = utils.IsNodeRefValue(vn); h {
 					isRef = true
-					ref, _, _, fctx := low.LocateRefNodeWithContext(ctx, vn, idx)
+					ref, fIdx, _, fctx := low.LocateRefNodeWithContext(foundCtx, vn, foundIdx)
 					if ref != nil {
 						refNode = vn
 						vn = ref
 						foundCtx = fctx
+						foundIdx = fIdx
 					} else {
 						err := fmt.Errorf("build schema failed: reference cannot be found: %s, line %d, col %d",
 							vn.Content[1].Value, vn.Content[1].Line, vn.Content[1].Column)
@@ -1166,14 +1172,8 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label
 					}
 				}
 				refBuilds++
-				go build(foundCtx, vn, vn, refNode, i, syncChan, isRef, refLocation)
+				r := build(foundCtx, foundIdx, vn, vn, refNode, i, syncChan, isRef, refLocation)
-			}
+				results[r.idx] = r.res
-
-			completedBuilds := 0
-			for completedBuilds < refBuilds {
-				res := <-syncChan
-				completedBuilds++
-				results[res.idx] = res.res
 			}
 
 			for _, r := range results {
@@ -1225,18 +1225,7 @@ func ExtractSchema(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) (
 		if schNode != nil {
 			h := false
 			if h, _, refLocation = utils.IsNodeRefValue(schNode); h {
-				var specPath string
+				ref, fIdx, _, nCtx := low.LocateRefNodeWithContext(foundCtx, schNode, foundIndex)
-				if ctx != nil && ctx.Value(index.CurrentPathKey) != nil {
-					specPath = ctx.Value(index.CurrentPathKey).(string)
-				}
-				if idx.GetSpecAbsolutePath() != specPath && (!strings.HasPrefix(refLocation, "http") &&
-					!strings.HasPrefix(idx.GetSpecAbsolutePath(), "http")) {
-					if !strings.HasSuffix(idx.GetSpecAbsolutePath(), "root.yaml") {
-						ctx = context.WithValue(ctx, index.CurrentPathKey, idx.GetSpecAbsolutePath())
-					}
-				}
-
-				ref, fIdx, _, nCtx := low.LocateRefNodeWithContext(ctx, schNode, idx)
 				if ref != nil {
 					refNode = schNode
 					schNode = ref

datamodel/low/base/schema_proxy.go

@@ -141,7 +141,14 @@ func (sp *SchemaProxy) Hash() [32]byte {
 			// only resolve this proxy if it's not a ref.
 			sch := sp.Schema()
 			sp.rendered = sch
-			return sch.Hash()
+			if sch != nil {
+				return sch.Hash()
+			}
+			logger := sp.idx.GetLogger()
+			if logger != nil {
+				logger.Warn("SchemaProxy.Hash() failed to resolve schema, returning empty hash", "error", sp.GetBuildError().Error())
+			}
+			return [32]byte{}
 		}
 	}
 	// hash reference value only, do not resolve!

datamodel/low/extraction_functions.go

@@ -533,7 +533,7 @@ func ExtractMapNoLookupExtensions[PT Buildable[N], N any](
 			var refNode *yaml.Node
 			// if value is a reference, we have to look it up in the index!
 			if h, _, rv := utils.IsNodeRefValue(node); h {
-				ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, node, idx)
+				ref, fIdx, err, nCtx := LocateRefNodeWithContext(foundContext, node, foundIndex)
 				if ref != nil {
 					refNode = node
 					node = ref
@@ -608,6 +608,8 @@ type mappingResult[T any] struct {
 type buildInput struct {
 	label *yaml.Node
 	value *yaml.Node
+	ctx   context.Context
+	idx   *index.SpecIndex
 }
 
 // ExtractMapExtensions will extract a map of KeyReference and ValueReference from a root yaml.Node. The 'label' is
@@ -626,14 +628,16 @@ func ExtractMapExtensions[PT Buildable[N], N any](
 	var labelNode, valueNode *yaml.Node
 	var circError error
 	root = utils.NodeAlias(root)
+	foundIndex := idx
+	foundContext := ctx
 	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
 		// locate reference in index.
 		ref, fIdx, err, fCtx := LocateRefNodeWithContext(ctx, root, idx)
 		if ref != nil {
 			valueNode = ref
 			labelNode = rl
-			ctx = fCtx
+			foundContext = fCtx
-			idx = fIdx
+			foundIndex = fIdx
 			if err != nil {
 				circError = err
 			}
@@ -649,8 +653,8 @@ func ExtractMapExtensions[PT Buildable[N], N any](
 				ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, valueNode, idx)
 				if ref != nil {
 					valueNode = ref
-					idx = fIdx
+					foundIndex = fIdx
-					ctx = nCtx
+					foundContext = nCtx
 					if err != nil {
 						circError = err
 					}
@@ -719,25 +723,29 @@ func ExtractMapExtensions[PT Buildable[N], N any](
 			wg.Done()
 		}()
 
+		startIdx := foundIndex
+		startCtx := foundContext
+
 		translateFunc := func(input buildInput) (mappingResult[PT], error) {
-			foundIndex := idx
-			foundContext := ctx
 
 			en := input.value
 
+			sCtx := startCtx
+			sIdx := startIdx
+
 			var refNode *yaml.Node
 			var referenceValue string
 			// check our valueNode isn't a reference still.
 			if h, _, refVal := utils.IsNodeRefValue(en); h {
-				ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, en, idx)
+				ref, fIdx, err, nCtx := LocateRefNodeWithContext(sCtx, en, sIdx)
 				if ref != nil {
 					refNode = en
 					en = ref
 					referenceValue = refVal
 					if fIdx != nil {
-						foundIndex = fIdx
+						sIdx = fIdx
 					}
-					foundContext = nCtx
+					sCtx = nCtx
 					if err != nil {
 						circError = err
 					}
@@ -752,7 +760,7 @@ func ExtractMapExtensions[PT Buildable[N], N any](
 			var n PT = new(N)
 			en = utils.NodeAlias(en)
 			_ = BuildModel(en, n)
-			err := n.Build(foundContext, input.label, en, foundIndex)
+			err := n.Build(sCtx, input.label, en, sIdx)
 			if err != nil {
 				return mappingResult[PT]{}, err
 			}
@@ -775,6 +783,7 @@ func ExtractMapExtensions[PT Buildable[N], N any](
 				v: v,
 			}, nil
 		}
+
 		err := datamodel.TranslatePipeline[buildInput, mappingResult[PT]](in, out, translateFunc)
 		wg.Wait()
 		if err != nil {

document_examples_test.go

@@ -6,6 +6,7 @@ package libopenapi
 import (
 	"bytes"
 	"fmt"
+	what_changed "github.com/pb33f/libopenapi/what-changed"
 	"log/slog"
 	"net/url"
 	"os"
@@ -127,7 +128,7 @@ func ExampleNewDocument_fromWithDocumentConfigurationSuccess() {
 		panic(fmt.Sprintf("cannot create new document: %e", err))
 	}
 
-	_, errors := doc.BuildV3Model()
+	m, errors := doc.BuildV3Model()
 
 	// if anything went wrong when building the v3 model, a slice of errors will be returned
 	if len(errors) > 0 {
@@ -135,6 +136,10 @@ func ExampleNewDocument_fromWithDocumentConfigurationSuccess() {
 	} else {
 		fmt.Println("Digital Ocean spec built successfully")
 	}
+
+	// running this through a change detection, will render out the entire model and
+	// any stage two rendering for the model will be caught.
+	what_changed.CompareOpenAPIDocuments(m.Model.GoLow(), m.Model.GoLow())
 	// Output: Digital Ocean spec built successfully
 }
 

index/resolver.go

@@ -200,13 +200,17 @@ func (resolver *Resolver) Resolve() []*ResolvingError {
 
 		if !resolver.circChecked {
 			resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{
-				ErrorRef: fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Definition),
+				ErrorRef:          fmt.Errorf("infinite circular reference detected: %s", circRef.Start.Definition),
-				Node:     circRef.ParentNode,
+				Node:              circRef.ParentNode,
-				Path:     circRef.GenerateJourneyPath(),
+				Path:              circRef.GenerateJourneyPath(),
+				CircularReference: circRef,
 			})
 		}
 	}
-
+	resolver.specIndex.SetCircularReferences(resolver.circularReferences)
+	resolver.specIndex.SetIgnoredArrayCircularReferences(resolver.ignoredArrayReferences)
+	resolver.specIndex.SetIgnoredPolymorphicCircularReferences(resolver.ignoredPolyReferences)
+	resolver.circChecked = true
 	return resolver.resolvingErrors
 }
 
@@ -394,6 +398,9 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
 			if foundRef != nil {
 				original = foundRef
 			}
+			if original == nil {
+				panic("help")
+			}
 			resolved := resolver.VisitReference(original, seen, journey, resolve)
 			if resolve && !original.Circular {
 				ref.Resolved = true
@@ -489,10 +496,14 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
 			if utils.IsNodeMap(n) || utils.IsNodeArray(n) {
 				depth++
 
-				foundRef, _ := resolver.specIndex.SearchIndexForReferenceByReference(ref)
+				//refIsRef, _, _ := utils.IsNodeRefValue(ref.Node)
+				var foundRef *Reference
+				//if refIsRef {
+				foundRef, _ = resolver.specIndex.SearchIndexForReferenceByReference(ref)
 				if foundRef != nil && !foundRef.Circular {
 					found = append(found, resolver.extractRelatives(foundRef, n, node, foundRelatives, journey, seen, resolve, depth)...)
 				}
+				//	}
 				if foundRef == nil {
 					found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, seen, resolve, depth)...)
 				}

index/resolver_test.go

@@ -683,8 +683,8 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
 	_ = yaml.Unmarshal(mixedref, &rootNode)
 
 	// create a test server.
-	server := test_buildMixedRefServer()
+	//server := test_buildMixedRefServer()
-	defer server.Close()
+	//defer server.Close()
 
 	// create a new config that allows local and remote to be mixed up.
 	cf := CreateOpenAPIIndexConfig()
@@ -692,9 +692,11 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
 	cf.AllowRemoteLookup = true
 	cf.AvoidCircularReferenceCheck = true
 	cf.BasePath = "../test_specs"
+	cf.SpecAbsolutePath, _ = filepath.Abs("../test_specs/mixedref-burgershop.openapi.yaml")
+	cf.ExtractRefsSequentially = true
 
 	// setting this baseURL will override the base
-	cf.BaseURL, _ = url.Parse(server.URL)
+	cf.BaseURL, _ = url.Parse("https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/")
 
 	// create a new rolodex
 	rolo := NewRolodex(cf)
@@ -703,7 +705,7 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
 	rolo.SetRootNode(&rootNode)
 
 	// create a new remote fs and set the config for indexing.
-	remoteFS, _ := NewRemoteFSWithRootURL(server.URL)
+	remoteFS, _ := NewRemoteFSWithConfig(cf)
 	remoteFS.SetIndexConfig(cf)
 
 	// set our remote handler func
@@ -715,8 +717,7 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
 	// configure the local filesystem.
 	fsCfg := LocalFSConfig{
 		BaseDirectory: cf.BasePath,
-		FileFilters:   []string{"burgershop.openapi.yaml"},
+		IndexConfig:   cf,
-		DirFS:         os.DirFS(cf.BasePath),
 	}
 
 	// create a new local filesystem.
@@ -725,7 +726,7 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
 
 	// add file systems to the rolodex
 	rolo.AddLocalFS(cf.BasePath, fileFS)
-	rolo.AddRemoteFS(server.URL, remoteFS)
+	rolo.AddRemoteFS("https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/", remoteFS)
 
 	// index the rolodex.
 	indexedErr := rolo.IndexTheRolodex()
@@ -737,12 +738,12 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
 	resolver := index().GetResolver()
 
 	assert.Len(t, resolver.GetCircularReferences(), 0)
-	assert.Equal(t, 2, resolver.GetIndexesVisited())
+	assert.Equal(t, 9, resolver.GetIndexesVisited())
 
 	// in v0.8.2 a new check was added when indexing, to prevent re-indexing the same file multiple times.
 	assert.Equal(t, 6, resolver.GetRelativesSeen())
-	assert.Equal(t, 6, resolver.GetJourneysTaken())
+	assert.Equal(t, 15, resolver.GetJourneysTaken())
-	assert.Equal(t, 8, resolver.GetReferenceVisited())
+	assert.Equal(t, 17, resolver.GetReferenceVisited())
 }
 
 func TestResolver_ResolveComponents_k8s(t *testing.T) {

index/rolodex.go

@@ -320,7 +320,7 @@ func (r *Rolodex) IndexTheRolodex() error {
 		if r.indexConfig.IgnorePolymorphicCircularReferences {
 			resolver.IgnorePolymorphicCircularReferences()
 		}
-
+		r.rootIndex = index
 		r.logger.Debug("[rolodex] starting root index build")
 		index.BuildIndex()
 		r.logger.Debug("[rolodex] root index build completed")
@@ -338,7 +338,7 @@ func (r *Rolodex) IndexTheRolodex() error {
 				r.ignoredCircularReferences = append(r.ignoredCircularReferences, resolver.GetIgnoredCircularArrayReferences()...)
 			}
 		}
-		r.rootIndex = index
+
 		if len(index.refErrors) > 0 {
 			caughtErrors = append(caughtErrors, index.refErrors...)
 		}

index/rolodex_remote_loader.go

@@ -420,7 +420,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 		name:         remoteParsedURL.Path,
 		extension:    fileExt,
 		data:         responseBytes,
-		fullPath:     absolutePath,
+		fullPath:     remoteParsedURL.String(),
 		URL:          remoteParsedURL,
 		lastModified: lastModifiedTime,
 	}

index/rolodex_remote_loader_test.go

@@ -192,7 +192,7 @@ func TestNewRemoteFS_BasicCheck_Relative_Deeper(t *testing.T) {
 	assert.Equal(t, YAML, file.(*RemoteFile).GetFileExtension())
 	assert.NotNil(t, file.(*RemoteFile).GetLastModified())
 	assert.Len(t, file.(*RemoteFile).GetErrors(), 0)
-	assert.Equal(t, "/deeper/even_deeper/file3.yaml", file.(*RemoteFile).GetFullPath())
+	assert.Contains(t, file.(*RemoteFile).GetFullPath(), "/deeper/even_deeper/file3.yaml")
 	assert.False(t, file.(*RemoteFile).IsDir())
 	assert.Nil(t, file.(*RemoteFile).Sys())
 	assert.Nil(t, file.(*RemoteFile).Close())

index/rolodex_test.go

@@ -392,10 +392,6 @@ properties:
 
 	third := `type: "object"
 properties:
-  name:
-    $ref: "http://the-space-race-is-all-about-space-and-time-dot.com/$4"
-  tame: 
-    $ref: "http://the-space-race-is-all-about-space-and-time-dot.com/$5#/"
   blame:
     $ref: "$_5"
 
@@ -482,16 +478,22 @@ components:
 
 	baseDir := "tmp-a"
 
+	cf := CreateOpenAPIIndexConfig()
+	cf.BasePath = baseDir
+	cf.IgnorePolymorphicCircularReferences = true
+	cf.SkipDocumentCheck = true
+
 	fsCfg := &LocalFSConfig{
 		BaseDirectory: baseDir,
-		DirFS:         os.DirFS(baseDir),
+		IndexConfig:   cf,
-		FileFilters: []string{
+		//DirFS:         os.DirFS(baseDir),
-			filepath.Base(firstFile.Name()),
+		//FileFilters: []string{
-			filepath.Base(secondFile.Name()),
+		//	filepath.Base(firstFile.Name()),
-			filepath.Base(thirdFile.Name()),
+		//	filepath.Base(secondFile.Name()),
-			filepath.Base(fourthFile.Name()),
+		//	filepath.Base(thirdFile.Name()),
-			filepath.Base(fifthFile.Name()),
+		//	filepath.Base(fourthFile.Name()),
-		},
+		//	filepath.Base(fifthFile.Name()),
+		//},
 	}
 
 	fileFS, err := NewLocalFSWithConfig(fsCfg)
@@ -499,29 +501,29 @@ components:
 		t.Fatal(err)
 	}
 
-	cf := CreateOpenAPIIndexConfig()
-	cf.BasePath = baseDir
-	cf.IgnorePolymorphicCircularReferences = true
-	cf.SkipDocumentCheck = true
-
 	// add logger to config
 	cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
-		Level: slog.LevelDebug,
+		Level: slog.LevelError,
 	}))
 
 	rolodex := NewRolodex(cf)
 	rolodex.AddLocalFS(baseDir, fileFS)
 
-	srv := test_rolodexDeepRefServer([]byte(first), []byte(second),
+	//srv := test_rolodexDeepRefServer([]byte(first), []byte(second),
-		[]byte(third), []byte(fourth), []byte(fifth))
+	//	[]byte(third), []byte(fourth), []byte(fifth))
-	defer srv.Close()
+	//defer srv.Close()
 
-	u, _ := url.Parse(srv.URL)
+	//u, _ := url.Parse(srv.URL)
-	cf.BaseURL = u
+	//cf.BaseURL = u
-	remoteFS, rErr := NewRemoteFSWithConfig(cf)
+	//remoteFS, rErr := NewRemoteFSWithConfig(cf)
-	assert.NoError(t, rErr)
+	//assert.NoError(t, rErr)
 
-	rolodex.AddRemoteFS(srv.URL, remoteFS)
+	//rolodex.AddRemoteFS(srv.URL, remoteFS)
+
+	var rootNode yaml.Node
+	err = yaml.Unmarshal([]byte(first), &rootNode)
+	assert.NoError(t, err)
+	rolodex.SetRootNode(&rootNode)
 
 	err = rolodex.IndexTheRolodex()
 	assert.NoError(t, err)
@@ -545,33 +547,33 @@ components:
 	assert.NotNil(t, x)
 	assert.NoError(t, y)
 
-	// extract a remote  file
+	//// extract a remote  file
-	f, _ = rolodex.Open("http://the-space-race-is-all-about-space-and-time-dot.com/" + filepath.Base(fourthFile.Name()))
+	//f, _ = rolodex.Open("http://the-space-race-is-all-about-space-and-time-dot.com/" + filepath.Base(fourthFile.Name()))
-
+	//
-	// index
+	//// index
-	x, y = f.(*rolodexFile).Index(cf)
+	//x, y = f.(*rolodexFile).Index(cf)
-	assert.NotNil(t, x)
+	//assert.NotNil(t, x)
-	assert.NoError(t, y)
+	//assert.NoError(t, y)
-
+	//
-	// re-index
+	//// re-index
-	x, y = f.(*rolodexFile).Index(cf)
+	//x, y = f.(*rolodexFile).Index(cf)
-	assert.NotNil(t, x)
+	//assert.NotNil(t, x)
-	assert.NoError(t, y)
+	//assert.NoError(t, y)
-
+	//
-	// extract another remote  file
+	//// extract another remote  file
-	f, _ = rolodex.Open("http://the-space-race-is-all-about-space-and-time-dot.com/" + filepath.Base(fifthFile.Name()))
+	//f, _ = rolodex.Open("http://the-space-race-is-all-about-space-and-time-dot.com/" + filepath.Base(fifthFile.Name()))
-
+	//
-	//change cf to perform document check (which should fail)
+	////change cf to perform document check (which should fail)
-	cf.SkipDocumentCheck = false
+	//cf.SkipDocumentCheck = false
-
+	//
-	// index and fail
+	//// index and fail
-	x, y = f.(*rolodexFile).Index(cf)
+	//x, y = f.(*rolodexFile).Index(cf)
-	assert.Nil(t, x)
+	//assert.Nil(t, x)
-	assert.Error(t, y)
+	//assert.Error(t, y)
-
+	//
-	// file that is not local, but is remote
+	//// file that is not local, but is remote
-	f, _ = rolodex.Open("https://pb33f.io/bingo/jingo.yaml")
+	//f, _ = rolodex.Open("https://pb33f.io/bingo/jingo.yaml")
-	assert.NotNil(t, f)
+	//assert.NotNil(t, f)
 
 }
 
@@ -619,9 +621,7 @@ properties:
 	third := `type: "object"
 properties:
   herbs:
-    $ref: "$1"
+    $ref: "$1"`
-  name:
-    $ref: "http://the-space-race-is-all-about-space-and-time-dot.com/$4"`
 
 	second := `openapi: 3.1.0
 components:
@@ -673,7 +673,7 @@ components:
 	first = strings.ReplaceAll(strings.ReplaceAll(first, "$2", secondFile.Name()), "\\", "\\\\")
 	second = strings.ReplaceAll(strings.ReplaceAll(second, "$3", thirdFile.Name()), "\\", "\\\\")
 	third = strings.ReplaceAll(strings.ReplaceAll(third, "$4", filepath.Base(fourthFile.Name())), "\\", "\\\\")
-	fourth = strings.ReplaceAll(strings.ReplaceAll(first, "$1", filepath.Base(firstFile.Name())), "\\", "\\\\")
+	third = strings.ReplaceAll(strings.ReplaceAll(first, "$1", filepath.Base(thirdFile.Name())), "\\", "\\\\")
 
 	firstFile.WriteString(first)
 	secondFile.WriteString(second)
@@ -682,7 +682,7 @@ components:
 
 	defer os.RemoveAll(tmp)
 
-	baseDir := tmp
+	baseDir, _ := filepath.Abs(tmp)
 	cf := CreateOpenAPIIndexConfig()
 	cf.BasePath = baseDir
 	cf.IgnorePolymorphicCircularReferences = true
@@ -704,17 +704,6 @@ components:
 	_ = yaml.Unmarshal([]byte(first), &rootNode)
 	rolodex.SetRootNode(&rootNode)
 
-	srv := test_rolodexDeepRefServer([]byte(first), []byte(second),
-		[]byte(third), []byte(fourth), nil)
-	defer srv.Close()
-
-	u, _ := url.Parse(srv.URL)
-	cf.BaseURL = u
-	remoteFS, rErr := NewRemoteFSWithConfig(cf)
-	assert.NoError(t, rErr)
-
-	rolodex.AddRemoteFS(srv.URL, remoteFS)
-
 	err = rolodex.IndexTheRolodex()
 	assert.Error(t, err)
 	assert.GreaterOrEqual(t, len(rolodex.GetCaughtErrors()), 1)
@@ -1183,7 +1172,7 @@ components:
 	assert.Len(t, rolodex.GetCaughtErrors(), 0)
 
 	assert.GreaterOrEqual(t, len(rolodex.GetIgnoredCircularReferences()), 1)
-	assert.Equal(t, rolodex.GetRootIndex().GetResolver().GetIndexesVisited(), 13)
+	assert.Equal(t, 174, rolodex.GetRootIndex().GetResolver().GetIndexesVisited())
 
 }
 

index/search_index.go

@@ -37,7 +37,8 @@ func (index *SpecIndex) SearchIndexForReferenceWithContext(ctx context.Context,
 func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx context.Context, searchRef *Reference) (*Reference, *SpecIndex, context.Context) {
 	if index.cache != nil {
 		if v, ok := index.cache.Load(searchRef.FullDefinition); ok {
-			return v.(*Reference), v.(*Reference).Index, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation)
+			idx := index.extractIndex(v.(*Reference))
+			return v.(*Reference), idx, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation)
 		}
 	}
 
@@ -100,20 +101,22 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 	}
 
 	if r, ok := index.allMappedRefs[ref]; ok {
+		idx := index.extractIndex(r)
 		index.cache.Store(ref, r)
-		return r, r.Index, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation)
+		return r, idx, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation)
 	}
 
 	if r, ok := index.allMappedRefs[refAlt]; ok {
-		index.cache.Store(refAlt, r)
+		idx := index.extractIndex(r)
-		return r, r.Index, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation)
+		idx.cache.Store(refAlt, r)
+		return r, idx, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation)
 	}
 
 	if r, ok := index.allComponentSchemaDefinitions.Load(refAlt); ok {
-		ref := r.(*Reference)
+		rf := r.(*Reference)
-
+		idx := index.extractIndex(rf)
 		index.cache.Store(refAlt, r)
-		return ref, ref.Index, context.WithValue(ctx, CurrentPathKey, ref.RemoteLocation)
+		return rf, idx, context.WithValue(ctx, CurrentPathKey, rf.RemoteLocation)
 	}
 
 	// check the rolodex for the reference.
@@ -141,10 +144,10 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 				refParsed = strings.ReplaceAll(ref, "./", "")
 			}
 
-			if strings.HasSuffix(n, refParsed) {
+			if strings.HasSuffix(refParsed, n) {
 				node, _ := rFile.GetContentAsYAMLNode()
 				if node != nil {
-					return &Reference{
+					r := &Reference{
 						FullDefinition: n,
 						Definition:     n,
 						IsRemote:       true,
@@ -152,7 +155,9 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 						Index:          rFile.GetIndex(),
 						Node:           node.Content[0],
 						ParentNode:     node,
-					}, rFile.GetIndex(), context.WithValue(ctx, CurrentPathKey, rFile.GetFullPath())
+					}
+					index.cache.Store(ref, r)
+					return r, rFile.GetIndex(), context.WithValue(ctx, CurrentPathKey, rFile.GetFullPath())
 				} else {
 					return nil, index, ctx
 				}
@@ -166,9 +171,8 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 
 				// check mapped refs.
 				if r, ok := idx.allMappedRefs[ref]; ok {
-					index.cache.Store(ref, r)
+					i := index.extractIndex(r)
-					idx.cache.Store(ref, r)
+					return r, i, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation)
-					return r, r.Index, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation)
 				}
 
 				// build a collection of all the inline schemas and search them
@@ -179,9 +183,10 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 				d = append(d, idx.allInlineSchemaObjectDefinitions...)
 				for _, s := range d {
 					if s.FullDefinition == ref {
+						i := index.extractIndex(s)
 						idx.cache.Store(ref, s)
 						index.cache.Store(ref, s)
-						return s, s.Index, context.WithValue(ctx, CurrentPathKey, s.RemoteLocation)
+						return s, i, context.WithValue(ctx, CurrentPathKey, s.RemoteLocation)
 					}
 				}
 
@@ -201,9 +206,9 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 					}
 
 					if found != nil {
-						idx.cache.Store(ref, found)
+						i := index.extractIndex(found)
-						index.cache.Store(ref, found)
+						i.cache.Store(ref, found)
-						return found, found.Index, context.WithValue(ctx, CurrentPathKey, found.RemoteLocation)
+						return found, i, context.WithValue(ctx, CurrentPathKey, found.RemoteLocation)
 					}
 				}
 			}
@@ -215,3 +220,16 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 	}
 	return nil, index, ctx
 }
+
+func (index *SpecIndex) extractIndex(r *Reference) *SpecIndex {
+	idx := r.Index
+	if idx != nil && r.Index.GetSpecAbsolutePath() != r.RemoteLocation {
+		for i := range r.Index.rolodex.indexes {
+			if r.Index.rolodex.indexes[i].GetSpecAbsolutePath() == r.RemoteLocation {
+				idx = r.Index.rolodex.indexes[i]
+				break
+			}
+		}
+	}
+	return idx
+}