a first working engine of the new design.

There is a horrible amount of work to be done to clean this up, and wire in remote support. but so far, this is working as expected and is now a much cleaner design, (once everything has been cleaned up that is)

Signed-off-by: quobix <dave@quobix.com>

datamodel/high/v3/document.go

@@ -89,7 +89,11 @@ type Document struct {
 	//
 	// This property is not a part of the OpenAPI schema, this is custom to libopenapi.
 	Index *index.SpecIndex `json:"-" yaml:"-"`
-	low   *low.Document
+
+	// Rolodex is the low-level rolodex used when creating this document.
+	// This in an internal structure and not part of the OpenAPI schema.
+	Rolodex *index.Rolodex  `json:"-" yaml:"-"`
+	low     *low.Document
 }
 
 // NewDocument will create a new high-level Document from a low-level one.

datamodel/low/v2/swagger.go

@@ -142,10 +142,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 
 	// build an index
 	idx := index.NewSpecIndexWithConfig(info.RootNode, &index.SpecIndexConfig{
-		BaseURL:           config.BaseURL,
+		BaseURL:          config.BaseURL,
-		RemoteURLHandler:  config.RemoteURLHandler,
+		RemoteURLHandler: config.RemoteURLHandler,
-		AllowRemoteLookup: config.AllowRemoteReferences,
+		//AllowRemoteLookup: config.AllowRemoteReferences,
-		AllowFileLookup:   config.AllowFileReferences,
+		//AllowFileLookup:   config.AllowFileReferences,
 	})
 	doc.Index = idx
 	doc.SpecInfo = info

datamodel/low/v3/create_document.go

@@ -3,13 +3,13 @@ package v3
 import (
 	"errors"
 	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/pb33f/libopenapi/datamodel"
 	"github.com/pb33f/libopenapi/datamodel/low"
 	"github.com/pb33f/libopenapi/datamodel/low/base"
 	"github.com/pb33f/libopenapi/index"
-	"github.com/pb33f/libopenapi/resolver"
 	"github.com/pb33f/libopenapi/utils"
 )
 
@@ -17,7 +17,7 @@ import (
 //
 // Deprecated: Use CreateDocumentFromConfig instead. This function will be removed in a later version, it
 // defaults to allowing file and remote references, and does not support relative file references.
-func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) {
+func CreateDocument(info *datamodel.SpecInfo) (*Document, error) {
 	config := datamodel.DocumentConfiguration{
 		AllowFileReferences:   true,
 		AllowRemoteReferences: true,
@@ -26,61 +26,92 @@ func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) {
 }
 
 // CreateDocumentFromConfig Create a new document from the provided SpecInfo and DocumentConfiguration pointer.
-func CreateDocumentFromConfig(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, []error) {
+func CreateDocumentFromConfig(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, error) {
 	return createDocument(info, config)
 }
 
-func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, []error) {
+func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfiguration) (*Document, error) {
 	_, labelNode, versionNode := utils.FindKeyNodeFull(OpenAPILabel, info.RootNode.Content)
 	var version low.NodeReference[string]
 	if versionNode == nil {
-		return nil, []error{errors.New("no openapi version/tag found, cannot create document")}
+		return nil, errors.New("no openapi version/tag found, cannot create document")
 	}
 	version = low.NodeReference[string]{Value: versionNode.Value, KeyNode: labelNode, ValueNode: versionNode}
 	doc := Document{Version: version}
 
 	// get current working directory as a basePath
 	cwd, _ := os.Getwd()
-
-	// If basePath is provided override it
 	if config.BasePath != "" {
 		cwd = config.BasePath
 	}
-	// build an index
-	idx := index.NewSpecIndexWithConfig(info.RootNode, &index.SpecIndexConfig{
-		BaseURL:           config.BaseURL,
-		RemoteURLHandler:  config.RemoteURLHandler,
-		BasePath:          cwd,
-		AllowFileLookup:   config.AllowFileReferences,
-		AllowRemoteLookup: config.AllowRemoteReferences,
-		AvoidBuildIndex:   config.AvoidIndexBuild,
-		SpecInfo:          info,
-	})
-	doc.Index = idx
 
+	// TODO: configure allowFileReferences and allowRemoteReferences stuff
+
+	// create an index config and shadow the document configuration.
+	idxConfig := index.CreateOpenAPIIndexConfig()
+	idxConfig.SpecInfo = info
+	idxConfig.BasePath = cwd
+	idxConfig.IgnoreArrayCircularReferences = config.IgnoreArrayCircularReferences
+	idxConfig.IgnorePolymorphicCircularReferences = config.IgnorePolymorphicCircularReferences
+	idxConfig.AvoidCircularReferenceCheck = config.SkipCircularReferenceCheck
+
+	rolodex := index.NewRolodex(idxConfig)
+	doc.Rolodex = rolodex
+
+	// If basePath is provided override it
+	if config.BasePath != "" {
+		var absError error
+		cwd, absError = filepath.Abs(config.BasePath)
+		if absError != nil {
+			return nil, absError
+		}
+
+		// create a local filesystem
+		fileFS, err := index.NewLocalFS(cwd, os.DirFS(cwd))
+		if err != nil {
+			return nil, err
+		}
+
+		// add the filesystem to the rolodex
+		rolodex.AddLocalFS(cwd, fileFS)
+
+	}
+
+	// TODO: Remote filesystem
+
+	// index the rolodex
+	err := rolodex.IndexTheRolodex()
 	var errs []error
+	if err != nil {
+		errs = append(errs, rolodex.GetCaughtErrors()...)
+	}
 
-	errs = idx.GetReferenceIndexErrors()
+	doc.Index = rolodex.GetRootIndex()
+
+	//errs = idx.GetReferenceIndexErrors()
 
 	// create resolver and check for circular references.
-	resolve := resolver.NewResolver(idx)
 
-	// if configured, ignore circular references in arrays and polymorphic schemas
+	//resolve := resolver.NewResolver(idx)
-	if config.IgnoreArrayCircularReferences {
+	//
-		resolve.IgnoreArrayCircularReferences()
+	//// if configured, ignore circular references in arrays and polymorphic schemas
-	}
+	//if config.IgnoreArrayCircularReferences {
-	if config.IgnorePolymorphicCircularReferences {
+	//	resolve.IgnoreArrayCircularReferences()
-		resolve.IgnorePolymorphicCircularReferences()
+	//}
-	}
+	//if config.IgnorePolymorphicCircularReferences {
-
+	//	resolve.IgnorePolymorphicCircularReferences()
-	// check for circular references.
+	//}
-	resolvingErrors := resolve.CheckForCircularReferences()
+	//
-
+	//if !config.AvoidIndexBuild {
-	if len(resolvingErrors) > 0 {
+	//	// check for circular references.
-		for r := range resolvingErrors {
+	//	resolvingErrors := resolve.CheckForCircularReferences()
-			errs = append(errs, resolvingErrors[r])
+	//
-		}
+	//	if len(resolvingErrors) > 0 {
-	}
+	//		for r := range resolvingErrors {
+	//			errs = append(errs, resolvingErrors[r])
+	//		}
+	//	}
+	//}
 
 	var wg sync.WaitGroup
 
@@ -117,10 +148,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 
 	wg.Add(len(extractionFuncs))
 	for _, f := range extractionFuncs {
-		go runExtraction(info, &doc, idx, f, &errs, &wg)
+		go runExtraction(info, &doc, rolodex.GetRootIndex(), f, &errs, &wg)
 	}
 	wg.Wait()
-	return &doc, errs
+	return &doc, errors.Join(errs...)
 }
 
 func extractInfo(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error {

datamodel/low/v3/document.go

@@ -82,6 +82,9 @@ type Document struct {
 	//
 	// This property is not a part of the OpenAPI schema, this is custom to libopenapi.
 	Index *index.SpecIndex
+
+	// Rolodex is a reference to the rolodex used when creating this document.
+	Rolodex *index.Rolodex
 }
 
 // FindSecurityRequirement will attempt to locate a security requirement string from a supplied name.

index/extract_refs.go

@@ -6,6 +6,7 @@ package index
 import (
 	"errors"
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	"github.com/pb33f/libopenapi/utils"
@@ -157,11 +158,31 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 				segs := strings.Split(value, "/")
 				name := segs[len(segs)-1]
 				_, p := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
+
+				// determine absolute path to this definition
+				iroot := filepath.Dir(index.specAbsolutePath)
+				uri := strings.Split(value, "#/")
+				var componentName string
+				var fullDefinitionPath string
+				if len(uri) == 2 {
+					if uri[0] == "" {
+						fullDefinitionPath = fmt.Sprintf("%s#/%s", index.specAbsolutePath, uri[1])
+					} else {
+						abs, _ := filepath.Abs(filepath.Join(iroot, uri[0]))
+						fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1])
+					}
+					componentName = fmt.Sprintf("#/%s", uri[1])
+				} else {
+					fullDefinitionPath = fmt.Sprintf("%s#/%s", iroot, uri[0])
+					componentName = fmt.Sprintf("#/%s", uri[0])
+				}
+
 				ref := &Reference{
-					Definition: value,
+					FullDefinition: fullDefinitionPath,
-					Name:       name,
+					Definition:     componentName,
-					Node:       node,
+					Name:           name,
-					Path:       p,
+					Node:           node,
+					Path:           p,
 				}
 
 				// add to raw sequenced refs
@@ -183,10 +204,11 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 				if len(node.Content) > 2 {
 					copiedNode := *node
 					copied := Reference{
-						Definition: ref.Definition,
+						FullDefinition: fullDefinitionPath,
-						Name:       ref.Name,
+						Definition:     ref.Definition,
-						Node:       &copiedNode,
+						Name:           ref.Name,
-						Path:       p,
+						Node:           &copiedNode,
+						Path:           p,
 					}
 					// protect this data using a copy, prevent the resolver from destroying things.
 					index.refsWithSiblings[value] = copied
@@ -413,19 +435,22 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
 	var found []*Reference
 
 	// run this async because when things get recursive, it can take a while
-	c := make(chan bool)
+	//c := make(chan bool)
 
 	locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
-		located := index.FindComponent(ref.Definition, ref.Node)
+		located := index.FindComponent(ref.FullDefinition, ref.Node)
 		if located != nil {
 			index.refLock.Lock()
 			if index.allMappedRefs[ref.Definition] == nil {
 				found = append(found, located)
 				index.allMappedRefs[ref.Definition] = located
-				sequence[refIndex] = &ReferenceMapped{
+				rm := &ReferenceMapped{
-					Reference:  located,
+					Reference:      located,
-					Definition: ref.Definition,
+					Definition:     ref.Definition,
+					FullDefinition: ref.FullDefinition,
 				}
+
+				sequence[refIndex] = rm
 			}
 			index.refLock.Unlock()
 		} else {
@@ -440,7 +465,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
 			index.refErrors = append(index.refErrors, indexError)
 			index.errorLock.Unlock()
 		}
-		c <- true
+		//c <- true
 	}
 
 	var refsToCheck []*Reference
@@ -464,17 +489,17 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
 
 	for r := range refsToCheck {
 		// expand our index of all mapped refs
-		go locate(refsToCheck[r], r, mappedRefsInSequence)
+		//go  locate(refsToCheck[r], r, mappedRefsInSequence)
-		// locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
+		locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
 	}
 
-	completedRefs := 0
+	//completedRefs := 0
-	for completedRefs < len(refsToCheck) {
+	//for completedRefs < len(refsToCheck) {
-		select {
+	//	select {
-		case <-c:
+	//	case <-c:
-			completedRefs++
+	//		completedRefs++
-		}
+	//	}
-	}
+	//}
 	for m := range mappedRefsInSequence {
 		if mappedRefsInSequence[m] != nil {
 			index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])

index/find_component.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -26,64 +25,33 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
 		return nil
 	}
 
-	remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
+	//remoteLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
-		if index.config.AllowRemoteLookup {
+	//	if index.config.AllowRemoteLookup {
-			return index.lookupRemoteReference(id)
+	//		return index.lookupRemoteReference(id)
-		} else {
+	//	} else {
-			return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
+	//		return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
-				"please set AllowRemoteLookup to true in the configuration")
+	//			"please set AllowRemoteLookup to true in the configuration")
-		}
+	//	}
-	}
+	//}
+	//
+	//fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
+	//	if index.config.AllowFileLookup {
+	//		return index.lookupFileReference(id)
+	//	} else {
+	//		return nil, nil, fmt.Errorf("local lookups are not permitted, " +
+	//			"please set AllowFileLookup to true in the configuration")
+	//	}
+	//}
 
-	fileLookup := func(id string) (*yaml.Node, *yaml.Node, error) {
+	//witch DetermineReferenceResolveType(componentId) {
-		if index.config.AllowFileLookup {
+	//case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case.
-			return index.lookupFileReference(id)
+	//	return index.FindComponentInRoot(componentId)
-		} else {
-			return nil, nil, fmt.Errorf("local lookups are not permitted, " +
-				"please set AllowFileLookup to true in the configuration")
-		}
-	}
 
-	switch DetermineReferenceResolveType(componentId) {
+	//case HttpResolve, FileResolve:
-	case LocalResolve: // ideally, every single ref in every single spec is local. however, this is not the case.
+	return index.performExternalLookup(strings.Split(componentId, "#/"))
-		return index.FindComponentInRoot(componentId)
 
-	case HttpResolve:
+	//}
-		uri := strings.Split(componentId, "#")
+	//return nil
-		if len(uri) >= 2 {
-			return index.performExternalLookup(uri, componentId, remoteLookup, parent)
-		}
-		if len(uri) == 1 {
-			// if there is no reference, second segment is empty / has no name
-			// this means there is no component to look-up and the entire file should be pulled in.
-			// to stop all the other code from breaking (that is expecting a component), let's just post-pend
-			// a hash to the end of the componentId and ensure the uri slice is as expected.
-			// described in https://github.com/pb33f/libopenapi/issues/37
-			componentId = fmt.Sprintf("%s#", componentId)
-			uri = append(uri, "")
-			return index.performExternalLookup(uri, componentId, remoteLookup, parent)
-		}
-
-	case FileResolve:
-		uri := strings.Split(componentId, "#")
-		if len(uri) == 2 {
-			return index.performExternalLookup(uri, componentId, fileLookup, parent)
-		}
-		if len(uri) == 1 {
-			// if there is no reference, second segment is empty / has no name
-			// this means there is no component to look-up and the entire file should be pulled in.
-			// to stop all the other code from breaking (that is expecting a component), let's just post-pend
-			// a hash to the end of the componentId and ensure the uri slice is as expected.
-			// described in https://github.com/pb33f/libopenapi/issues/37
-			//
-			// ^^ this same issue was re-reported in file based lookups in vacuum.
-			// more info here: https://github.com/daveshanley/vacuum/issues/225
-			componentId = fmt.Sprintf("%s#", componentId)
-			uri = append(uri, "")
-			return index.performExternalLookup(uri, componentId, fileLookup, parent)
-		}
-	}
-	return nil
 }
 
 var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second}
@@ -107,106 +75,106 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
 
 func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
 	// split string to remove file reference
-	uri := strings.Split(ref, "#")
+	//uri := strings.Split(ref, "#")
-
+	//
-	// have we already seen this remote source?
+	//// have we already seen this remote source?
-	var parsedRemoteDocument *yaml.Node
+	//var parsedRemoteDocument *yaml.Node
-	alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
+	//alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
-
+	//
-	if alreadySeen {
+	//if alreadySeen {
-		parsedRemoteDocument = foundDocument
+	//	parsedRemoteDocument = foundDocument
-	} else {
+	//} else {
-
+	//
-		d := make(chan bool)
+	//	d := make(chan bool)
-		var body []byte
+	//	var body []byte
-		var err error
+	//	var err error
-
+	//
-		go func(uri string) {
+	//	go func(uri string) {
-			bc := make(chan []byte)
+	//		bc := make(chan []byte)
-			ec := make(chan error)
+	//		ec := make(chan error)
-			var getter RemoteURLHandler = httpClient.Get
+	//		var getter = httpClient.Get
-			if index.config != nil && index.config.RemoteURLHandler != nil {
+	//		if index.config != nil && index.config.RemoteURLHandler != nil {
-				getter = index.config.RemoteURLHandler
+	//			getter = index.config.RemoteURLHandler
-			}
+	//		}
-
+	//
-			// if we have a remote handler, use it instead of the default.
+	//		// if we have a remote handler, use it instead of the default.
-			if index.config != nil && index.config.FSHandler != nil {
+	//		if index.config != nil && index.config.FSHandler != nil {
-				go func() {
+	//			go func() {
-					remoteFS := index.config.FSHandler
+	//				remoteFS := index.config.FSHandler
-					remoteFile, rErr := remoteFS.Open(uri)
+	//				remoteFile, rErr := remoteFS.Open(uri)
-					if rErr != nil {
+	//				if rErr != nil {
-						e := fmt.Errorf("unable to open remote file: %s", rErr)
+	//					e := fmt.Errorf("unable to open remote file: %s", rErr)
-						ec <- e
+	//					ec <- e
-						return
+	//					return
-					}
+	//				}
-					b, ioErr := io.ReadAll(remoteFile)
+	//				b, ioErr := io.ReadAll(remoteFile)
-					if ioErr != nil {
+	//				if ioErr != nil {
-						e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
+	//					e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
-						ec <- e
+	//					ec <- e
-						return
+	//					return
-					}
+	//				}
-					bc <- b
+	//				bc <- b
-				}()
+	//			}()
-			} else {
+	//		} else {
-				go getRemoteDoc(getter, uri, bc, ec)
+	//			go getRemoteDoc(getter, uri, bc, ec)
-			}
+	//		}
-			select {
+	//		select {
-			case v := <-bc:
+	//		case v := <-bc:
-				body = v
+	//			body = v
-				break
+	//			break
-			case er := <-ec:
+	//		case er := <-ec:
-				err = er
+	//			err = er
-				break
+	//			break
-			}
+	//		}
-			if len(body) > 0 {
+	//		if len(body) > 0 {
-				var remoteDoc yaml.Node
+	//			var remoteDoc yaml.Node
-				er := yaml.Unmarshal(body, &remoteDoc)
+	//			er := yaml.Unmarshal(body, &remoteDoc)
-				if er != nil {
+	//			if er != nil {
-					err = er
+	//				err = er
-					d <- true
+	//				d <- true
-					return
+	//				return
-				}
+	//			}
-				parsedRemoteDocument = &remoteDoc
+	//			parsedRemoteDocument = &remoteDoc
-				if index.config != nil {
+	//			if index.config != nil {
-					index.config.seenRemoteSources.Store(uri, &remoteDoc)
+	//				index.config.seenRemoteSources.Store(uri, &remoteDoc)
-				}
+	//			}
-			}
+	//		}
-			d <- true
+	//		d <- true
-		}(uri[0])
+	//	}(uri[0])
-
+	//
-		// wait for double go fun.
+	//	// wait for double go fun.
-		<-d
+	//	<-d
-		if err != nil {
+	//	if err != nil {
-			// no bueno.
+	//		// no bueno.
-			return nil, nil, err
+	//		return nil, nil, err
-		}
+	//	}
-	}
+	//}
-
+	//
-	// lookup item from reference by using a path query.
+	//// lookup item from reference by using a path query.
-	var query string
+	//var query string
-	if len(uri) >= 2 {
+	//if len(uri) >= 2 {
-		query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
+	//	query = fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
-	} else {
+	//} else {
-		query = "$"
+	//	query = "$"
-	}
+	//}
-
+	//
-	query, err := url.PathUnescape(query)
+	//query, err := url.PathUnescape(query)
-	if err != nil {
+	//if err != nil {
-		return nil, nil, err
+	//	return nil, nil, err
-	}
+	//}
-
+	//
-	// remove any URL encoding
+	//// remove any URL encoding
-	query = strings.Replace(query, "~1", "./", 1)
+	//query = strings.Replace(query, "~1", "./", 1)
-	query = strings.ReplaceAll(query, "~1", "/")
+	//query = strings.ReplaceAll(query, "~1", "/")
-
+	//
-	path, err := yamlpath.NewPath(query)
+	//path, err := yamlpath.NewPath(query)
-	if err != nil {
+	//if err != nil {
-		return nil, nil, err
+	//	return nil, nil, err
-	}
+	//}
-	result, _ := path.Find(parsedRemoteDocument)
+	//result, _ := path.Find(parsedRemoteDocument)
-	if len(result) == 1 {
+	//if len(result) == 1 {
-		return result[0], parsedRemoteDocument, nil
+	//	return result[0], parsedRemoteDocument, nil
-	}
+	//}
 	return nil, nil, nil
 }
 
@@ -214,73 +182,83 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
 	// split string to remove file reference
 	uri := strings.Split(ref, "#")
 	file := strings.ReplaceAll(uri[0], "file:", "")
-	filePath := filepath.Dir(file)
+	//filePath := filepath.Dir(file)
-	fileName := filepath.Base(file)
+	//fileName := filepath.Base(file)
+	absoluteFileLocation, _ := filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file))
 
-	var parsedRemoteDocument *yaml.Node
+	// extract the document from the rolodex.
-
+	rFile, rError := index.rolodex.Open(absoluteFileLocation)
-	if index.seenRemoteSources[file] != nil {
+	if rError != nil {
-		parsedRemoteDocument = index.seenRemoteSources[file]
+		return nil, nil, rError
-	} else {
-
-		base := index.config.BasePath
-		fileToRead := filepath.Join(base, filePath, fileName)
-		var body []byte
-		var err error
-
-		// if we have an FS handler, use it instead of the default behavior
-		if index.config != nil && index.config.FSHandler != nil {
-			remoteFS := index.config.FSHandler
-			remoteFile, rErr := remoteFS.Open(fileToRead)
-			if rErr != nil {
-				e := fmt.Errorf("unable to open file: %s", rErr)
-				return nil, nil, e
-			}
-			body, err = io.ReadAll(remoteFile)
-			if err != nil {
-				e := fmt.Errorf("unable to read file bytes: %s", err)
-				return nil, nil, e
-			}
-
-		} else {
-
-			// try and read the file off the local file system, if it fails
-			// check for a baseURL and then ask our remote lookup function to go try and get it.
-			body, err = os.ReadFile(fileToRead)
-
-			if err != nil {
-
-				// if we have a baseURL, then we can try and get the file from there.
-				if index.config != nil && index.config.BaseURL != nil {
-
-					u := index.config.BaseURL
-					remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
-					a, b, e := index.lookupRemoteReference(remoteRef)
-					if e != nil {
-						// give up, we can't find the file, not locally, not remotely. It's toast.
-						return nil, nil, e
-					}
-					return a, b, nil
-
-				} else {
-					// no baseURL? then we can't do anything, give up.
-					return nil, nil, err
-				}
-			}
-		}
-		var remoteDoc yaml.Node
-		err = yaml.Unmarshal(body, &remoteDoc)
-		if err != nil {
-			return nil, nil, err
-		}
-		parsedRemoteDocument = &remoteDoc
-		if index.seenLocalSources != nil {
-			index.sourceLock.Lock()
-			index.seenLocalSources[file] = &remoteDoc
-			index.sourceLock.Unlock()
-		}
 	}
 
+	parsedDocument, err := rFile.GetContentAsYAMLNode()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	//if index.seenRemoteSources[file] != nil {
+	//	parsedDocument = index.seenRemoteSources[file]
+	//} else {
+	//
+	//	base := index.config.BasePath
+	//	fileToRead := filepath.Join(base, filePath, fileName)
+	//	var body []byte
+	//	var err error
+	//
+	//	// if we have an FS handler, use it instead of the default behavior
+	//	if index.config != nil && index.config.FSHandler != nil {
+	//		remoteFS := index.config.FSHandler
+	//		remoteFile, rErr := remoteFS.Open(fileToRead)
+	//		if rErr != nil {
+	//			e := fmt.Errorf("unable to open file: %s", rErr)
+	//			return nil, nil, e
+	//		}
+	//		body, err = io.ReadAll(remoteFile)
+	//		if err != nil {
+	//			e := fmt.Errorf("unable to read file bytes: %s", err)
+	//			return nil, nil, e
+	//		}
+	//
+	//	} else {
+	//
+	//		// try and read the file off the local file system, if it fails
+	//		// check for a baseURL and then ask our remote lookup function to go try and get it.
+	//		body, err = os.ReadFile(fileToRead)
+	//
+	//		if err != nil {
+	//
+	//			// if we have a baseURL, then we can try and get the file from there.
+	//			if index.config != nil && index.config.BaseURL != nil {
+	//
+	//				u := index.config.BaseURL
+	//				remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
+	//				a, b, e := index.lookupRemoteReference(remoteRef)
+	//				if e != nil {
+	//					// give up, we can't find the file, not locally, not remotely. It's toast.
+	//					return nil, nil, e
+	//				}
+	//				return a, b, nil
+	//
+	//			} else {
+	//				// no baseURL? then we can't do anything, give up.
+	//				return nil, nil, err
+	//			}
+	//		}
+	//	}
+	//	var remoteDoc yaml.Node
+	//	err = yaml.Unmarshal(body, &remoteDoc)
+	//	if err != nil {
+	//		return nil, nil, err
+	//	}
+	//	parsedDocument = &remoteDoc
+	//	if index.seenLocalSources != nil {
+	//		index.sourceLock.Lock()
+	//		index.seenLocalSources[file] = &remoteDoc
+	//		index.sourceLock.Unlock()
+	//	}
+	//}
+
 	// lookup item from reference by using a path query.
 	var query string
 	if len(uri) >= 2 {
@@ -289,7 +267,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
 		query = "$"
 	}
 
-	query, err := url.PathUnescape(query)
+	query, err = url.PathUnescape(query)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -302,154 +280,214 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
 	if err != nil {
 		return nil, nil, err
 	}
-	result, _ := path.Find(parsedRemoteDocument)
+	result, _ := path.Find(parsedDocument)
 	if len(result) == 1 {
-		return result[0], parsedRemoteDocument, nil
+		return result[0], parsedDocument, nil
 	}
 
-	return nil, parsedRemoteDocument, nil
+	return nil, parsedDocument, nil
 }
 
-func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference {
+func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference {
-	if index.root != nil {
+	// check component for url encoding.
+	if strings.Contains(componentId, "%") {
+		// decode the url.
+		componentId, _ = url.QueryUnescape(componentId)
+	}
 
-		// check component for url encoding.
+	name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId)
-		if strings.Contains(componentId, "%") {
+	path, err := yamlpath.NewPath(friendlySearch)
-			// decode the url.
+	if path == nil || err != nil {
-			componentId, _ = url.QueryUnescape(componentId)
+		return nil // no component found
+	}
+	res, _ := path.Find(root)
+
+	if len(res) == 1 {
+		resNode := res[0]
+		if res[0].Kind == yaml.DocumentNode {
+			resNode = res[0].Content[0]
 		}
 
-		name, friendlySearch := utils.ConvertComponentIdIntoFriendlyPathSearch(componentId)
+		fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId)
-		path, err := yamlpath.NewPath(friendlySearch)
-		if path == nil || err != nil {
-			return nil // no component found
-		}
-		res, _ := path.Find(index.root)
 
-		if len(res) == 1 {
+		// extract properties
-			resNode := res[0]
-			if res[0].Kind == yaml.DocumentNode {
-				resNode = res[0].Content[0]
-			}
-			ref := &Reference{
-				Definition:            componentId,
-				Name:                  name,
-				Node:                  resNode,
-				Path:                  friendlySearch,
-				RequiredRefProperties: index.extractDefinitionRequiredRefProperties(res[0], map[string][]string{}),
-			}
 
-			return ref
+		ref := &Reference{
+			FullDefinition:        fullDef,
+			Definition:            componentId,
+			Name:                  name,
+			Node:                  resNode,
+			Path:                  friendlySearch,
+			RequiredRefProperties: extractDefinitionRequiredRefProperties(resNode, map[string][]string{}),
 		}
+
+		return ref
 	}
 	return nil
 }
 
-func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
+//func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference {
-	lookupFunction ExternalLookupFunction, parent *yaml.Node) *Reference {
+//	if index.root != nil {
+//		return FindComponent(index.root, componentId, )
+//	}
+//	return nil
+//}
+
+func (index *SpecIndex) performExternalLookup(uri []string) *Reference {
+
 	if len(uri) > 0 {
-		index.externalLock.RLock()
-		externalSpecIndex := index.externalSpecIndex[uri[0]]
-		index.externalLock.RUnlock()
 
-		if externalSpecIndex == nil {
+		// split string to remove file reference
-			_, newRoot, err := lookupFunction(componentId)
+		file := strings.ReplaceAll(uri[0], "file:", "")
-			if err != nil {
+		fileName := filepath.Base(file)
-				indexError := &IndexingError{
-					Err:  err,
-					Node: parent,
-					Path: componentId,
-				}
-				index.errorLock.Lock()
-				index.refErrors = append(index.refErrors, indexError)
-				index.errorLock.Unlock()
-				return nil
-			}
 
-			// cool, cool, lets index this spec also. This is a recursive action and will keep going
+		var absoluteFileLocation string
-			// until all remote references have been found.
+		if filepath.IsAbs(file) {
-			var bp *url.URL
+			absoluteFileLocation = file
-			var bd string
+		} else {
-
+			absoluteFileLocation, _ = filepath.Abs(filepath.Join(filepath.Dir(index.specAbsolutePath), file))
-			if index.config.BaseURL != nil {
-				bp = index.config.BaseURL
-			}
-			if index.config.BasePath != "" {
-				bd = index.config.BasePath
-			}
-
-			var path, newBasePath string
-			var newUrl *url.URL
-
-			if bp != nil {
-				path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false)
-				newUrl, _ = url.Parse(path)
-				newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path)))
-			}
-			if bd != "" {
-				if len(uri[0]) > 0 {
-					// if there is no base url defined, but we can know we have been requested remotely,
-					// set the base url to the remote url base path.
-					// first check if the first param is actually a URL
-					io, er := url.ParseRequestURI(uri[0])
-					if er != nil {
-						newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
-					} else {
-						if newUrl == nil || newUrl.String() != io.String() {
-							newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path)))
-						}
-						newBasePath = filepath.Dir(filepath.Join(bd, uri[1]))
-					}
-				} else {
-					newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
-				}
-			}
-
-			if newUrl != nil || newBasePath != "" {
-				newConfig := &SpecIndexConfig{
-					BaseURL:           newUrl,
-					BasePath:          newBasePath,
-					AllowRemoteLookup: index.config.AllowRemoteLookup,
-					AllowFileLookup:   index.config.AllowFileLookup,
-					ParentIndex:       index,
-					seenRemoteSources: index.config.seenRemoteSources,
-					remoteLock:        index.config.remoteLock,
-					uri:               uri,
-				}
-
-				var newIndex *SpecIndex
-				seen := index.SearchAncestryForSeenURI(uri[0])
-				if seen == nil {
-
-					newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
-					index.refLock.Lock()
-					index.externalLock.Lock()
-					index.externalSpecIndex[uri[0]] = newIndex
-					index.externalLock.Unlock()
-					newIndex.relativePath = path
-					newIndex.parentIndex = index
-					index.AddChild(newIndex)
-					index.refLock.Unlock()
-					externalSpecIndex = newIndex
-				} else {
-					externalSpecIndex = seen
-				}
-			}
 		}
 
-		if externalSpecIndex != nil {
+		// extract the document from the rolodex.
-			foundRef := externalSpecIndex.FindComponentInRoot(uri[1])
+		rFile, rError := index.rolodex.Open(absoluteFileLocation)
+		if rError != nil {
+			logger.Error("unable to open rolodex file", "file", absoluteFileLocation, "error", rError)
+			return nil
+		}
+
+		parsedDocument, err := rFile.GetContentAsYAMLNode()
+		if err != nil {
+			logger.Error("unable to parse rolodex file", "file", absoluteFileLocation, "error", err)
+			return nil
+		}
+
+		//fmt.Printf("parsedDocument: %v\n", parsedDocument)
+
+		//index.externalLock.RLock()
+		//externalSpecIndex := index.externalSpecIndex[uri[0]]
+		//index.externalLock.RUnlock()
+
+		//if externalSpecIndex == nil {
+		//	_, newRoot, err := lookupFunction(componentId)
+		//	if err != nil {
+		//		indexError := &IndexingError{
+		//			Err:  err,
+		//			Node: parent,
+		//			Path: componentId,
+		//		}
+		//		index.errorLock.Lock()
+		//		index.refErrors = append(index.refErrors, indexError)
+		//		index.errorLock.Unlock()
+		//		return nil
+		//	}
+		//
+		//	// cool, cool, lets index this spec also. This is a recursive action and will keep going
+		//	// until all remote references have been found.
+		//	var bp *url.URL
+		//	var bd string
+		//
+		//	if index.config.BaseURL != nil {
+		//		bp = index.config.BaseURL
+		//	}
+		//	if index.config.BasePath != "" {
+		//		bd = index.config.BasePath
+		//	}
+		//
+		//	var path, newBasePath string
+		//	var newUrl *url.URL
+		//
+		//	if bp != nil {
+		//		path = GenerateCleanSpecConfigBaseURL(bp, uri[0], false)
+		//		newUrl, _ = url.Parse(path)
+		//		newBasePath = filepath.Dir(filepath.Join(index.config.BasePath, filepath.Dir(newUrl.Path)))
+		//	}
+		//	if bd != "" {
+		//		if len(uri[0]) > 0 {
+		//			// if there is no base url defined, but we can know we have been requested remotely,
+		//			// set the base url to the remote url base path.
+		//			// first check if the first param is actually a URL
+		//			io, er := url.ParseRequestURI(uri[0])
+		//			if er != nil {
+		//				newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
+		//			} else {
+		//				if newUrl == nil || newUrl.String() != io.String() {
+		//					newUrl, _ = url.Parse(fmt.Sprintf("%s://%s%s", io.Scheme, io.Host, filepath.Dir(io.Path)))
+		//				}
+		//				newBasePath = filepath.Dir(filepath.Join(bd, uri[1]))
+		//			}
+		//		} else {
+		//			newBasePath = filepath.Dir(filepath.Join(bd, uri[0]))
+		//		}
+		//	}
+		//
+		//	if newUrl != nil || newBasePath != "" {
+		//		newConfig := &SpecIndexConfig{
+		//			BaseURL:           newUrl,
+		//			BasePath:          newBasePath,
+		//			AllowRemoteLookup: index.config.AllowRemoteLookup,
+		//			AllowFileLookup:   index.config.AllowFileLookup,
+		//			ParentIndex:       index,
+		//			seenRemoteSources: index.config.seenRemoteSources,
+		//			remoteLock:        index.config.remoteLock,
+		//			uri:               uri,
+		//			AvoidBuildIndex:   index.config.AvoidBuildIndex,
+		//		}
+		//
+		//		var newIndex *SpecIndex
+		//		seen := index.SearchAncestryForSeenURI(uri[0])
+		//		if seen == nil {
+		//
+		//			newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
+		//			index.refLock.Lock()
+		//			index.externalLock.Lock()
+		//			index.externalSpecIndex[uri[0]] = newIndex
+		//			index.externalLock.Unlock()
+		//			newIndex.relativePath = path
+		//			newIndex.parentIndex = index
+		//			index.AddChild(newIndex)
+		//			index.refLock.Unlock()
+		//			externalSpecIndex = newIndex
+		//		} else {
+		//			externalSpecIndex = seen
+		//		}
+		//	}
+		//}
+
+		wholeFile := false
+		query := ""
+		if len(uri) < 2 {
+			wholeFile = true
+		} else {
+			query = fmt.Sprintf("#/%s", strings.Replace(uri[1], "~1", "./", 1))
+			query = strings.ReplaceAll(query, "~1", "/")
+		}
+
+		// check if there is a component we want to suck in, or if the
+		// entire root needs to come in.
+		var foundRef *Reference
+		if wholeFile {
+			if parsedDocument.Kind == yaml.DocumentNode {
+				parsedDocument = parsedDocument.Content[0]
+			}
+
+			// TODO: remote locations
+
+			foundRef = &Reference{
+				Definition:            fileName,
+				Name:                  fileName,
+				Node:                  parsedDocument,
+				IsRemote:              true,
+				RemoteLocation:        absoluteFileLocation,
+				Path:                  "$",
+				RequiredRefProperties: extractDefinitionRequiredRefProperties(parsedDocument, map[string][]string{}),
+			}
+			return foundRef
+		} else {
+			foundRef = FindComponent(parsedDocument, query, absoluteFileLocation)
 			if foundRef != nil {
-				nameSegs := strings.Split(uri[1], "/")
+				foundRef.IsRemote = true
-				ref := &Reference{
+				foundRef.RemoteLocation = absoluteFileLocation
-					Definition:     componentId,
+				return foundRef
-					Name:           nameSegs[len(nameSegs)-1],
-					Node:           foundRef.Node,
-					IsRemote:       true,
-					RemoteLocation: componentId,
-					Path:           foundRef.Path,
-				}
-				return ref
 			}
 		}
 	}

index/find_component_test.go

@@ -44,8 +44,6 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) {
 	c.BasePath = "../test_specs"
 	index := NewSpecIndexWithConfig(&rootNode, c)
 	assert.Nil(t, index.uri)
-	assert.NotNil(t, index.children[0].uri)
-	assert.NotNil(t, index.children[0].children[0].uri)
 	assert.NotNil(t, index.SearchIndexForReference("second.yaml#/properties/property2"))
 	assert.NotNil(t, index.SearchIndexForReference("second.yaml"))
 	assert.Nil(t, index.SearchIndexForReference("fourth.yaml"))
@@ -504,3 +502,70 @@ paths:
 	assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
 	assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
 }
+
+func TestSpecIndex_Complex_Local_File_Design(t *testing.T) {
+
+	main := `openapi: 3.1.0
+paths:
+  /anything/circularReference:
+    get:
+      operationId: circularReferenceGet
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "components.yaml#/components/schemas/validCircularReferenceObject"
+  /anything/oneOfCircularReference:
+    get:
+      operationId: oneOfCircularReferenceGet
+      tags:
+        - generation
+      responses:
+        "200":
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: "components.yaml#/components/schemas/oneOfCircularReferenceObject"`
+
+	components := `components:
+  schemas:
+    validCircularReferenceObject:
+      type: object
+      properties:
+        circular:
+          type: array
+          items:
+            $ref: "#/components/schemas/validCircularReferenceObject"
+    oneOfCircularReferenceObject:
+      type: object
+      properties:
+        child:
+          oneOf:
+            - $ref: "#/components/schemas/oneOfCircularReferenceObject"
+            - $ref: "#/components/schemas/simpleObject"
+      required:
+        - child
+    simpleObject:
+      description: "simple"
+      type: object
+      properties:
+        str:
+          type: string
+          description: "A string property."
+          example: "example" `
+
+	_ = os.WriteFile("components.yaml", []byte(components), 0644)
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(main), &rootNode)
+
+	c := CreateOpenAPIIndexConfig()
+	index := NewSpecIndexWithConfig(&rootNode, c)
+
+	assert.Len(t, index.GetReferenceIndexErrors(), 2)
+	assert.Equal(t, `invalid URL escape "%$p"`, index.GetReferenceIndexErrors()[0].Error())
+	assert.Equal(t, "component 'exisiting.yaml#/paths/~1pet~1%$petId%7D/get/parameters' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
+}

index/index_model.go

@@ -11,7 +11,6 @@ import (
 	"os"
 	"sync"
 
-	"golang.org/x/sync/syncmap"
 	"gopkg.in/yaml.v3"
 )
 
@@ -25,15 +24,17 @@ const (
 // Reference is a wrapper around *yaml.Node results to make things more manageable when performing
 // algorithms on data models. the *yaml.Node def is just a bit too low level for tracking state.
 type Reference struct {
-	Definition            string
+	FullDefinition       string
-	Name                  string
+	Definition           string
-	Node                  *yaml.Node
+	Name                 string
-	ParentNode            *yaml.Node
+	Node                 *yaml.Node
-	ParentNodeSchemaType  string // used to determine if the parent node is an array or not.
+	ParentNode           *yaml.Node
-	Resolved              bool
+	ParentNodeSchemaType string // used to determine if the parent node is an array or not.
-	Circular              bool
+	Resolved             bool
-	Seen                  bool
+	Circular             bool
-	IsRemote              bool
+	Seen                 bool
+	IsRemote             bool
+	//FileLocation          string
 	RemoteLocation        string
 	Path                  string              // this won't always be available.
 	RequiredRefProperties map[string][]string // definition names (eg, #/definitions/One) to a list of required properties on this definition which reference that definition
@@ -41,8 +42,9 @@ type Reference struct {
 
 // ReferenceMapped is a helper struct for mapped references put into sequence (we lose the key)
 type ReferenceMapped struct {
-	Reference  *Reference
+	Reference      *Reference
-	Definition string
+	Definition     string
+	FullDefinition string
 }
 
 // SpecIndexConfig is a configuration struct for the SpecIndex introduced in 0.6.0 that provides an expandable
@@ -67,6 +69,7 @@ type SpecIndexConfig struct {
 	// If resolving remotely, the RemoteURLHandler will be used to fetch the remote document.
 	// If not set, the default http client will be used.
 	// Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132
+	// deprecated: Use the Rolodex instead
 	RemoteURLHandler func(url string) (*http.Response, error)
 
 	// FSHandler is an entity that implements the `fs.FS` interface that will be used to fetch local or remote documents.
@@ -81,6 +84,7 @@ type SpecIndexConfig struct {
 	// it also overrides the RemoteURLHandler if set.
 	//
 	// Resolves[#85] https://github.com/pb33f/libopenapi/issues/85
+	// deprecated: Use the Rolodex instead
 	FSHandler fs.FS
 
 	// If resolving locally, the BasePath will be the root from which relative references will be resolved from
@@ -92,12 +96,13 @@ type SpecIndexConfig struct {
 	// exploits, but it's better to be safe than sorry.
 	//
 	// To read more about this, you can find a discussion here: https://github.com/pb33f/libopenapi/pull/64
-	AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
+	//AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
-	AllowFileLookup   bool // Allow file lookups for references. Defaults to false
+	//AllowFileLookup   bool // Allow file lookups for references. Defaults to false
 
 	// ParentIndex allows the index to be created with knowledge of a parent, before being parsed. This allows
-	// a breakglass to be used to prevent loops, checking the tree before recursing down.
+	// a breakglass to be used to prevent loops, checking the tree before cursing down.
-	ParentIndex *SpecIndex
+	// deprecated: Use the Rolodex instead, this is no longer needed, indexes are finite and do not have children.
+	//ParentIndex *SpecIndex
 
 	// If set to true, the index will not be built out, which means only the foundational elements will be
 	// parsed and added to the index. This is useful to avoid building out an index if the specification is
@@ -106,14 +111,40 @@ type SpecIndexConfig struct {
 	// Use the `BuildIndex()` method on the index to build it out once resolved/ready.
 	AvoidBuildIndex bool
 
+	AvoidCircularReferenceCheck bool
+
 	// SpecInfo is a pointer to the SpecInfo struct that contains the root node and the spec version. It's the
 	// struct that was used to create this index.
 	SpecInfo *datamodel.SpecInfo
 
+	// Rolodex is what provides all file and remote based lookups. Without the rolodex, no remote or file lookups
+	// can be used. Normally you won't need to worry about setting this as each root document gets a rolodex
+	// of its own automatically.
+	Rolodex *Rolodex
+
+	SpecAbsolutePath string
+
+	// IgnorePolymorphicCircularReferences will skip over checking for circular references in polymorphic schemas.
+	// A polymorphic schema is any schema that is composed other schemas using references via `oneOf`, `anyOf` of `allOf`.
+	// This is disabled by default, which means polymorphic circular references will be checked.
+	IgnorePolymorphicCircularReferences bool
+
+	// IgnoreArrayCircularReferences will skip over checking for circular references in arrays. Sometimes a circular
+	// reference is required to describe a data-shape correctly. Often those shapes are valid circles if the
+	// type of the schema implementing the loop is an array. An empty array would technically break the loop.
+	// So if libopenapi is returning circular references for this use case, then this option should be enabled.
+	// this is disabled by default, which means array circular references will be checked.
+	IgnoreArrayCircularReferences bool
+
+	// SkipCircularReferenceCheck will skip over checking for circular references. This is disabled by default, which
+	// means circular references will be checked. This is useful for developers building out models that should be
+	// indexed later on.
+	//SkipCircularReferenceCheck bool
+
 	// private fields
-	seenRemoteSources *syncmap.Map
+	//seenRemoteSources *syncmap.Map
-	remoteLock        *sync.Mutex
+	//remoteLock        *sync.Mutex
-	uri               []string
+	uri []string
 }
 
 // CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
@@ -123,10 +154,10 @@ type SpecIndexConfig struct {
 func CreateOpenAPIIndexConfig() *SpecIndexConfig {
 	cw, _ := os.Getwd()
 	return &SpecIndexConfig{
-		BasePath:          cw,
+		BasePath: cw,
-		AllowRemoteLookup: true,
+		//AllowRemoteLookup: true,
-		AllowFileLookup:   true,
+		//AllowFileLookup:   true,
-		seenRemoteSources: &syncmap.Map{},
+		//seenRemoteSources: &syncmap.Map{},
 	}
 }
 
@@ -137,10 +168,10 @@ func CreateOpenAPIIndexConfig() *SpecIndexConfig {
 func CreateClosedAPIIndexConfig() *SpecIndexConfig {
 	cw, _ := os.Getwd()
 	return &SpecIndexConfig{
-		BasePath:          cw,
+		BasePath: cw,
-		AllowRemoteLookup: false,
+		//AllowRemoteLookup: false,
-		AllowFileLookup:   false,
+		//AllowFileLookup:   false,
-		seenRemoteSources: &syncmap.Map{},
+		//seenRemoteSources: &syncmap.Map{},
 	}
 }
 
@@ -148,6 +179,7 @@ func CreateClosedAPIIndexConfig() *SpecIndexConfig {
 // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules,
 // everything is pre-walked if you need it.
 type SpecIndex struct {
+	rolodex                             *Rolodex                                      // the rolodex is used to fetch remote and file based documents.
 	allRefs                             map[string]*Reference                         // all (deduplicated) refs
 	rawSequencedRefs                    []*Reference                                  // all raw references in sequence as they are scanned, not deduped.
 	linesWithRefs                       map[int]bool                                  // lines that link to references.
@@ -183,7 +215,8 @@ type SpecIndex struct {
 	rootSecurity                        []*Reference                                  // root security definitions.
 	rootSecurityNode                    *yaml.Node                                    // root security node.
 	refsWithSiblings                    map[string]Reference                          // references with sibling elements next to them
-	pathRefsLock                        sync.Mutex                                    // create lock for all refs maps, we want to build data as fast as we can
+	pathRefsLock                        sync.RWMutex                                  // create lock for all refs maps, we want to build data as fast as we can
+	operationLock                       sync.Mutex                                    // create lock for operations
 	externalDocumentsCount              int                                           // number of externalDocument nodes found
 	operationTagsCount                  int                                           // number of unique tags in operations
 	globalTagsCount                     int                                           // number of global tags defined
@@ -248,12 +281,17 @@ type SpecIndex struct {
 	componentIndexChan                  chan bool
 	polyComponentIndexChan              chan bool
 
-	// when things get complex (looking at you digital ocean) then we need to know
+	specAbsolutePath string
-	// what we have seen across indexes, so we need to be able to travel back up to the root
+	resolver         *Resolver
-	// cto avoid re-downloading sources.
+
-	parentIndex *SpecIndex
+	//parentIndex *SpecIndex
-	uri         []string
+	uri []string
-	children    []*SpecIndex
+	//children    []*SpecIndex
+}
+
+// GetResolver returns the resolver for this index.
+func (index *SpecIndex) GetResolver() *Resolver {
+	return index.resolver
 }
 
 // GetConfig returns the SpecIndexConfig for this index.
@@ -261,15 +299,15 @@ func (index *SpecIndex) GetConfig() *SpecIndexConfig {
 	return index.config
 }
 
-// AddChild adds a child index to this index, a child index is an index created from a remote or file reference.
+//// AddChild adds a child index to this index, a child index is an index created from a remote or file reference.
-func (index *SpecIndex) AddChild(child *SpecIndex) {
+//func (index *SpecIndex) AddChild(child *SpecIndex) {
-	index.children = append(index.children, child)
+//    index.children = append(index.children, child)
-}
+//}
-
+//
-// GetChildren returns the children of this index.
+//// GetChildren returns the children of this index.
-func (index *SpecIndex) GetChildren() []*SpecIndex {
+//func (index *SpecIndex) GetChildren() []*SpecIndex {
-	return index.children
+//    return index.children
-}
+//}
 
 // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the
 // URI based document. Decides if the reference is local, remote or in a file.

index/index_model_test.go

@@ -8,21 +8,21 @@ import (
 	"testing"
 )
 
-func TestSpecIndex_Children(t *testing.T) {
+//func TestSpecIndex_Children(t *testing.T) {
-	idx1 := new(SpecIndex)
+//	idx1 := new(SpecIndex)
-	idx2 := new(SpecIndex)
+//	idx2 := new(SpecIndex)
-	idx3 := new(SpecIndex)
+//	idx3 := new(SpecIndex)
-	idx4 := new(SpecIndex)
+//	idx4 := new(SpecIndex)
-	idx5 := new(SpecIndex)
+//	idx5 := new(SpecIndex)
-	idx1.AddChild(idx2)
+//	idx1.AddChild(idx2)
-	idx1.AddChild(idx3)
+//	idx1.AddChild(idx3)
-	idx3.AddChild(idx4)
+//	idx3.AddChild(idx4)
-	idx4.AddChild(idx5)
+//	idx4.AddChild(idx5)
-	assert.Equal(t, 2, len(idx1.GetChildren()))
+//	assert.Equal(t, 2, len(idx1.GetChildren()))
-	assert.Equal(t, 1, len(idx3.GetChildren()))
+//	assert.Equal(t, 1, len(idx3.GetChildren()))
-	assert.Equal(t, 1, len(idx4.GetChildren()))
+//	assert.Equal(t, 1, len(idx4.GetChildren()))
-	assert.Equal(t, 0, len(idx5.GetChildren()))
+//	assert.Equal(t, 0, len(idx5.GetChildren()))
-}
+//}
 
 func TestSpecIndex_GetConfig(t *testing.T) {
 	idx1 := new(SpecIndex)

index/resolver.go

@@ -4,11 +4,9 @@
 package index
 
 import (
-	"fmt"
+    "fmt"
-	"strings"
+    "github.com/pb33f/libopenapi/utils"
-
+    "gopkg.in/yaml.v3"
-	"github.com/pb33f/libopenapi/utils"
-	"gopkg.in/yaml.v3"
 )
 
 // ResolvingError represents an issue the resolver had trying to stitch the tree together.
@@ -259,7 +257,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j
 	}
 
 	journey = append(journey, ref)
-	relatives := resolver.extractRelatives(ref.Node, nil, seen, journey, resolve)
+	relatives := resolver.extractRelatives(ref, ref.Node, nil, seen, journey, resolve)
 
 	seen = make(map[string]bool)
 
@@ -362,7 +360,7 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDe
 	return false, visitedDefinitions
 }
 
-func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
+func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.Node,
 	foundRelatives map[string]bool,
 	journey []*Reference, resolve bool) []*Reference {
 
@@ -400,7 +398,7 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
 				//	}
 				//}
 
-				found = append(found, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...)
+				found = append(found, resolver.extractRelatives(ref, n, node, foundRelatives, journey, resolve)...)
 			}
 
 			if i%2 == 0 && n.Value == "$ref" {
@@ -411,9 +409,9 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
 
 				value := node.Content[i+1].Value
 
-				ref := resolver.specIndex.SearchIndexForReference(value)
+				locatedRef := resolver.specIndex.SearchIndexForReference(value)
 
-				if ref == nil {
+				if locatedRef == nil {
 					_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
 					err := &ResolvingError{
 						ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value),
@@ -434,15 +432,9 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
 					}
 				}
 
-				r := &Reference{
+				locatedRef[0].ParentNodeSchemaType = schemaType
-					Definition:           value,
-					Name:                 value,
-					Node:                 node,
-					ParentNode:           parent,
-					ParentNodeSchemaType: schemaType,
-				}
 
-				found = append(found, r)
+				found = append(found, locatedRef[0])
 
 				foundRelatives[value] = true
 			}
@@ -459,30 +451,30 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
 						if _, v := utils.FindKeyNodeTop("items", node.Content[i+1].Content); v != nil {
 							if utils.IsNodeMap(v) {
 								if d, _, l := utils.IsNodeRefValue(v); d {
-									ref := resolver.specIndex.GetMappedReferences()[l]
+									mappedRefs := resolver.specIndex.GetMappedReferences()[l]
-									if ref != nil && !ref.Circular {
+									if mappedRefs != nil && !mappedRefs.Circular {
 										circ := false
 										for f := range journey {
-											if journey[f].Definition == ref.Definition {
+											if journey[f].Definition == mappedRefs.Definition {
 												circ = true
 												break
 											}
 										}
 										if !circ {
-											resolver.VisitReference(ref, foundRelatives, journey, resolve)
+											resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve)
 										} else {
-											loop := append(journey, ref)
+											loop := append(journey, mappedRefs)
 											circRef := &CircularReferenceResult{
 												Journey:             loop,
-												Start:               ref,
+												Start:               mappedRefs,
 												LoopIndex:           i,
-												LoopPoint:           ref,
+												LoopPoint:           mappedRefs,
 												PolymorphicType:     n.Value,
 												IsPolymorphicResult: true,
 											}
 
-											ref.Seen = true
+											mappedRefs.Seen = true
-											ref.Circular = true
+											mappedRefs.Circular = true
 											if resolver.IgnorePoly {
 												resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef)
 											} else {
@@ -501,36 +493,31 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node,
 							v := node.Content[i+1].Content[q]
 							if utils.IsNodeMap(v) {
 								if d, _, l := utils.IsNodeRefValue(v); d {
-									strangs := strings.Split(l, "/#")
+									mappedRefs := resolver.specIndex.GetMappedReferences()[l]
-									if len(strangs) == 2 {
+									if mappedRefs != nil && !mappedRefs.Circular {
-										fmt.Println("wank")
-									}
-
-									ref := resolver.specIndex.GetMappedReferences()[l]
-									if ref != nil && !ref.Circular {
 										circ := false
 										for f := range journey {
-											if journey[f].Definition == ref.Definition {
+											if journey[f].Definition == mappedRefs.Definition {
 												circ = true
 												break
 											}
 										}
 										if !circ {
-											resolver.VisitReference(ref, foundRelatives, journey, resolve)
+											resolver.VisitReference(mappedRefs, foundRelatives, journey, resolve)
 										} else {
-											loop := append(journey, ref)
+											loop := append(journey, mappedRefs)
 
 											circRef := &CircularReferenceResult{
 												Journey:             loop,
-												Start:               ref,
+												Start:               mappedRefs,
 												LoopIndex:           i,
-												LoopPoint:           ref,
+												LoopPoint:           mappedRefs,
 												PolymorphicType:     n.Value,
 												IsPolymorphicResult: true,
 											}
 
-											ref.Seen = true
+											mappedRefs.Seen = true
-											ref.Circular = true
+											mappedRefs.Circular = true
 											if resolver.IgnorePoly {
 												resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef)
 											} else {

index/rolodex.go

@@ -51,17 +51,18 @@ type RolodexFS interface {
 }
 
 type Rolodex struct {
-	localFS          map[string]fs.FS
+	localFS                   map[string]fs.FS
-	remoteFS         map[string]fs.FS
+	remoteFS                  map[string]fs.FS
-	indexed          bool
+	indexed                   bool
-	built            bool
+	built                     bool
-	resolved         bool
+	resolved                  bool
-	circChecked      bool
+	circChecked               bool
-	indexConfig      *SpecIndexConfig
+	indexConfig               *SpecIndexConfig
-	indexingDuration time.Duration
+	indexingDuration          time.Duration
-	indexes          []*SpecIndex
+	indexes                   []*SpecIndex
-	rootIndex        *SpecIndex
+	rootIndex                 *SpecIndex
-	caughtErrors     []error
+	caughtErrors              []error
+	ignoredCircularReferences []*CircularReferenceResult
 }
 
 type rolodexFile struct {
@@ -209,6 +210,10 @@ func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex {
 	return r
 }
 
+func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult {
+	return r.ignoredCircularReferences
+}
+
 func (r *Rolodex) GetIndexingDuration() time.Duration {
 	return r.indexingDuration
 }
@@ -275,10 +280,12 @@ func (r *Rolodex) IndexTheRolodex() error {
 			if copiedConfig.IgnorePolymorphicCircularReferences {
 				resolver.IgnorePolymorphicCircularReferences()
 			}
-			resolvingErrors := resolver.CheckForCircularReferences()
+			//if !copiedConfig.AvoidCircularReferenceCheck {
-			for e := range resolvingErrors {
+			//	resolvingErrors := resolver.CheckForCircularReferences()
-				caughtErrors = append(caughtErrors, resolvingErrors[e])
+			//	for e := range resolvingErrors {
-			}
+			//		caughtErrors = append(caughtErrors, resolvingErrors[e])
+			//	}
+			//}
 
 			if err != nil {
 				errChan <- err
@@ -331,6 +338,13 @@ func (r *Rolodex) IndexTheRolodex() error {
 	if !r.indexConfig.AvoidBuildIndex {
 		for _, idx := range indexBuildQueue {
 			idx.BuildIndex()
+			if r.indexConfig.AvoidCircularReferenceCheck {
+				continue
+			}
+			errs := idx.resolver.CheckForCircularReferences()
+			for e := range errs {
+				caughtErrors = append(caughtErrors, errs[e])
+			}
 		}
 	}
 
@@ -369,6 +383,27 @@ func (r *Rolodex) CheckForCircularReferences() {
 		for e := range resolvingErrors {
 			r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
 		}
+		if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
+			r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
+		}
+		if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
+			r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
+		}
+	}
+}
+
+func (r *Rolodex) Resolve() {
+	if r.rootIndex != nil && r.rootIndex.resolver != nil {
+		resolvingErrors := r.rootIndex.resolver.Resolve()
+		for e := range resolvingErrors {
+			r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
+		}
+		if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
+			r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
+		}
+		if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
+			r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
+		}
 	}
 }
 

index/search_index.go

@@ -3,21 +3,39 @@
 
 package index
 
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
 // SearchIndexForReference searches the index for a reference, first looking through the mapped references
 // and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes
 // extracted when parsing the OpenAPI Spec.
 func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
 
+	absPath := index.specAbsolutePath
+	var roloLookup string
+	uri := strings.Split(ref, "#/")
+	if len(uri) == 2 {
+		if uri[0] != "" {
+			roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0]))
+		}
+		ref = fmt.Sprintf("#/%s", uri[1])
+	}
+
 	if r, ok := index.allMappedRefs[ref]; ok {
 		return []*Reference{r}
 	}
 
 	// TODO: look in the rolodex.
 
+	panic("should not be here")
+	fmt.Println(roloLookup)
 	return nil
 
 	//if r, ok := index.allMappedRefs[ref]; ok {
-	//	return []*Reference{r}
+	//	return []*Reference{r}jh
 	//}
 	//for c := range index.children {
 	//	found := goFindMeSomething(index.children[c], ref)