Working through test cases

There are still gaps to be found based on various combinations of crazy references.

Signed-off-by: quobix <dave@quobix.com>

document_examples_test.go

@@ -101,7 +101,7 @@ func ExampleNewDocument_fromWithDocumentConfigurationFailure() {
 	if len(errors) > 0 {
 		fmt.Println("Error building Digital Ocean spec errors reported")
 	}
-	// Output: There are 474 errors logged
+	// Output: There are 475 errors logged
 	//Error building Digital Ocean spec errors reported
 }
 

index/extract_refs.go

@@ -225,7 +225,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 							} else {
 
 								// if the index has a base path, use that to resolve the path
-								if index.config.BasePath != "" {
+								if index.config.BasePath != "" && index.config.BaseURL == nil {
 									abs, _ := filepath.Abs(filepath.Join(index.config.BasePath, uri[0]))
 									if abs != defRoot {
 										abs, _ = filepath.Abs(filepath.Join(defRoot, uri[0]))
@@ -234,13 +234,19 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 									componentName = fmt.Sprintf("#/%s", uri[1])
 								} else {
 									// if the index has a base URL, use that to resolve the path.
-									if index.config.BaseURL != nil {
+									if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) {
 
 										u := *index.config.BaseURL
 										abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0]))
 										u.Path = abs
 										fullDefinitionPath = fmt.Sprintf("%s#/%s", u.String(), uri[1])
 										componentName = fmt.Sprintf("#/%s", uri[1])
+
+									} else {
+
+										abs, _ := filepath.Abs(filepath.Join(defRoot, uri[0]))
+										fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1])
+										componentName = fmt.Sprintf("#/%s", uri[1])
 									}
 								}
 							}

index/find_component.go

@@ -44,6 +44,7 @@ func (index *SpecIndex) FindComponent(componentId string) *Reference {
 		// root search
 		return index.FindComponentInRoot(componentId)
 	}
+	return nil
 }
 
 func FindComponent(root *yaml.Node, componentId, absoluteFilePath string, index *SpecIndex) *Reference {

index/resolver.go

@@ -330,6 +330,8 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDe
 		return false, visitedDefinitions
 	}
 
+	// TODO: pick up here, required ref properties are not extracted correctly.
+
 	for refDefinition := range ref.RequiredRefProperties {
 		r, _ := resolver.specIndex.SearchIndexForReference(refDefinition)
 		if initialRef != nil && initialRef.Definition == r.Definition {
@@ -399,6 +401,7 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
 							u, _ := url.Parse(httpExp[0])
 							abs, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0]))
 							u.Path = abs
+							u.Fragment = ""
 							fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1])
 
 						} else {
@@ -412,7 +415,9 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
 								fileDef := strings.Split(ref.FullDefinition, "#/")
 
 								// extract the location of the ref and build a full def path.
-								fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0]))
+								abs, _ := filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0]))
+								fullDef = fmt.Sprintf("%s#/%s", abs, exp[1])
+
 							}
 
 						}
@@ -518,11 +523,30 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
 						if _, v := utils.FindKeyNodeTop("items", node.Content[i+1].Content); v != nil {
 							if utils.IsNodeMap(v) {
 								if d, _, l := utils.IsNodeRefValue(v); d {
-									mappedRefs := resolver.specIndex.GetMappedReferences()[l]
+
+									// create full definition lookup based on ref.
+									def := ref.FullDefinition
+									exp := strings.Split(ref.FullDefinition, "#/")
+									if len(exp) == 2 {
+										if exp[0] != "" {
+											if !strings.HasPrefix(ref.FullDefinition, "http") {
+												if !filepath.IsAbs(exp[0]) {
+													abs, _ := filepath.Abs(fmt.Sprintf("%s#/%s", filepath.Dir(ref.FullDefinition), exp[0]))
+													def = fmt.Sprintf("%s#/%s", abs, l)
+												}
+											}
+										}
+									} else {
+										if !strings.Contains(ref.FullDefinition, "#") {
+
+										}
+									}
+
+									mappedRefs := resolver.specIndex.GetMappedReferences()[def]
 									if mappedRefs != nil && !mappedRefs.Circular {
 										circ := false
 										for f := range journey {
-											if journey[f].Definition == mappedRefs.Definition {
+											if journey[f].FullDefinition == mappedRefs.FullDefinition {
 												circ = true
 												break
 											}
@@ -560,11 +584,78 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No
 							v := node.Content[i+1].Content[q]
 							if utils.IsNodeMap(v) {
 								if d, _, l := utils.IsNodeRefValue(v); d {
-									mappedRefs := resolver.specIndex.GetMappedReferences()[l]
+
+									// create full definition lookup based on ref.
+									def := l
+									exp := strings.Split(l, "#/")
+									if len(exp) == 2 {
+										if exp[0] != "" {
+											if !strings.HasPrefix(exp[0], "http") {
+												if !filepath.IsAbs(exp[0]) {
+
+													if strings.HasPrefix(ref.FullDefinition, "http") {
+
+														u, _ := url.Parse(ref.FullDefinition)
+														p, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0]))
+														u.Path = p
+														def = fmt.Sprintf("%s#/%s", u.String(), exp[1])
+
+													} else {
+														abs, _ := filepath.Abs(filepath.Join(filepath.Dir(ref.FullDefinition), exp[0]))
+														def = fmt.Sprintf("%s#/%s", abs, exp[1])
+													}
+												}
+											} else {
+												panic("mummmmma mia")
+											}
+
+										} else {
+											if strings.HasPrefix(ref.FullDefinition, "http") {
+												u, _ := url.Parse(ref.FullDefinition)
+												u.Fragment = ""
+												def = fmt.Sprintf("%s#/%s", u.String(), exp[1])
+
+											} else {
+												if strings.HasPrefix(ref.FullDefinition, "#/") {
+													def = fmt.Sprintf("#/%s", exp[1])
+												} else {
+													def = fmt.Sprintf("%s#/%s", ref.FullDefinition, exp[1])
+												}
+											}
+										}
+									} else {
+
+										if strings.HasPrefix(l, "http") {
+											def = l
+										} else {
+											if filepath.IsAbs(l) {
+												def = l
+											} else {
+
+												// check if were dealing with a remote file
+												if strings.HasPrefix(ref.FullDefinition, "http") {
+
+													// split the url.
+													u, _ := url.Parse(ref.FullDefinition)
+													abs, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), l))
+													u.Path = abs
+													u.Fragment = ""
+													def = u.String()
+												} else {
+													lookupRef := strings.Split(ref.FullDefinition, "#/")
+													abs, _ := filepath.Abs(filepath.Join(filepath.Dir(lookupRef[0]), l))
+													def = abs
+												}
+											}
+										}
+										//panic("oh no")
+									}
+
+									mappedRefs, _ := resolver.specIndex.SearchIndexForReference(def)
 									if mappedRefs != nil && !mappedRefs.Circular {
 										circ := false
 										for f := range journey {
-											if journey[f].Definition == mappedRefs.Definition {
+											if journey[f].FullDefinition == mappedRefs.FullDefinition {
 												circ = true
 												break
 											}

index/rolodex.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
 	"time"
 )
@@ -272,6 +273,11 @@ func (r *Rolodex) IndexTheRolodex() error {
 			copiedConfig.AvoidBuildIndex = true // we will build out everything in two steps.
 			idx, err := idxFile.Index(&copiedConfig)
 
+			if err != nil {
+				errChan <- err
+			}
+
+			if err == nil {
 				// for each index, we need a resolver
 				resolver := NewResolver(idx)
 
@@ -282,13 +288,11 @@ func (r *Rolodex) IndexTheRolodex() error {
 				if copiedConfig.IgnorePolymorphicCircularReferences {
 					resolver.IgnorePolymorphicCircularReferences()
 				}
-
-			if err != nil {
-				errChan <- err
-			}
 				indexChan <- idx
 			}
 
+		}
+
 		if lfs, ok := fs.(RolodexFS); ok {
 			wait := false
 			for _, f := range lfs.GetFiles() {
@@ -339,6 +343,11 @@ func (r *Rolodex) IndexTheRolodex() error {
 	// now that we have indexed all the files, we can build the index.
 	r.indexes = indexBuildQueue
 	//if !r.indexConfig.AvoidBuildIndex {
+
+	sort.Slice(indexBuildQueue, func(i, j int) bool {
+		return indexBuildQueue[i].specAbsolutePath < indexBuildQueue[j].specAbsolutePath
+	})
+
 	for _, idx := range indexBuildQueue {
 		idx.BuildIndex()
 		if r.indexConfig.AvoidCircularReferenceCheck {

index/rolodex_remote_loader.go

@@ -264,12 +264,18 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 		// can we block threads.
 		if runtime.GOMAXPROCS(-1) > 2 {
 			i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String())
+
+			f := make(chan *RemoteFile)
+			fwait := func(path string, c chan *RemoteFile) {
 				for {
 					if wf, ko := i.Files.Load(remoteParsedURL.Path); ko {
-					return wf.(*RemoteFile), nil
+						c <- wf.(*RemoteFile)
 					}
 				}
 			}
+			go fwait(remoteParsedURL.Path, f)
+			return <-f, nil
+		}
 	}
 
 	// add to processing
@@ -366,12 +372,16 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 		copiedCfg.BaseURL = newBaseURL
 	}
 	copiedCfg.SpecAbsolutePath = remoteParsedURL.String()
-	idx, idxError := remoteFile.Index(&copiedCfg)
 
 	if len(remoteFile.data) > 0 {
 		i.logger.Debug("successfully loaded file", "file", absolutePath)
 	}
 	//i.seekRelatives(remoteFile)
+	// remove from processing
+	i.ProcessingFiles.Delete(remoteParsedURL.Path)
+	i.Files.Store(absolutePath, remoteFile)
+
+	idx, idxError := remoteFile.Index(&copiedCfg)
 
 	if idxError != nil && idx == nil {
 		i.remoteErrors = append(i.remoteErrors, idxError)
@@ -383,10 +393,6 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 		idx.BuildIndex()
 	}
 
-	// remove from processing
-	i.ProcessingFiles.Delete(remoteParsedURL.Path)
-	i.Files.Store(absolutePath, remoteFile)
-
 	//if !i.remoteRunning {
 	return remoteFile, errors.Join(i.remoteErrors...)
 	//	} else {

index/rolodex_test.go

@@ -245,7 +245,7 @@ components:
 	first := `openapi: 3.1.0
 components:
   schemas:
-    CircleTest:
+    StartTest:
       type: object
       required:
         - muffins
@@ -268,9 +268,9 @@ components:
 		BaseDirectory: baseDir,
 		DirFS:         os.DirFS(baseDir),
 		FileFilters: []string{
-			//		"first.yaml",
-			//		"second.yaml",
-			//		"third.yaml",
+			"first.yaml",
+			"second.yaml",
+			"third.yaml",
 			"fourth.yaml",
 		},
 	}
@@ -300,10 +300,11 @@ components:
 	assert.NoError(t, err)
 	assert.Len(t, rolodex.GetCaughtErrors(), 0)
 
-	// there are two circles. Once when reading the journey from first.yaml, and then a second internal look in second.yaml
+	// there are three circles. Once when reading the journey from first.yaml, and then a second internal look in second.yaml
 	// the index won't find three, because by the time that 'three' has been read, it's already been indexed and the journey
-	// discovered.
-	assert.Len(t, rolodex.GetIgnoredCircularReferences(), 2)
+	// discovered. The third is the entirely 'new' circle that is sucked down via `fourth.yaml` from the simulated remote server, which contains
+	// all the same specs, it's just they are now being sucked in remotely.
+	assert.Len(t, rolodex.GetIgnoredCircularReferences(), 3)
 }
 
 func test_rolodexDeepRefServer(a, b, c []byte) *httptest.Server {

index/search_index.go

@@ -138,7 +138,18 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex
 				// does component exist in the root?
 				node, _ := rFile.GetContentAsYAMLNode()
 				if node != nil {
-					found := idx.FindComponent(ref)
+					var found *Reference
+					exp := strings.Split(ref, "#/")
+					compId := ref
+
+					if len(exp) == 2 {
+						compId = fmt.Sprintf("#/%s", exp[1])
+						found = FindComponent(node, compId, exp[0], idx)
+					}
+					if found == nil {
+						found = idx.FindComponent(ref)
+					}
+
 					if found != nil {
 						idx.cache.Store(ref, found)
 						index.cache.Store(ref, found)

index/utility_methods.go

@@ -103,11 +103,11 @@ func extractDefinitionRequiredRefProperties(schemaNode *yaml.Node, reqRefProps m
 
 // extractRequiredReferenceProperties returns a map of definition names to the property or properties which reference it within a node
 func extractRequiredReferenceProperties(fulldef string, requiredPropDefNode *yaml.Node, propName string, reqRefProps map[string][]string) map[string][]string {
-	isRef, _, defPath := utils.IsNodeRefValue(requiredPropDefNode)
+	isRef, _, _ := utils.IsNodeRefValue(requiredPropDefNode)
 	if !isRef {
 		_, defItems := utils.FindKeyNodeTop("items", requiredPropDefNode.Content)
 		if defItems != nil {
-			isRef, _, defPath = utils.IsNodeRefValue(defItems)
+			isRef, _, _ = utils.IsNodeRefValue(defItems)
 		}
 	}
 
@@ -115,8 +115,10 @@ func extractRequiredReferenceProperties(fulldef string, requiredPropDefNode *yam
 		return reqRefProps
 	}
 
+	defPath := fulldef
+
 	// explode defpath
-	exp := strings.Split(defPath, "#/")
+	exp := strings.Split(fulldef, "#/")
 	if len(exp) == 2 {
 		if exp[0] != "" {
 			if !strings.HasPrefix(exp[0], "http") {
@@ -141,6 +143,7 @@ func extractRequiredReferenceProperties(fulldef string, requiredPropDefNode *yam
 
 			}
 		}
+
 	} else {
 
 		if strings.HasPrefix(exp[0], "http") {