Working through fix #73

Quite a bit of surgery required.

datamodel/low/extraction_functions.go

@@ -89,6 +89,11 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 			}
 		}
 
+		foundRefs := idx.SearchIndexForReference(rv)
+		if len(foundRefs) > 0 {
+			return foundRefs[0].Node, nil
+		}
+
 		// cant be found? last resort is to try a path lookup
 		_, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv)
 		if friendly != "" {

index/extract_references.go

@@ -1,82 +0,0 @@
-// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
-// SPDX-License-Identifier: MIT
-
-package index
-
-import (
-    "fmt"
-    "github.com/pb33f/libopenapi/utils"
-    "strings"
-)
-
-// ExtractComponentsFromRefs returns located components from references. The returned nodes from here
-// can be used for resolving as they contain the actual object properties.
-func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference {
-    var found []*Reference
-
-    //run this async because when things get recursive, it can take a while
-    c := make(chan bool)
-
-    locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
-        located := index.FindComponent(ref.Definition, ref.Node)
-        if located != nil {
-            index.refLock.Lock()
-            if index.allMappedRefs[ref.Definition] == nil {
-                found = append(found, located)
-                index.allMappedRefs[ref.Definition] = located
-                sequence[refIndex] = &ReferenceMapped{
-                    Reference:  located,
-                    Definition: ref.Definition,
-                }
-            }
-            index.refLock.Unlock()
-        } else {
-
-            _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
-            indexError := &IndexingError{
-                Err:  fmt.Errorf("component '%s' does not exist in the specification", ref.Definition),
-                Node: ref.Node,
-                Path: path,
-            }
-            index.refErrors = append(index.refErrors, indexError)
-        }
-        c <- true
-    }
-
-    var refsToCheck []*Reference
-    for _, ref := range refs {
-
-        // check reference for backslashes (hah yeah seen this too!)
-        if strings.Contains(ref.Definition, "\\") { // this was from blazemeter.com haha!
-            _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
-            indexError := &IndexingError{
-                Err:  fmt.Errorf("component '%s' contains a backslash '\\'. It's not valid", ref.Definition),
-                Node: ref.Node,
-                Path: path,
-            }
-            index.refErrors = append(index.refErrors, indexError)
-            continue
-
-        }
-        refsToCheck = append(refsToCheck, ref)
-    }
-    mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
-    for r := range refsToCheck {
-        // expand our index of all mapped refs
-        go locate(refsToCheck[r], r, mappedRefsInSequence)
-    }
-
-    completedRefs := 0
-    for completedRefs < len(refsToCheck) {
-        select {
-        case <-c:
-            completedRefs++
-        }
-    }
-    for m := range mappedRefsInSequence {
-        if mappedRefsInSequence[m] != nil {
-            index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])
-        }
-    }
-    return found
-}

index/extract_refs.go

@@ -318,3 +318,91 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
     return found
 }
+
+// ExtractComponentsFromRefs returns located components from references. The returned nodes from here
+// can be used for resolving as they contain the actual object properties.
+func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference {
+    var found []*Reference
+
+    //run this async because when things get recursive, it can take a while
+    //c := make(chan bool)
+    //tm := make(chan bool)
+
+    locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
+        located := index.FindComponent(ref.Definition, ref.Node)
+        if located != nil {
+            index.refLock.Lock()
+            if index.allMappedRefs[ref.Definition] == nil {
+                found = append(found, located)
+                index.allMappedRefs[ref.Definition] = located
+                sequence[refIndex] = &ReferenceMapped{
+                    Reference:  located,
+                    Definition: ref.Definition,
+                }
+            }
+            index.refLock.Unlock()
+        } else {
+
+            _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
+            indexError := &IndexingError{
+                Err:  fmt.Errorf("component '%s' does not exist in the specification", ref.Definition),
+                Node: ref.Node,
+                Path: path,
+            }
+            index.refErrors = append(index.refErrors, indexError)
+        }
+        //c <- true
+    }
+
+    var refsToCheck []*Reference
+    for _, ref := range refs {
+
+        // check reference for backslashes (hah yeah seen this too!)
+        if strings.Contains(ref.Definition, "\\") { // this was from blazemeter.com haha!
+            _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
+            indexError := &IndexingError{
+                Err:  fmt.Errorf("component '%s' contains a backslash '\\'. It's not valid", ref.Definition),
+                Node: ref.Node,
+                Path: path,
+            }
+            index.refErrors = append(index.refErrors, indexError)
+            continue
+
+        }
+        refsToCheck = append(refsToCheck, ref)
+    }
+    mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
+    //go func() {
+    //    time.Sleep(20 * time.Second)
+    //    tm <- true
+    //}()
+
+    for r := range refsToCheck {
+        // expand our index of all mapped refs
+        // go locate(refsToCheck[r], r, mappedRefsInSequence)
+        locate(refsToCheck[r], r, mappedRefsInSequence)
+    }
+
+    //completedRefs := 0
+    //for completedRefs < len(refsToCheck) {
+    //    select {
+    //    case <-c:
+    //        completedRefs++
+    //        if completedRefs >= len(refsToCheck) {
+    //            fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck))
+    //            break
+    //        } else {
+    //            //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck))
+    //        }
+    //        //case <-tm:
+    //        //    panic("OH NO")
+    //    }
+    //}
+    for m := range mappedRefsInSequence {
+        if mappedRefsInSequence[m] != nil {
+            index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])
+        }
+    }
+    return found
+}
+

index/find_component.go

@@ -9,7 +9,9 @@ import (
     "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
     "gopkg.in/yaml.v3"
     "io/ioutil"
+    "net/url"
     "strings"
+    "sync"
 )
 
 // FindComponent will locate a component by its reference, returns nil if nothing is found.
@@ -80,17 +82,27 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
     return nil
 }
 
+var n sync.Mutex
+
+var openCalls = 0
+var closedCalls = 0
+var totalCalls = 0
+
 func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
     // split string to remove file reference
     uri := strings.Split(ref, "#")
 
+    // have we already seen this remote source?
     var parsedRemoteDocument *yaml.Node
-    if index.seenRemoteSources[uri[0]] != nil {
+    alreadySeen, foundDocument := index.CheckForSeenRemoteSource(uri[0])
-        parsedRemoteDocument = index.seenRemoteSources[uri[0]]
+
+    if alreadySeen {
+        parsedRemoteDocument = foundDocument
     } else {
-        index.httpLock.Lock()
         resp, err := index.httpClient.Get(uri[0])
-        index.httpLock.Unlock()
+        totalCalls++
+        fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls)
+
         if err != nil {
             return nil, nil, err
         }
@@ -105,9 +117,12 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
             return nil, nil, err
         }
         parsedRemoteDocument = &remoteDoc
-        index.remoteLock.Lock()
+        //n.Lock()
-        index.seenRemoteSources[uri[0]] = &remoteDoc
+        //index.seenRemoteSources[uri[0]] = &remoteDoc
-        index.remoteLock.Unlock()
+        if index.config != nil {
+            index.config.seenRemoteSources[uri[0]] = &remoteDoc
+        }
+        //n.Unlock()
     }
 
     // lookup item from reference by using a path query.
@@ -158,17 +173,25 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
             if index.config != nil && index.config.BaseURL != nil {
 
                 u := index.config.BaseURL
-                remoteRef := fmt.Sprintf("%s://%s%s/%s", u.Scheme, u.Host, u.Path, ref)
+
+                remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
                 a, b, e := index.lookupRemoteReference(remoteRef)
                 if e != nil {
                     // give up, we can't find the file, not locally, not remotely. It's toast.
                     return nil, nil, e
                 }
 
-                // everything looks good, lets just make sure we also add a key to the raw reference name.
+                //// everything looks good, lets just make sure we also add a key to the raw reference name.
-                if _, ok := index.seenRemoteSources[file]; !ok {
+                //if _, ok := index.seenRemoteSources[file]; !ok {
-                    index.seenRemoteSources[file] = b
+                //    if b != nil {
-                }
+                //        //index.seenRemoteSources[ref] = b
+                //    } else {
+                //        panic("oh now")
+                //    }
+                //
+                //} else {
+                //    panic("oh no")
+                //}
 
                 return a, b, nil
 
@@ -184,7 +207,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
             return nil, nil, err
         }
         parsedRemoteDocument = &remoteDoc
-        index.seenRemoteSources[file] = &remoteDoc
+        index.seenLocalSources[file] = &remoteDoc
     }
 
     // lookup item from reference by using a path query.
@@ -238,7 +261,9 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
     lookupFunction ExternalLookupFunction, parent *yaml.Node,
 ) *Reference {
     if len(uri) > 0 {
+        //index.fileLock.Lock()
         externalSpecIndex := index.externalSpecIndex[uri[0]]
+
         if externalSpecIndex == nil {
             _, newRoot, err := lookupFunction(componentId)
             if err != nil {
@@ -253,11 +278,33 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
 
             // cool, cool, lets index this spec also. This is a recursive action and will keep going
             // until all remote references have been found.
-            newIndex := NewSpecIndexWithConfig(newRoot, index.config)
+
-            index.fileLock.Lock()
+            // TODO: start here tomorrow, need to pass in the config to the new spec index.
+            // set the base URL to the path.
+
+            path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false)
+
+            newUrl, e := url.Parse(path)
+            if e == nil {
+                newConfig := &SpecIndexConfig{
+                    BaseURL:           newUrl,
+                    AllowRemoteLookup: index.config.AllowRemoteLookup,
+                    AllowFileLookup:   index.config.AllowFileLookup,
+                    seenRemoteSources: index.config.seenRemoteSources,
+                    remoteLock:        index.config.remoteLock,
+                }
+
+                var newIndex *SpecIndex
+                newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
                 index.externalSpecIndex[uri[0]] = newIndex
-            index.fileLock.Unlock()
+                newIndex.relativePath = path
+                newIndex.parentIndex = index
+                index.AddChild(newIndex)
                 externalSpecIndex = newIndex
+
+            } else {
+                return nil
+            }
         }
 
         foundRef := externalSpecIndex.FindComponentInRoot(uri[1])

index/index_model.go

@@ -66,6 +66,14 @@ type SpecIndexConfig struct {
     // To read more about this, you can find a discussion here: https://github.com/pb33f/libopenapi/pull/64
     AllowRemoteLookup bool // Allow remote lookups for references. Defaults to false
     AllowFileLookup   bool // Allow file lookups for references. Defaults to false
+
+    // Indexes can be deeply nested, depending on how complex each spec is. If creating a sub-index
+    // this will help lookups remain efficient.
+    RootIndex   *SpecIndex // if this is a sub-index, the root knows about everything below.
+    ParentIndex *SpecIndex // Who owns this index?
+
+    seenRemoteSources map[string]*yaml.Node
+    remoteLock        *sync.Mutex
 }
 
 // SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and
@@ -166,16 +174,29 @@ type SpecIndex struct {
     descriptionCount                 int
     summaryCount                     int
     seenRemoteSources                map[string]*yaml.Node
-    remoteLock                          sync.Mutex
+    seenLocalSources                 map[string]*yaml.Node
-    httpLock                            sync.Mutex
+    remoteLock                       sync.RWMutex
+    httpLock                         sync.RWMutex
     fileLock                         sync.Mutex
     refLock                          sync.Mutex
     circularReferences               []*CircularReferenceResult // only available when the resolver has been used.
     allowCircularReferences          bool                       // decide if you want to error out, or allow circular references, default is false.
+    relativePath                     string                     // relative path of the spec file.
     config                           *SpecIndexConfig           // configuration for the index
     httpClient                       *http.Client
     componentIndexChan               chan bool
     polyComponentIndexChan           chan bool
+
+    // when things get complex (looking at you digital ocean) then we need to know
+    // what we have seen across indexes, so we need to be able to travel back up to the root
+    // cto avoid re-downloading sources.
+    rootIndex   *SpecIndex
+    parentIndex *SpecIndex
+    children    []*SpecIndex
+}
+
+func (index *SpecIndex) AddChild(child *SpecIndex) {
+    index.children = append(index.children, child)
 }
 
 // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the

index/index_utils.go

@@ -83,6 +83,7 @@ func boostrapIndexCollections(rootNode *yaml.Node, index *SpecIndex) {
     index.polymorphicRefs = make(map[string]*Reference)
     index.refsWithSiblings = make(map[string]Reference)
     index.seenRemoteSources = make(map[string]*yaml.Node)
+    index.seenLocalSources = make(map[string]*yaml.Node)
     index.opServersRefs = make(map[string]map[string][]*Reference)
     index.httpClient = &http.Client{Timeout: time.Duration(5) * time.Second}
     index.componentIndexChan = make(chan bool)

index/search_index.go

@@ -0,0 +1,18 @@
+// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
+// SPDX-License-Identifier: MIT
+
+package index
+
+func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
+
+    if r, ok := index.allMappedRefs[ref]; ok {
+        return []*Reference{r}
+    }
+    for c := range index.children {
+        found := index.children[c].SearchIndexForReference(ref)
+        if found != nil {
+            return found
+        }
+    }
+    return nil
+}

index/spec_index.go

@@ -26,6 +26,10 @@ import (
 // how the index is set up.
 func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex {
 	index := new(SpecIndex)
+	if config != nil && config.seenRemoteSources == nil {
+		config.seenRemoteSources = make(map[string]*yaml.Node)
+	}
+	config.remoteLock = &sync.Mutex{}
 	index.config = config
 	boostrapIndexCollections(rootNode, index)
 	return createNewIndex(rootNode, index)
@@ -45,6 +49,7 @@ func NewSpecIndex(rootNode *yaml.Node) *SpecIndex {
 	index.config = &SpecIndexConfig{
 		AllowRemoteLookup: true,
 		AllowFileLookup:   true,
+		seenRemoteSources: make(map[string]*yaml.Node),
 	}
 	boostrapIndexCollections(rootNode, index)
 	return createNewIndex(rootNode, index)
@@ -103,7 +108,7 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex {
 	index.GetInlineDuplicateParamCount()
 	index.GetAllDescriptionsCount()
 	index.GetTotalTagsCount()
-
+	index.seenRemoteSources = index.config.seenRemoteSources
 	return index
 }
 
@@ -1132,3 +1137,10 @@ func (index *SpecIndex) GetAllDescriptionsCount() int {
 func (index *SpecIndex) GetAllSummariesCount() int {
 	return len(index.allSummaries)
 }
+
+func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) {
+	if _, ok := index.config.seenRemoteSources[url]; ok {
+		return true, index.config.seenRemoteSources[url]
+	}
+	return false, nil
+}

index/utility_methods.go

@@ -7,6 +7,8 @@ import (
     "fmt"
     "github.com/pb33f/libopenapi/utils"
     "gopkg.in/yaml.v3"
+    "net/url"
+    "strings"
     "sync"
 )
 
@@ -370,3 +372,43 @@ func runIndexFunction(funcs []func() int, wg *sync.WaitGroup) {
         }(wg, cFunc)
     }
 }
+
+func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bool) string {
+
+    cleanedPath := baseURL.Path // not cleaned yet!
+
+    // create a slice of path segments from existing path
+    pathSegs := strings.Split(cleanedPath, "/")
+    dirSegs := strings.Split(dir, "/")
+
+    var cleanedSegs []string
+    if !includeFile {
+        dirSegs = dirSegs[:len(dirSegs)-1]
+    }
+
+    // relative paths are a pain in the ass, damn you digital ocean, use a single spec, and break them
+    // down into services, please don't blast apart specs into a billion shards.
+    if strings.Contains(dir, "../") {
+        for s := range dirSegs {
+            if dirSegs[s] == ".." {
+                // chop off the last segment of the base path.
+                if len(pathSegs) > 0 {
+                    pathSegs = pathSegs[:len(pathSegs)-1]
+                }
+            } else {
+                cleanedSegs = append(cleanedSegs, dirSegs[s])
+            }
+        }
+        cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/"))
+    } else {
+        cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/"))
+    }
+    p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
+    if strings.HasSuffix(p, "/") {
+        p = p[:len(p)-1]
+    }
+    return p
+
+}
+
+

index/utility_methods_test.go

@@ -0,0 +1,27 @@
+// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
+// SPDX-License-Identifier: MIT
+
+package index
+
+import (
+    "github.com/stretchr/testify/assert"
+    "net/url"
+    "testing"
+)
+
+func TestGenerateCleanSpecConfigBaseURL(t *testing.T) {
+    u, _ := url.Parse("https://pb33f.io/things/stuff")
+    path := "."
+    assert.Equal(t, "https://pb33f.io/things/stuff",
+        GenerateCleanSpecConfigBaseURL(u, path, false))
+}
+
+func TestGenerateCleanSpecConfigBaseURL_RelativeDeep(t *testing.T) {
+    u, _ := url.Parse("https://pb33f.io/things/stuff/jazz/cakes/winter/oil")
+    path := "../../../../foo/bar/baz/crap.yaml#thang"
+    assert.Equal(t, "https://pb33f.io/things/stuff/foo/bar/baz",
+        GenerateCleanSpecConfigBaseURL(u, path, false))
+
+    assert.Equal(t, "https://pb33f.io/things/stuff/foo/bar/baz/crap.yaml#thang",
+        GenerateCleanSpecConfigBaseURL(u, path, true))
+}

resolver/resolver.go

@@ -183,6 +183,10 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError {
 
 // VisitReference will visit a reference as part of a journey and will return resolved nodes.
 func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
+	if ref == nil {
+		panic("what?")
+	}
+
 	if ref.Resolved || ref.Seen {
 		return ref.Node.Content
 	}
@@ -198,7 +202,12 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b
 		skip := false
 		for i, j := range journey {
 			if j.Definition == r.Definition {
-				foundDup := resolver.specIndex.GetMappedReferences()[r.Definition]
+
+				var foundDup *index.Reference
+				foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition)
+				if len(foundRefs) > 0 {
+					foundDup = foundRefs[0]
+				}
 
 				var circRef *index.CircularReferenceResult
 				if !foundDup.Circular {
@@ -223,7 +232,11 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b
 		}
 
 		if !skip {
-			original := resolver.specIndex.GetMappedReferences()[r.Definition]
+			var original *index.Reference
+			foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition)
+			if len(foundRefs) > 0 {
+				original = foundRefs[0]
+			}
 			resolved := resolver.VisitReference(original, seen, journey, resolve)
 			if resolve {
 				r.Node.Content = resolved // this is where we perform the actual resolving.
@@ -292,7 +305,7 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
 
 				value := node.Content[i+1].Value
 
-				ref := resolver.specIndex.GetMappedReferences()[value]
+				ref := resolver.specIndex.SearchIndexForReference(value)
 
 				if ref == nil {
 					// TODO handle error, missing ref, can't resolve.