Support for relative links now in place #73

Tests all passing, runs super fast, pulls in every single DigitalOcean spec and parses it. There may be some issues deeper down in the models, but for now high level tests all pass.

datamodel/low/extraction_functions.go

@@ -39,7 +39,6 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re
 		idx.GetAllHeaders,
 		idx.GetAllCallbacks,
 		idx.GetAllLinks,
-		idx.GetAllExternalDocuments,
 		idx.GetAllExamples,
 		idx.GetAllRequestBodies,
 		idx.GetAllResponses,
@@ -51,6 +50,7 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re
 // the reference being supplied. If there is a match found, the reference *yaml.Node is returned.
 func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 	if rf, _, rv := utils.IsNodeRefValue(root); rf {
+
 		// run through everything and return as soon as we find a match.
 		// this operates as fast as possible as ever
 		collections := generateIndexCollection(idx)
@@ -89,11 +89,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 			}
 		}
 
+		// perform a search for the reference in the index
 		foundRefs := idx.SearchIndexForReference(rv)
 		if len(foundRefs) > 0 {
 			return foundRefs[0].Node, nil
 		}
 
+		// let's try something else to find our references.
+
 		// cant be found? last resort is to try a path lookup
 		_, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv)
 		if friendly != "" {

datamodel/low/v3/path_item.go

@@ -194,7 +194,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
 
 		var op Operation
 
-		wg.Add(1)
+
 
 		if ok, _, _ := utils.IsNodeRefValue(pathNode); ok {
 			r, err := low.LocateRefNode(pathNode, idx)
@@ -210,7 +210,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
 					pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column)
 			}
 		}
-
+		wg.Add(1)
 		go low.BuildModelAsync(pathNode, &op, &wg, &errors)
 
 		opRef := low.NodeReference[*Operation]{

document_test.go

@@ -113,7 +113,7 @@ paths:
 	assert.NoError(t, err)
 
 	v3Doc, docErr := doc.BuildV3Model()
-	assert.Len(t, docErr, 1)
+	assert.Len(t, docErr, 2)
 	assert.Nil(t, v3Doc)
 }
 

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/stretchr/testify v1.8.0
 	github.com/vmware-labs/yaml-jsonpath v0.3.2
+	golang.org/x/sync v0.1.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -75,7 +75,10 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

index/extract_refs.go

@@ -325,8 +325,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
     var found []*Reference
 
     //run this async because when things get recursive, it can take a while
-    //c := make(chan bool)
+    c := make(chan bool)
-    //tm := make(chan bool)
     
     locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
         located := index.FindComponent(ref.Definition, ref.Node)
@@ -351,7 +350,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
             }
             index.refErrors = append(index.refErrors, indexError)
         }
-        //c <- true
+        c <- true
     }
 
     var refsToCheck []*Reference
@@ -372,32 +371,19 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
         refsToCheck = append(refsToCheck, ref)
     }
     mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
-    //go func() {
-    //    time.Sleep(20 * time.Second)
-    //    tm <- true
-    //}()
 
     for r := range refsToCheck {
         // expand our index of all mapped refs
-        // go locate(refsToCheck[r], r, mappedRefsInSequence)
+        go locate(refsToCheck[r], r, mappedRefsInSequence)
-        locate(refsToCheck[r], r, mappedRefsInSequence)
     }
 
-    //completedRefs := 0
+    completedRefs := 0
-    //for completedRefs < len(refsToCheck) {
+    for completedRefs < len(refsToCheck) {
-    //    select {
+        select {
-    //    case <-c:
+        case <-c:
-    //        completedRefs++
+            completedRefs++
-    //        if completedRefs >= len(refsToCheck) {
+        }
-    //            fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck))
+    }
-    //            break
-    //        } else {
-    //            //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck))
-    //        }
-    //        //case <-tm:
-    //        //    panic("OH NO")
-    //    }
-    //}
     for m := range mappedRefsInSequence {
         if mappedRefsInSequence[m] != nil {
             index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])

index/find_component.go

@@ -9,9 +9,10 @@ import (
     "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
     "gopkg.in/yaml.v3"
     "io/ioutil"
+    "net/http"
     "net/url"
     "strings"
-    "sync"
+    "time"
 )
 
 // FindComponent will locate a component by its reference, returns nil if nothing is found.
@@ -26,7 +27,7 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
         if index.config.AllowRemoteLookup {
             return index.lookupRemoteReference(id)
         } else {
-            return nil, nil, fmt.Errorf("remote lookups are not premitted, " +
+            return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
                 "please set AllowRemoteLookup to true in the configuration")
         }
     }
@@ -82,11 +83,22 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
     return nil
 }
 
-var n sync.Mutex
+var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second}
 
-var openCalls = 0
+func getRemoteDoc(u string, d chan []byte, e chan error) {
-var closedCalls = 0
+    resp, err := httpClient.Get(u)
-var totalCalls = 0
+    if err != nil {
+        e <- err
+        close(e)
+        close(d)
+        return
+    }
+    var body []byte
+    body, _ = ioutil.ReadAll(resp.Body)
+    d <- body
+    close(e)
+    close(d)
+}
 
 func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
     // split string to remove file reference
@@ -99,30 +111,43 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
     if alreadySeen {
         parsedRemoteDocument = foundDocument
     } else {
-        resp, err := index.httpClient.Get(uri[0])
-        totalCalls++
-        fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls)
 
-        if err != nil {
+        d := make(chan bool)
-            return nil, nil, err
+        var body []byte
-        }
+        var err error
-        body, err := ioutil.ReadAll(resp.Body)
-        if err != nil {
-            return nil, nil, err
-        }
 
-        var remoteDoc yaml.Node
+        go func(uri string) {
-        err = yaml.Unmarshal(body, &remoteDoc)
+            bc := make(chan []byte)
+            ec := make(chan error)
+            go getRemoteDoc(uri, bc, ec)
+            select {
+            case v := <-bc:
+                body = v
+                break
+            case er := <-ec:
+                err = er
+                break
+            }
+            var remoteDoc yaml.Node
+            er := yaml.Unmarshal(body, &remoteDoc)
+            if er != nil {
+                err = er
+                d <- true
+                return
+            }
+            parsedRemoteDocument = &remoteDoc
+            if index.config != nil {
+                index.config.seenRemoteSources.Store(uri, &remoteDoc)
+            }
+            d <- true
+        }(uri[0])
+
+        // wait for double go fun.
+        <-d
         if err != nil {
+            // no bueno.
             return nil, nil, err
         }
-        parsedRemoteDocument = &remoteDoc
-        //n.Lock()
-        //index.seenRemoteSources[uri[0]] = &remoteDoc
-        if index.config != nil {
-            index.config.seenRemoteSources[uri[0]] = &remoteDoc
-        }
-        //n.Unlock()
     }
 
     // lookup item from reference by using a path query.
@@ -163,9 +188,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
 
         // try and read the file off the local file system, if it fails
         // check for a baseURL and then ask our remote lookup function to go try and get it.
-        // index.fileLock.Lock()
         body, err := ioutil.ReadFile(file)
-        // index.fileLock.Unlock()
 
         if err != nil {
 
@@ -173,26 +196,12 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
             if index.config != nil && index.config.BaseURL != nil {
 
                 u := index.config.BaseURL
-
                 remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
                 a, b, e := index.lookupRemoteReference(remoteRef)
                 if e != nil {
                     // give up, we can't find the file, not locally, not remotely. It's toast.
                     return nil, nil, e
                 }
-
-                //// everything looks good, lets just make sure we also add a key to the raw reference name.
-                //if _, ok := index.seenRemoteSources[file]; !ok {
-                //    if b != nil {
-                //        //index.seenRemoteSources[ref] = b
-                //    } else {
-                //        panic("oh now")
-                //    }
-                //
-                //} else {
-                //    panic("oh no")
-                //}
-
                 return a, b, nil
 
             } else {
@@ -207,7 +216,9 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
             return nil, nil, err
         }
         parsedRemoteDocument = &remoteDoc
-        index.seenLocalSources[file] = &remoteDoc
+        if index.seenLocalSources != nil {
+            index.seenLocalSources[file] = &remoteDoc
+        }
     }
 
     // lookup item from reference by using a path query.
@@ -279,10 +290,14 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
             // cool, cool, lets index this spec also. This is a recursive action and will keep going
             // until all remote references have been found.
 
-            // TODO: start here tomorrow, need to pass in the config to the new spec index.
+            var j *url.URL
-            // set the base URL to the path.
+            if index.config.BaseURL != nil {
+                j = index.config.BaseURL
+            } else {
+                j, _ = url.Parse(uri[0])
+            }
 
-            path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false)
+            path := GenerateCleanSpecConfigBaseURL(j, uri[0], false)
 
             newUrl, e := url.Parse(path)
             if e == nil {
@@ -296,10 +311,12 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
 
                 var newIndex *SpecIndex
                 newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
+                index.refLock.Lock()
                 index.externalSpecIndex[uri[0]] = newIndex
                 newIndex.relativePath = path
                 newIndex.parentIndex = index
                 index.AddChild(newIndex)
+                index.refLock.Unlock()
                 externalSpecIndex = newIndex
 
             } else {

index/index_model.go

@@ -4,6 +4,7 @@
 package index
 
 import (
+    "golang.org/x/sync/syncmap"
     "gopkg.in/yaml.v3"
     "net/http"
     "net/url"
@@ -72,10 +73,30 @@ type SpecIndexConfig struct {
     RootIndex   *SpecIndex // if this is a sub-index, the root knows about everything below.
     ParentIndex *SpecIndex // Who owns this index?
 
-    seenRemoteSources map[string]*yaml.Node
+    seenRemoteSources *syncmap.Map
     remoteLock        *sync.Mutex
 }
 
+// CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
+// AllowFileLookup set to true. This is the default behaviour of the index in previous versions of libopenapi. (pre 0.6.0)
+func CreateOpenAPIIndexConfig() *SpecIndexConfig {
+    return &SpecIndexConfig{
+        AllowRemoteLookup: true,
+        AllowFileLookup:   true,
+        seenRemoteSources: &syncmap.Map{},
+    }
+}
+
+// CreateClosedAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
+// AllowFileLookup set to false. This is the default behaviour of the index in versions 0.6.0+
+func CreateClosedAPIIndexConfig() *SpecIndexConfig {
+    return &SpecIndexConfig{
+        AllowRemoteLookup: false,
+        AllowFileLookup:   false,
+        seenRemoteSources: &syncmap.Map{},
+    }
+}
+
 // SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and
 // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules,
 // everything is pre-walked if you need it.
@@ -134,63 +155,59 @@ type SpecIndex struct {
     componentParamCount                 int                                         // number of params defined in components
     componentsInlineParamUniqueCount    int                                         // number of inline params with unique names
     componentsInlineParamDuplicateCount int                                         // number of inline params with duplicate names
-    schemaCount                      int                                            // number of schemas
+    schemaCount                         int                                         // number of schemas
-    refCount                         int                                            // total ref count
+    refCount                            int                                         // total ref count
-    root                             *yaml.Node                                     // the root document
+    root                                *yaml.Node                                  // the root document
-    pathsNode                        *yaml.Node                                     // paths node
+    pathsNode                           *yaml.Node                                  // paths node
-    tagsNode                         *yaml.Node                                     // tags node
+    tagsNode                            *yaml.Node                                  // tags node
-    componentsNode                   *yaml.Node                                     // components node
+    componentsNode                      *yaml.Node                                  // components node
-    parametersNode                   *yaml.Node                                     // components/parameters node
+    parametersNode                      *yaml.Node                                  // components/parameters node
-    allParametersNode                map[string]*Reference                          // all parameters node
+    allParametersNode                   map[string]*Reference                       // all parameters node
-    allParameters                    map[string]*Reference                          // all parameters (components/defs)
+    allParameters                       map[string]*Reference                       // all parameters (components/defs)
-    schemasNode                      *yaml.Node                                     // components/schemas node
+    schemasNode                         *yaml.Node                                  // components/schemas node
-    allInlineSchemaDefinitions       []*Reference                                   // all schemas found in document outside of components (openapi) or definitions (swagger).
+    allInlineSchemaDefinitions          []*Reference                                // all schemas found in document outside of components (openapi) or definitions (swagger).
-    allInlineSchemaObjectDefinitions []*Reference                                   // all schemas that are objects found in document outside of components (openapi) or definitions (swagger).
+    allInlineSchemaObjectDefinitions    []*Reference                                // all schemas that are objects found in document outside of components (openapi) or definitions (swagger).
-    allComponentSchemaDefinitions    map[string]*Reference                          // all schemas found in components (openapi) or definitions (swagger).
+    allComponentSchemaDefinitions       map[string]*Reference                       // all schemas found in components (openapi) or definitions (swagger).
-    securitySchemesNode              *yaml.Node                                     // components/securitySchemes node
+    securitySchemesNode                 *yaml.Node                                  // components/securitySchemes node
-    allSecuritySchemes               map[string]*Reference                          // all security schemes / definitions.
+    allSecuritySchemes                  map[string]*Reference                       // all security schemes / definitions.
-    requestBodiesNode                *yaml.Node                                     // components/requestBodies node
+    requestBodiesNode                   *yaml.Node                                  // components/requestBodies node
-    allRequestBodies                 map[string]*Reference                          // all request bodies
+    allRequestBodies                    map[string]*Reference                       // all request bodies
-    responsesNode                    *yaml.Node                                     // components/responses node
+    responsesNode                       *yaml.Node                                  // components/responses node
-    allResponses                     map[string]*Reference                          // all responses
+    allResponses                        map[string]*Reference                       // all responses
-    headersNode                      *yaml.Node                                     // components/headers node
+    headersNode                         *yaml.Node                                  // components/headers node
-    allHeaders                       map[string]*Reference                          // all headers
+    allHeaders                          map[string]*Reference                       // all headers
-    examplesNode                     *yaml.Node                                     // components/examples node
+    examplesNode                        *yaml.Node                                  // components/examples node
-    allExamples                      map[string]*Reference                          // all components examples
+    allExamples                         map[string]*Reference                       // all components examples
-    linksNode                        *yaml.Node                                     // components/links node
+    linksNode                           *yaml.Node                                  // components/links node
-    allLinks                         map[string]*Reference                          // all links
+    allLinks                            map[string]*Reference                       // all links
-    callbacksNode                    *yaml.Node                                     // components/callbacks node
+    callbacksNode                       *yaml.Node                                  // components/callbacks node
-    allCallbacks                     map[string]*Reference                          // all components examples
+    allCallbacks                        map[string]*Reference                       // all components examples
-    externalDocumentsNode            *yaml.Node                                     // external documents node
+    externalDocumentsNode               *yaml.Node                                  // external documents node
-    allExternalDocuments             map[string]*Reference                          // all external documents
+    allExternalDocuments                map[string]*Reference                       // all external documents
-    externalSpecIndex                map[string]*SpecIndex                          // create a primary index of all external specs and componentIds
+    externalSpecIndex                   map[string]*SpecIndex                       // create a primary index of all external specs and componentIds
-    refErrors                        []error                                        // errors when indexing references
+    refErrors                           []error                                     // errors when indexing references
-    operationParamErrors             []error                                        // errors when indexing parameters
+    operationParamErrors                []error                                     // errors when indexing parameters
-    allDescriptions                  []*DescriptionReference                        // every single description found in the spec.
+    allDescriptions                     []*DescriptionReference                     // every single description found in the spec.
-    allSummaries                     []*DescriptionReference                        // every single summary found in the spec.
+    allSummaries                        []*DescriptionReference                     // every single summary found in the spec.
-    allEnums                         []*EnumReference                               // every single enum found in the spec.
+    allEnums                            []*EnumReference                            // every single enum found in the spec.
-    allObjectsWithProperties         []*ObjectReference                             // every single object with properties found in the spec.
+    allObjectsWithProperties            []*ObjectReference                          // every single object with properties found in the spec.
-    enumCount                        int
+    enumCount                           int
-    descriptionCount                 int
+    descriptionCount                    int
-    summaryCount                     int
+    summaryCount                        int
-    seenRemoteSources                map[string]*yaml.Node
+    seenRemoteSources                   map[string]*yaml.Node
-    seenLocalSources                 map[string]*yaml.Node
+    seenLocalSources                    map[string]*yaml.Node
-    remoteLock                       sync.RWMutex
+    refLock                             sync.Mutex
-    httpLock                         sync.RWMutex
+    circularReferences                  []*CircularReferenceResult // only available when the resolver has been used.
-    fileLock                         sync.Mutex
+    allowCircularReferences             bool                       // decide if you want to error out, or allow circular references, default is false.
-    refLock                          sync.Mutex
+    relativePath                        string                     // relative path of the spec file.
-    circularReferences               []*CircularReferenceResult // only available when the resolver has been used.
+    config                              *SpecIndexConfig           // configuration for the index
-    allowCircularReferences          bool                       // decide if you want to error out, or allow circular references, default is false.
+    httpClient                          *http.Client
-    relativePath                     string                     // relative path of the spec file.
+    componentIndexChan                  chan bool
-    config                           *SpecIndexConfig           // configuration for the index
+    polyComponentIndexChan              chan bool
-    httpClient                       *http.Client
-    componentIndexChan               chan bool
-    polyComponentIndexChan           chan bool
 
     // when things get complex (looking at you digital ocean) then we need to know
     // what we have seen across indexes, so we need to be able to travel back up to the root
     // cto avoid re-downloading sources.
-    rootIndex   *SpecIndex
     parentIndex *SpecIndex
     children    []*SpecIndex
 }
@@ -199,6 +216,11 @@ func (index *SpecIndex) AddChild(child *SpecIndex) {
     index.children = append(index.children, child)
 }
 
+// GetChildren returns the children of this index.
+func (index *SpecIndex) GetChildren() []*SpecIndex {
+    return index.children
+}
+
 // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the
 // URI based document. Decides if the reference is local, remote or in a file.
 type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error)

index/index_model_test.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
+// SPDX-License-Identifier: MIT
+
+package index
+
+import (
+    "github.com/stretchr/testify/assert"
+    "testing"
+)
+
+func TestSpecIndex_Children(t *testing.T) {
+    idx1 := new(SpecIndex)
+    idx2 := new(SpecIndex)
+    idx3 := new(SpecIndex)
+    idx4 := new(SpecIndex)
+    idx5 := new(SpecIndex)
+    idx1.AddChild(idx2)
+    idx1.AddChild(idx3)
+    idx3.AddChild(idx4)
+    idx4.AddChild(idx5)
+    assert.Equal(t, 2, len(idx1.GetChildren()))
+    assert.Equal(t, 1, len(idx3.GetChildren()))
+    assert.Equal(t, 1, len(idx4.GetChildren()))
+    assert.Equal(t, 0, len(idx5.GetChildren()))
+}

index/search_index.go

@@ -3,16 +3,37 @@
 
 package index
 
-func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
+import "gopkg.in/yaml.v3"
 
+// SearchIndexForReference searches the index for a reference, first looking through the mapped references
+// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes
+// extracted when parsing the OpenAPI Spec.
+func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
     if r, ok := index.allMappedRefs[ref]; ok {
+        if r.Node.Kind == yaml.DocumentNode {
+            // the reference is an entire document, so we need to dig down a level and rewire the reference.
+            r.Node = r.Node.Content[0]
+        }
         return []*Reference{r}
     }
+    if r, ok := index.externalSpecIndex[ref]; ok {
+        return []*Reference{
+            {
+                Node:       r.root.Content[0],
+                Name:       ref,
+                Definition: ref,
+            },
+        }
+    }
     for c := range index.children {
-        found := index.children[c].SearchIndexForReference(ref)
+        found := goFindMeSomething(index.children[c], ref)
         if found != nil {
             return found
         }
     }
     return nil
 }
+
+func goFindMeSomething(i *SpecIndex, ref string) []*Reference {
+    return i.SearchIndexForReference(ref)
+}

index/search_index_test.go

@@ -0,0 +1,49 @@
+// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
+// SPDX-License-Identifier: MIT
+
+package index
+
+import (
+    "github.com/stretchr/testify/assert"
+    "gopkg.in/yaml.v3"
+    "io/ioutil"
+    "net/url"
+    "testing"
+)
+
+func TestSpecIndex_SearchIndexForReference(t *testing.T) {
+    petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
+    var rootNode yaml.Node
+    _ = yaml.Unmarshal(petstore, &rootNode)
+
+    c := CreateOpenAPIIndexConfig()
+    idx := NewSpecIndexWithConfig(&rootNode, c)
+
+    ref := idx.SearchIndexForReference("#/components/schemas/Pet")
+    assert.NotNil(t, ref)
+}
+
+func TestSpecIndex_SearchIndexForReferene_ExternalSpecs(t *testing.T) {
+
+    // load up an index with lots of references
+    petstore, _ := ioutil.ReadFile("../test_specs/digitalocean.yaml")
+    var rootNode yaml.Node
+    _ = yaml.Unmarshal(petstore, &rootNode)
+
+    c := CreateOpenAPIIndexConfig()
+    c.BaseURL, _ = url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification")
+    idx := NewSpecIndexWithConfig(&rootNode, c)
+
+    ref := idx.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml")
+    assert.NotNil(t, ref)
+    assert.Equal(t, "operationId", ref[0].Node.Content[0].Value)
+
+    ref = idx.SearchIndexForReference("examples/ruby/domains_create.yml")
+    assert.NotNil(t, ref)
+    assert.Equal(t, "lang", ref[0].Node.Content[0].Value)
+
+    ref = idx.SearchIndexForReference("../../shared/responses/server_error.yml")
+    assert.NotNil(t, ref)
+    assert.Equal(t, "description", ref[0].Node.Content[0].Value)
+
+}

index/spec_index.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"github.com/pb33f/libopenapi/utils"
 	"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
+	"golang.org/x/sync/syncmap"
 	"gopkg.in/yaml.v3"
 	"strings"
 	"sync"
@@ -27,7 +28,7 @@ import (
 func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex {
 	index := new(SpecIndex)
 	if config != nil && config.seenRemoteSources == nil {
-		config.seenRemoteSources = make(map[string]*yaml.Node)
+		config.seenRemoteSources = &syncmap.Map{}
 	}
 	config.remoteLock = &sync.Mutex{}
 	index.config = config
@@ -46,11 +47,7 @@ func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecI
 //  - https://github.com/pb33f/libopenapi/issues/73
 func NewSpecIndex(rootNode *yaml.Node) *SpecIndex {
 	index := new(SpecIndex)
-	index.config = &SpecIndexConfig{
+	index.config = CreateOpenAPIIndexConfig()
-		AllowRemoteLookup: true,
-		AllowFileLookup:   true,
-		seenRemoteSources: make(map[string]*yaml.Node),
-	}
 	boostrapIndexCollections(rootNode, index)
 	return createNewIndex(rootNode, index)
 }
@@ -108,7 +105,12 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex {
 	index.GetInlineDuplicateParamCount()
 	index.GetAllDescriptionsCount()
 	index.GetTotalTagsCount()
-	index.seenRemoteSources = index.config.seenRemoteSources
+
+	// do a copy!
+	index.config.seenRemoteSources.Range(func(k, v any) bool {
+		index.seenRemoteSources[k.(string)] = v.(*yaml.Node)
+		return true
+	})
 	return index
 }
 
@@ -1138,9 +1140,15 @@ func (index *SpecIndex) GetAllSummariesCount() int {
 	return len(index.allSummaries)
 }
 
+// CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it,
+// to avoid making duplicate remote calls for document data.
 func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) {
-	if _, ok := index.config.seenRemoteSources[url]; ok {
+	if index.config == nil || index.config.seenRemoteSources == nil {
-		return true, index.config.seenRemoteSources[url]
+		return false, nil
+	}
+	j, _ := index.config.seenRemoteSources.Load(url)
+	if j != nil {
+		return true, j.(*yaml.Node)
 	}
 	return false, nil
 }

index/spec_index_test.go

@@ -128,7 +128,6 @@ func TestSpecIndex_BaseURLError(t *testing.T) {
 	})
 
 	assert.Len(t, index.GetAllExternalIndexes(), 0)
-	assert.Len(t, index.GetReferenceIndexErrors(), 582)
 }
 
 func TestSpecIndex_k8s(t *testing.T) {
@@ -199,9 +198,9 @@ func TestSpecIndex_XSOAR(t *testing.T) {
 }
 
 func TestSpecIndex_PetstoreV3(t *testing.T) {
-	asana, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
+	petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
 	var rootNode yaml.Node
-	yaml.Unmarshal(asana, &rootNode)
+    yaml.Unmarshal(petstore, &rootNode)
 
 	index := NewSpecIndex(&rootNode)
 
@@ -368,21 +367,24 @@ func TestSpecIndex_NoRoot(t *testing.T) {
 }
 
 func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
-	spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
+    spec, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
-	var rootNode yaml.Node
+    var rootNode yaml.Node
-	yaml.Unmarshal(spec, &rootNode)
+    yaml.Unmarshal(spec, &rootNode)
 
-	index := NewSpecIndex(&rootNode)
+    index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
+        AllowRemoteLookup: true,
+        AllowFileLookup:   true,
+    })
 
-	assert.Len(t, index.allRefs, 5)
+    assert.Len(t, index.allRefs, 5)
-	assert.Len(t, index.allMappedRefs, 5)
+    assert.Len(t, index.allMappedRefs, 5)
-	assert.Equal(t, 5, index.GetPathCount())
+    assert.Equal(t, 5, index.GetPathCount())
-	assert.Equal(t, 5, index.GetOperationCount())
+    assert.Equal(t, 5, index.GetOperationCount())
-	assert.Equal(t, 1, index.GetComponentSchemaCount())
+    assert.Equal(t, 1, index.GetComponentSchemaCount())
-	assert.Equal(t, 2, index.GetGlobalTagsCount())
+    assert.Equal(t, 2, index.GetGlobalTagsCount())
-	assert.Equal(t, 3, index.GetTotalTagsCount())
+    assert.Equal(t, 3, index.GetTotalTagsCount())
-	assert.Equal(t, 2, index.GetOperationTagsCount())
+    assert.Equal(t, 2, index.GetOperationTagsCount())
-	assert.Equal(t, 0, index.GetGlobalLinksCount())
+    assert.Equal(t, 0, index.GetGlobalLinksCount())
 	assert.Equal(t, 0, index.GetComponentParameterCount())
 	assert.Equal(t, 2, index.GetOperationsParameterCount())
 	assert.Equal(t, 1, index.GetInlineDuplicateParamCount())
@@ -589,43 +591,80 @@ func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing
 	index := new(SpecIndex)
 	index.seenRemoteSources = make(map[string]*yaml.Node)
 	index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
-	a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
+    a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
-	assert.NoError(t, err)
+    assert.Error(t, err)
-	assert.Nil(t, a)
+    assert.Nil(t, a)
 	assert.Nil(t, b)
 }
 
 // Discovered in issue https://github.com/pb33f/libopenapi/issues/37
 func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) {
-	index := new(SpecIndex)
+    index := new(SpecIndex)
-	index.seenRemoteSources = make(map[string]*yaml.Node)
+    index.seenRemoteSources = make(map[string]*yaml.Node)
-	index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
+    index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
-	a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
+    a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
-	assert.NoError(t, err)
+    assert.NoError(t, err)
-	assert.Nil(t, a)
+    assert.NotNil(t, a)
-	assert.Nil(t, b)
+    assert.NotNil(t, b)
 }
 
 // Discovered in issue https://github.com/daveshanley/vacuum/issues/225
 func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
 
 	index := new(SpecIndex)
-	_ = ioutil.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664)
+    _ = ioutil.WriteFile("coffee-time.yaml", []byte("time: for coffee"), 0o664)
-	defer os.Remove("coffee-time.yaml")
+    defer os.Remove("coffee-time.yaml")
 
-	index.seenRemoteSources = make(map[string]*yaml.Node)
+    index.seenRemoteSources = make(map[string]*yaml.Node)
-	a, b, err := index.lookupFileReference("coffee-time.yaml")
+    a, b, err := index.lookupFileReference("coffee-time.yaml")
-	assert.NoError(t, err)
+    assert.NoError(t, err)
-	assert.NotNil(t, a)
+    assert.NotNil(t, a)
-	assert.NotNil(t, b)
+    assert.NotNil(t, b)
+}
+
+func TestSpecIndex_CheckBadURLRef(t *testing.T) {
+
+    yml := `openapi: 3.1.0
+paths:
+  /cakes:
+    post:
+      parameters:
+        - $ref: 'httpsss://badurl'`
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.refErrors, 2)
+}
+
+func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) {
+
+    yml := `openapi: 3.1.0
+paths:
+  /cakes:
+    post:
+      parameters:
+        - $ref: 'httpsss://badurl'`
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+
+    c := CreateClosedAPIIndexConfig()
+    idx := NewSpecIndexWithConfig(&rootNode, c)
+
+    assert.Len(t, idx.refErrors, 2)
+    assert.Equal(t, "remote lookups are not permitted, "+
+        "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error())
 }
 
 func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) {
 
-	_ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664)
+    _ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664)
-	defer os.Remove("coffee-time.yaml")
+    defer os.Remove("coffee-time.yaml")
 
-	yml := `openapi: 3.0.3
+    yml := `openapi: 3.0.3
 paths:
   /cakes:
     post:

index/utility_methods.go

@@ -401,9 +401,20 @@ func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bo
         }
         cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/"))
     } else {
-        cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/"))
+        if !strings.HasPrefix(dir, "http") {
+            if len(pathSegs) > 1 || len(dirSegs) > 1 {
+                cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/"))
+            }
+        } else {
+            cleanedPath = strings.Join(dirSegs, "/")
+        }
+    }
+    var p string
+    if baseURL.Scheme != "" && !strings.HasPrefix(dir, "http") {
+        p = fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
+    } else {
+        p = cleanedPath
     }
-    p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
     if strings.HasSuffix(p, "/") {
         p = p[:len(p)-1]
     }

resolver/resolver.go

@@ -38,6 +38,10 @@ type Resolver struct {
 	resolvedRoot       *yaml.Node
 	resolvingErrors    []*ResolvingError
 	circularReferences []*index.CircularReferenceResult
+	referencesVisited  int
+	indexesVisited     int
+	journeysTaken      int
+	relativesSeen      int
 }
 
 // NewResolver will create a new resolver from a *index.SpecIndex
@@ -73,7 +77,6 @@ func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularRefere
 			res = append(res, resolver.circularReferences[i])
 		}
 	}
-
 	return res
 }
 
@@ -93,40 +96,33 @@ func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularRef
 	return res
 }
 
+// GetJourneysTaken returns the number of journeys taken by the resolver
+func (resolver *Resolver) GetJourneysTaken() int {
+	return resolver.journeysTaken
+}
+
+// GetReferenceVisited returns the number of references visited by the resolver
+func (resolver *Resolver) GetReferenceVisited() int {
+	return resolver.referencesVisited
+}
+
+// GetIndexesVisited returns the number of indexes visited by the resolver
+func (resolver *Resolver) GetIndexesVisited() int {
+	return resolver.indexesVisited
+}
+
+//
+func (resolver *Resolver) GetRelativesSeen() int {
+	return resolver.relativesSeen
+}
+
 // Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved.
 // this data can get big, it results in a massive duplication of data. This is a destructive method and will permanently
 // re-organize the node tree. Make sure you have copied your original tree before running this (if you want to preserve
 // original data)
 func (resolver *Resolver) Resolve() []*ResolvingError {
-	mapped := resolver.specIndex.GetMappedReferencesSequenced()
-	mappedIndex := resolver.specIndex.GetMappedReferences()
 
-	for _, ref := range mapped {
+	visitIndex(resolver, resolver.specIndex)
-		seenReferences := make(map[string]bool)
-		var journey []*index.Reference
-		if ref != nil && ref.Reference != nil {
-			ref.Reference.Node.Content = resolver.VisitReference(ref.Reference, seenReferences, journey, true)
-		}
-	}
-
-	schemas := resolver.specIndex.GetAllComponentSchemas()
-	for s, schemaRef := range schemas {
-		if mappedIndex[s] == nil {
-			seenReferences := make(map[string]bool)
-			var journey []*index.Reference
-			schemaRef.Node.Content = resolver.VisitReference(schemaRef, seenReferences, journey, true)
-		}
-	}
-
-	// map everything
-	for _, sequenced := range resolver.specIndex.GetAllSequencedReferences() {
-		locatedDef := mappedIndex[sequenced.Definition]
-		if locatedDef != nil {
-			if !locatedDef.Circular && locatedDef.Seen {
-				sequenced.Node.Content = locatedDef.Node.Content
-			}
-		}
-	}
 
 	for _, circRef := range resolver.circularReferences {
 		// If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one
@@ -181,12 +177,47 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError {
 	return resolver.resolvingErrors
 }
 
-// VisitReference will visit a reference as part of a journey and will return resolved nodes.
+func visitIndex(res *Resolver, idx *index.SpecIndex) {
-func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
+	mapped := idx.GetMappedReferencesSequenced()
-	if ref == nil {
+	mappedIndex := idx.GetMappedReferences()
-		panic("what?")
+	res.indexesVisited++
+
+	for _, ref := range mapped {
+		seenReferences := make(map[string]bool)
+		var journey []*index.Reference
+		res.journeysTaken++
+		if ref != nil && ref.Reference != nil {
+			ref.Reference.Node.Content = res.VisitReference(ref.Reference, seenReferences, journey, true)
+		}
 	}
 
+	schemas := idx.GetAllComponentSchemas()
+	for s, schemaRef := range schemas {
+		if mappedIndex[s] == nil {
+			seenReferences := make(map[string]bool)
+			var journey []*index.Reference
+			res.journeysTaken++
+			schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true)
+		}
+	}
+
+	// map everything
+	for _, sequenced := range idx.GetAllSequencedReferences() {
+		locatedDef := mappedIndex[sequenced.Definition]
+		if locatedDef != nil {
+			if !locatedDef.Circular && locatedDef.Seen {
+				sequenced.Node.Content = locatedDef.Node.Content
+			}
+		}
+	}
+	for _, c := range idx.GetChildren() {
+		visitIndex(res, c)
+	}
+}
+
+// VisitReference will visit a reference as part of a journey and will return resolved nodes.
+func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
+	resolver.referencesVisited++
 	if ref.Resolved || ref.Seen {
 		return ref.Node.Content
 	}
@@ -308,7 +339,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
 				ref := resolver.specIndex.SearchIndexForReference(value)
 
 				if ref == nil {
-					// TODO handle error, missing ref, can't resolve.
 					_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
 					err := &ResolvingError{
 						ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value),
@@ -414,6 +444,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
 			}
 		}
 	}
-
+	resolver.relativesSeen += len(found)
 	return found
 }

resolver/resolver_test.go

@@ -170,19 +170,56 @@ components:
 	assert.Len(t, circ, 0)
 }
 
-// TODO: Test for remote and file references with the option switched on and off
+func TestResolver_ResolveComponents_Missing(t *testing.T) {
-func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
+	yml := `paths:
-	mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
+  /hey:
+    get:
+      responses:
+        "200":
+          $ref: '#/components/schemas/crackers'
+components:
+  schemas:
+    cheese:
+      description: cheese
+      properties:
+        thang:
+          $ref: '#/components/schemas/crackers' 
+    crackers:
+      description: crackers
+      properties:
+        butter:
+          $ref: 'go home, I am drunk'`
+
 	var rootNode yaml.Node
-	yaml.Unmarshal(mixedref, &rootNode)
+	yaml.Unmarshal([]byte(yml), &rootNode)
 
 	index := index.NewSpecIndex(&rootNode)
 
 	resolver := NewResolver(index)
 	assert.NotNil(t, resolver)
 
+	err := resolver.Resolve()
+	assert.Len(t, err, 1)
+	assert.Equal(t, "cannot resolve reference `go home, I am drunk`, it's missing: $go home, I am drunk [18:11]", err[0].Error())
+}
+
+func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
+	mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
+	var rootNode yaml.Node
+	yaml.Unmarshal(mixedref, &rootNode)
+
+	b := index.CreateOpenAPIIndexConfig()
+	idx := index.NewSpecIndexWithConfig(&rootNode, b)
+
+	resolver := NewResolver(idx)
+	assert.NotNil(t, resolver)
+
 	circ := resolver.Resolve()
-	assert.Len(t, circ, 10)
+	assert.Len(t, circ, 0)
+	assert.Equal(t, 5, resolver.GetIndexesVisited())
+	assert.Equal(t, 209, resolver.GetRelativesSeen())
+	assert.Equal(t, 35, resolver.GetJourneysTaken())
+	assert.Equal(t, 62, resolver.GetReferenceVisited())
 }
 
 func TestResolver_ResolveComponents_k8s(t *testing.T) {
@@ -211,7 +248,8 @@ func ExampleNewResolver() {
 	_ = yaml.Unmarshal(stripeBytes, &rootNode)
 
 	// create a new spec index (resolver depends on it)
-	index := index.NewSpecIndex(&rootNode)
+	indexConfig := index.CreateClosedAPIIndexConfig()
+	index := index.NewSpecIndexWithConfig(&rootNode, indexConfig)
 
 	// create a new resolver using the index.
 	resolver := NewResolver(index)