Support for relative links now in place #73

Tests all passing, runs super fast, pulls in every single DigitalOcean spec and parses it. There may be some issues deeper down in the models, but for now high level tests all pass.

datamodel/low/extraction_functions.go

@@ -39,7 +39,6 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re
 		idx.GetAllHeaders,
 		idx.GetAllCallbacks,
 		idx.GetAllLinks,
-		idx.GetAllExternalDocuments,
 		idx.GetAllExamples,
 		idx.GetAllRequestBodies,
 		idx.GetAllResponses,
@@ -51,6 +50,7 @@ func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Re
 // the reference being supplied. If there is a match found, the reference *yaml.Node is returned.
 func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 	if rf, _, rv := utils.IsNodeRefValue(root); rf {
+
 		// run through everything and return as soon as we find a match.
 		// this operates as fast as possible as ever
 		collections := generateIndexCollection(idx)
@@ -89,11 +89,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 			}
 		}
 
+		// perform a search for the reference in the index
 		foundRefs := idx.SearchIndexForReference(rv)
 		if len(foundRefs) > 0 {
 			return foundRefs[0].Node, nil
 		}
 
+		// let's try something else to find our references.
+
 		// cant be found? last resort is to try a path lookup
 		_, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv)
 		if friendly != "" {

datamodel/low/v3/path_item.go

@@ -194,7 +194,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
 
 		var op Operation
 
-		wg.Add(1)
+
 
 		if ok, _, _ := utils.IsNodeRefValue(pathNode); ok {
 			r, err := low.LocateRefNode(pathNode, idx)
@@ -210,7 +210,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
 					pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column)
 			}
 		}
-
+		wg.Add(1)
 		go low.BuildModelAsync(pathNode, &op, &wg, &errors)
 
 		opRef := low.NodeReference[*Operation]{

document_test.go

@@ -113,7 +113,7 @@ paths:
 	assert.NoError(t, err)
 
 	v3Doc, docErr := doc.BuildV3Model()
-	assert.Len(t, docErr, 1)
+	assert.Len(t, docErr, 2)
 	assert.Nil(t, v3Doc)
 }
 

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/stretchr/testify v1.8.0
 	github.com/vmware-labs/yaml-jsonpath v0.3.2
+	golang.org/x/sync v0.1.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -75,7 +75,10 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

index/extract_refs.go

@@ -325,8 +325,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
     var found []*Reference
 
     //run this async because when things get recursive, it can take a while
-    //c := make(chan bool)
+    c := make(chan bool)
-    //tm := make(chan bool)
     
     locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
         located := index.FindComponent(ref.Definition, ref.Node)
@@ -351,7 +350,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
             }
             index.refErrors = append(index.refErrors, indexError)
         }
-        //c <- true
+        c <- true
     }
 
     var refsToCheck []*Reference
@@ -372,32 +371,19 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
         refsToCheck = append(refsToCheck, ref)
     }
     mappedRefsInSequence := make([]*ReferenceMapped, len(refsToCheck))
-    //go func() {
-    //    time.Sleep(20 * time.Second)
-    //    tm <- true
-    //}()
 
     for r := range refsToCheck {
         // expand our index of all mapped refs
-        // go locate(refsToCheck[r], r, mappedRefsInSequence)
+        go locate(refsToCheck[r], r, mappedRefsInSequence)
-        locate(refsToCheck[r], r, mappedRefsInSequence)
     }
 
-    //completedRefs := 0
+    completedRefs := 0
-    //for completedRefs < len(refsToCheck) {
+    for completedRefs < len(refsToCheck) {
-    //    select {
+        select {
-    //    case <-c:
+        case <-c:
-    //        completedRefs++
+            completedRefs++
-    //        if completedRefs >= len(refsToCheck) {
+        }
-    //            fmt.Printf("done parsing on %d of %d refs\n", completedRefs, len(refsToCheck))
+    }
-    //            break
-    //        } else {
-    //            //fmt.Printf("waiting on %d of %d refs\n", completedRefs, len(refsToCheck))
-    //        }
-    //        //case <-tm:
-    //        //    panic("OH NO")
-    //    }
-    //}
     for m := range mappedRefsInSequence {
         if mappedRefsInSequence[m] != nil {
             index.allMappedRefsSequenced = append(index.allMappedRefsSequenced, mappedRefsInSequence[m])

index/find_component.go

@@ -9,9 +9,10 @@ import (
     "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
     "gopkg.in/yaml.v3"
     "io/ioutil"
+    "net/http"
     "net/url"
     "strings"
-    "sync"
+    "time"
 )
 
 // FindComponent will locate a component by its reference, returns nil if nothing is found.
@@ -26,7 +27,7 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
         if index.config.AllowRemoteLookup {
             return index.lookupRemoteReference(id)
         } else {
-            return nil, nil, fmt.Errorf("remote lookups are not premitted, " +
+            return nil, nil, fmt.Errorf("remote lookups are not permitted, " +
                 "please set AllowRemoteLookup to true in the configuration")
         }
     }
@@ -82,11 +83,22 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re
     return nil
 }
 
-var n sync.Mutex
+var httpClient = &http.Client{Timeout: time.Duration(60) * time.Second}
 
-var openCalls = 0
+func getRemoteDoc(u string, d chan []byte, e chan error) {
-var closedCalls = 0
+    resp, err := httpClient.Get(u)
-var totalCalls = 0
+    if err != nil {
+        e <- err
+        close(e)
+        close(d)
+        return
+    }
+    var body []byte
+    body, _ = ioutil.ReadAll(resp.Body)
+    d <- body
+    close(e)
+    close(d)
+}
 
 func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Node, error) {
     // split string to remove file reference
@@ -99,30 +111,43 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
     if alreadySeen {
         parsedRemoteDocument = foundDocument
     } else {
-        resp, err := index.httpClient.Get(uri[0])
-        totalCalls++
-        fmt.Printf("Closed: %s (t: %d)\n", uri[0], totalCalls)
 
-        if err != nil {
+        d := make(chan bool)
-            return nil, nil, err
+        var body []byte
-        }
+        var err error
-        body, err := ioutil.ReadAll(resp.Body)
-        if err != nil {
-            return nil, nil, err
-        }
 
+        go func(uri string) {
+            bc := make(chan []byte)
+            ec := make(chan error)
+            go getRemoteDoc(uri, bc, ec)
+            select {
+            case v := <-bc:
+                body = v
+                break
+            case er := <-ec:
+                err = er
+                break
+            }
             var remoteDoc yaml.Node
-        err = yaml.Unmarshal(body, &remoteDoc)
+            er := yaml.Unmarshal(body, &remoteDoc)
-        if err != nil {
+            if er != nil {
-            return nil, nil, err
+                err = er
+                d <- true
+                return
             }
             parsedRemoteDocument = &remoteDoc
-        //n.Lock()
-        //index.seenRemoteSources[uri[0]] = &remoteDoc
             if index.config != nil {
-            index.config.seenRemoteSources[uri[0]] = &remoteDoc
+                index.config.seenRemoteSources.Store(uri, &remoteDoc)
+            }
+            d <- true
+        }(uri[0])
+
+        // wait for double go fun.
+        <-d
+        if err != nil {
+            // no bueno.
+            return nil, nil, err
         }
-        //n.Unlock()
     }
 
     // lookup item from reference by using a path query.
@@ -163,9 +188,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
 
         // try and read the file off the local file system, if it fails
         // check for a baseURL and then ask our remote lookup function to go try and get it.
-        // index.fileLock.Lock()
         body, err := ioutil.ReadFile(file)
-        // index.fileLock.Unlock()
 
         if err != nil {
 
@@ -173,26 +196,12 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
             if index.config != nil && index.config.BaseURL != nil {
 
                 u := index.config.BaseURL
-
                 remoteRef := GenerateCleanSpecConfigBaseURL(u, ref, true)
                 a, b, e := index.lookupRemoteReference(remoteRef)
                 if e != nil {
                     // give up, we can't find the file, not locally, not remotely. It's toast.
                     return nil, nil, e
                 }
-
-                //// everything looks good, lets just make sure we also add a key to the raw reference name.
-                //if _, ok := index.seenRemoteSources[file]; !ok {
-                //    if b != nil {
-                //        //index.seenRemoteSources[ref] = b
-                //    } else {
-                //        panic("oh now")
-                //    }
-                //
-                //} else {
-                //    panic("oh no")
-                //}
-
                 return a, b, nil
 
             } else {
@@ -207,8 +216,10 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
             return nil, nil, err
         }
         parsedRemoteDocument = &remoteDoc
+        if index.seenLocalSources != nil {
             index.seenLocalSources[file] = &remoteDoc
         }
+    }
 
     // lookup item from reference by using a path query.
     var query string
@@ -279,10 +290,14 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
             // cool, cool, lets index this spec also. This is a recursive action and will keep going
             // until all remote references have been found.
 
-            // TODO: start here tomorrow, need to pass in the config to the new spec index.
+            var j *url.URL
-            // set the base URL to the path.
+            if index.config.BaseURL != nil {
+                j = index.config.BaseURL
+            } else {
+                j, _ = url.Parse(uri[0])
+            }
 
-            path := GenerateCleanSpecConfigBaseURL(index.config.BaseURL, uri[0], false)
+            path := GenerateCleanSpecConfigBaseURL(j, uri[0], false)
 
             newUrl, e := url.Parse(path)
             if e == nil {
@@ -296,10 +311,12 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
 
                 var newIndex *SpecIndex
                 newIndex = NewSpecIndexWithConfig(newRoot, newConfig)
+                index.refLock.Lock()
                 index.externalSpecIndex[uri[0]] = newIndex
                 newIndex.relativePath = path
                 newIndex.parentIndex = index
                 index.AddChild(newIndex)
+                index.refLock.Unlock()
                 externalSpecIndex = newIndex
 
             } else {

index/index_model.go

@@ -4,6 +4,7 @@
 package index
 
 import (
+    "golang.org/x/sync/syncmap"
     "gopkg.in/yaml.v3"
     "net/http"
     "net/url"
@@ -72,10 +73,30 @@ type SpecIndexConfig struct {
     RootIndex   *SpecIndex // if this is a sub-index, the root knows about everything below.
     ParentIndex *SpecIndex // Who owns this index?
 
-    seenRemoteSources map[string]*yaml.Node
+    seenRemoteSources *syncmap.Map
     remoteLock        *sync.Mutex
 }
 
+// CreateOpenAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
+// AllowFileLookup set to true. This is the default behaviour of the index in previous versions of libopenapi. (pre 0.6.0)
+func CreateOpenAPIIndexConfig() *SpecIndexConfig {
+    return &SpecIndexConfig{
+        AllowRemoteLookup: true,
+        AllowFileLookup:   true,
+        seenRemoteSources: &syncmap.Map{},
+    }
+}
+
+// CreateClosedAPIIndexConfig is a helper function to create a new SpecIndexConfig with the AllowRemoteLookup and
+// AllowFileLookup set to false. This is the default behaviour of the index in versions 0.6.0+
+func CreateClosedAPIIndexConfig() *SpecIndexConfig {
+    return &SpecIndexConfig{
+        AllowRemoteLookup: false,
+        AllowFileLookup:   false,
+        seenRemoteSources: &syncmap.Map{},
+    }
+}
+
 // SpecIndex is a complete pre-computed index of the entire specification. Numbers are pre-calculated and
 // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules,
 // everything is pre-walked if you need it.
@@ -175,9 +196,6 @@ type SpecIndex struct {
     summaryCount                        int
     seenRemoteSources                   map[string]*yaml.Node
     seenLocalSources                    map[string]*yaml.Node
-    remoteLock                       sync.RWMutex
-    httpLock                         sync.RWMutex
-    fileLock                         sync.Mutex
     refLock                             sync.Mutex
     circularReferences                  []*CircularReferenceResult // only available when the resolver has been used.
     allowCircularReferences             bool                       // decide if you want to error out, or allow circular references, default is false.
@@ -190,7 +208,6 @@ type SpecIndex struct {
     // when things get complex (looking at you digital ocean) then we need to know
     // what we have seen across indexes, so we need to be able to travel back up to the root
     // cto avoid re-downloading sources.
-    rootIndex   *SpecIndex
     parentIndex *SpecIndex
     children    []*SpecIndex
 }
@@ -199,6 +216,11 @@ func (index *SpecIndex) AddChild(child *SpecIndex) {
     index.children = append(index.children, child)
 }
 
+// GetChildren returns the children of this index.
+func (index *SpecIndex) GetChildren() []*SpecIndex {
+    return index.children
+}
+
 // ExternalLookupFunction is for lookup functions that take a JSONSchema reference and tries to find that node in the
 // URI based document. Decides if the reference is local, remote or in a file.
 type ExternalLookupFunction func(id string) (foundNode *yaml.Node, rootNode *yaml.Node, lookupError error)

index/index_model_test.go

@@ -0,0 +1,25 @@
+// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
+// SPDX-License-Identifier: MIT
+
+package index
+
+import (
+    "github.com/stretchr/testify/assert"
+    "testing"
+)
+
+func TestSpecIndex_Children(t *testing.T) {
+    idx1 := new(SpecIndex)
+    idx2 := new(SpecIndex)
+    idx3 := new(SpecIndex)
+    idx4 := new(SpecIndex)
+    idx5 := new(SpecIndex)
+    idx1.AddChild(idx2)
+    idx1.AddChild(idx3)
+    idx3.AddChild(idx4)
+    idx4.AddChild(idx5)
+    assert.Equal(t, 2, len(idx1.GetChildren()))
+    assert.Equal(t, 1, len(idx3.GetChildren()))
+    assert.Equal(t, 1, len(idx4.GetChildren()))
+    assert.Equal(t, 0, len(idx5.GetChildren()))
+}

index/search_index.go

@@ -3,16 +3,37 @@
 
 package index
 
-func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
+import "gopkg.in/yaml.v3"
 
+// SearchIndexForReference searches the index for a reference, first looking through the mapped references
+// and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes
+// extracted when parsing the OpenAPI Spec.
+func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference {
     if r, ok := index.allMappedRefs[ref]; ok {
+        if r.Node.Kind == yaml.DocumentNode {
+            // the reference is an entire document, so we need to dig down a level and rewire the reference.
+            r.Node = r.Node.Content[0]
+        }
         return []*Reference{r}
     }
+    if r, ok := index.externalSpecIndex[ref]; ok {
+        return []*Reference{
+            {
+                Node:       r.root.Content[0],
+                Name:       ref,
+                Definition: ref,
+            },
+        }
+    }
     for c := range index.children {
-        found := index.children[c].SearchIndexForReference(ref)
+        found := goFindMeSomething(index.children[c], ref)
         if found != nil {
             return found
         }
     }
     return nil
 }
+
+func goFindMeSomething(i *SpecIndex, ref string) []*Reference {
+    return i.SearchIndexForReference(ref)
+}

index/search_index_test.go

@@ -0,0 +1,49 @@
+// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
+// SPDX-License-Identifier: MIT
+
+package index
+
+import (
+    "github.com/stretchr/testify/assert"
+    "gopkg.in/yaml.v3"
+    "io/ioutil"
+    "net/url"
+    "testing"
+)
+
+func TestSpecIndex_SearchIndexForReference(t *testing.T) {
+    petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
+    var rootNode yaml.Node
+    _ = yaml.Unmarshal(petstore, &rootNode)
+
+    c := CreateOpenAPIIndexConfig()
+    idx := NewSpecIndexWithConfig(&rootNode, c)
+
+    ref := idx.SearchIndexForReference("#/components/schemas/Pet")
+    assert.NotNil(t, ref)
+}
+
+func TestSpecIndex_SearchIndexForReferene_ExternalSpecs(t *testing.T) {
+
+    // load up an index with lots of references
+    petstore, _ := ioutil.ReadFile("../test_specs/digitalocean.yaml")
+    var rootNode yaml.Node
+    _ = yaml.Unmarshal(petstore, &rootNode)
+
+    c := CreateOpenAPIIndexConfig()
+    c.BaseURL, _ = url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification")
+    idx := NewSpecIndexWithConfig(&rootNode, c)
+
+    ref := idx.SearchIndexForReference("resources/apps/apps_list_instanceSizes.yml")
+    assert.NotNil(t, ref)
+    assert.Equal(t, "operationId", ref[0].Node.Content[0].Value)
+
+    ref = idx.SearchIndexForReference("examples/ruby/domains_create.yml")
+    assert.NotNil(t, ref)
+    assert.Equal(t, "lang", ref[0].Node.Content[0].Value)
+
+    ref = idx.SearchIndexForReference("../../shared/responses/server_error.yml")
+    assert.NotNil(t, ref)
+    assert.Equal(t, "description", ref[0].Node.Content[0].Value)
+
+}

index/spec_index.go

@@ -16,6 +16,7 @@ import (
 	"fmt"
 	"github.com/pb33f/libopenapi/utils"
 	"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
+	"golang.org/x/sync/syncmap"
 	"gopkg.in/yaml.v3"
 	"strings"
 	"sync"
@@ -27,7 +28,7 @@ import (
 func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex {
 	index := new(SpecIndex)
 	if config != nil && config.seenRemoteSources == nil {
-		config.seenRemoteSources = make(map[string]*yaml.Node)
+		config.seenRemoteSources = &syncmap.Map{}
 	}
 	config.remoteLock = &sync.Mutex{}
 	index.config = config
@@ -46,11 +47,7 @@ func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecI
 //  - https://github.com/pb33f/libopenapi/issues/73
 func NewSpecIndex(rootNode *yaml.Node) *SpecIndex {
 	index := new(SpecIndex)
-	index.config = &SpecIndexConfig{
+	index.config = CreateOpenAPIIndexConfig()
-		AllowRemoteLookup: true,
-		AllowFileLookup:   true,
-		seenRemoteSources: make(map[string]*yaml.Node),
-	}
 	boostrapIndexCollections(rootNode, index)
 	return createNewIndex(rootNode, index)
 }
@@ -108,7 +105,12 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex) *SpecIndex {
 	index.GetInlineDuplicateParamCount()
 	index.GetAllDescriptionsCount()
 	index.GetTotalTagsCount()
-	index.seenRemoteSources = index.config.seenRemoteSources
+
+	// do a copy!
+	index.config.seenRemoteSources.Range(func(k, v any) bool {
+		index.seenRemoteSources[k.(string)] = v.(*yaml.Node)
+		return true
+	})
 	return index
 }
 
@@ -1138,9 +1140,15 @@ func (index *SpecIndex) GetAllSummariesCount() int {
 	return len(index.allSummaries)
 }
 
+// CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it,
+// to avoid making duplicate remote calls for document data.
 func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) {
-	if _, ok := index.config.seenRemoteSources[url]; ok {
+	if index.config == nil || index.config.seenRemoteSources == nil {
-		return true, index.config.seenRemoteSources[url]
+		return false, nil
+	}
+	j, _ := index.config.seenRemoteSources.Load(url)
+	if j != nil {
+		return true, j.(*yaml.Node)
 	}
 	return false, nil
 }

index/spec_index_test.go

@@ -128,7 +128,6 @@ func TestSpecIndex_BaseURLError(t *testing.T) {
 	})
 
 	assert.Len(t, index.GetAllExternalIndexes(), 0)
-	assert.Len(t, index.GetReferenceIndexErrors(), 582)
 }
 
 func TestSpecIndex_k8s(t *testing.T) {
@@ -199,9 +198,9 @@ func TestSpecIndex_XSOAR(t *testing.T) {
 }
 
 func TestSpecIndex_PetstoreV3(t *testing.T) {
-	asana, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
+	petstore, _ := ioutil.ReadFile("../test_specs/petstorev3.json")
 	var rootNode yaml.Node
-	yaml.Unmarshal(asana, &rootNode)
+    yaml.Unmarshal(petstore, &rootNode)
 
 	index := NewSpecIndex(&rootNode)
 
@@ -372,7 +371,10 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
     var rootNode yaml.Node
     yaml.Unmarshal(spec, &rootNode)
 
-	index := NewSpecIndex(&rootNode)
+    index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{
+        AllowRemoteLookup: true,
+        AllowFileLookup:   true,
+    })
 
     assert.Len(t, index.allRefs, 5)
     assert.Len(t, index.allMappedRefs, 5)
@@ -590,7 +592,7 @@ func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadFind(t *testing
 	index.seenRemoteSources = make(map[string]*yaml.Node)
 	index.seenRemoteSources["https://no-hope-for-a-dope.com"] = &yaml.Node{}
     a, b, err := index.lookupRemoteReference("https://no-hope-for-a-dope.com#/hey")
-	assert.NoError(t, err)
+    assert.Error(t, err)
     assert.Nil(t, a)
 	assert.Nil(t, b)
 }
@@ -602,8 +604,8 @@ func TestSpecIndex_lookupRemoteReference_NoComponent(t *testing.T) {
     index.seenRemoteSources["https://api.rest.sh/schemas/ErrorModel.json"] = &yaml.Node{}
     a, b, err := index.lookupRemoteReference("https://api.rest.sh/schemas/ErrorModel.json")
     assert.NoError(t, err)
-	assert.Nil(t, a)
+    assert.NotNil(t, a)
-	assert.Nil(t, b)
+    assert.NotNil(t, b)
 }
 
 // Discovered in issue https://github.com/daveshanley/vacuum/issues/225
@@ -620,6 +622,43 @@ func TestSpecIndex_lookupFileReference_NoComponent(t *testing.T) {
     assert.NotNil(t, b)
 }
 
+func TestSpecIndex_CheckBadURLRef(t *testing.T) {
+
+    yml := `openapi: 3.1.0
+paths:
+  /cakes:
+    post:
+      parameters:
+        - $ref: 'httpsss://badurl'`
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.refErrors, 2)
+}
+
+func TestSpecIndex_CheckBadURLRefNoRemoteAllowed(t *testing.T) {
+
+    yml := `openapi: 3.1.0
+paths:
+  /cakes:
+    post:
+      parameters:
+        - $ref: 'httpsss://badurl'`
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+
+    c := CreateClosedAPIIndexConfig()
+    idx := NewSpecIndexWithConfig(&rootNode, c)
+
+    assert.Len(t, idx.refErrors, 2)
+    assert.Equal(t, "remote lookups are not permitted, "+
+        "please set AllowRemoteLookup to true in the configuration", idx.refErrors[0].Error())
+}
+
 func TestSpecIndex_CheckIndexDiscoversNoComponentLocalFileReference(t *testing.T) {
 
     _ = ioutil.WriteFile("coffee-time.yaml", []byte("name: time for coffee"), 0o664)

index/utility_methods.go

@@ -401,9 +401,20 @@ func GenerateCleanSpecConfigBaseURL(baseURL *url.URL, dir string, includeFile bo
         }
         cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(cleanedSegs, "/"))
     } else {
+        if !strings.HasPrefix(dir, "http") {
+            if len(pathSegs) > 1 || len(dirSegs) > 1 {
                 cleanedPath = fmt.Sprintf("%s/%s", strings.Join(pathSegs, "/"), strings.Join(dirSegs, "/"))
             }
-    p := fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
+        } else {
+            cleanedPath = strings.Join(dirSegs, "/")
+        }
+    }
+    var p string
+    if baseURL.Scheme != "" && !strings.HasPrefix(dir, "http") {
+        p = fmt.Sprintf("%s://%s%s", baseURL.Scheme, baseURL.Host, cleanedPath)
+    } else {
+        p = cleanedPath
+    }
     if strings.HasSuffix(p, "/") {
         p = p[:len(p)-1]
     }

resolver/resolver.go

@@ -38,6 +38,10 @@ type Resolver struct {
 	resolvedRoot       *yaml.Node
 	resolvingErrors    []*ResolvingError
 	circularReferences []*index.CircularReferenceResult
+	referencesVisited  int
+	indexesVisited     int
+	journeysTaken      int
+	relativesSeen      int
 }
 
 // NewResolver will create a new resolver from a *index.SpecIndex
@@ -73,7 +77,6 @@ func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularRefere
 			res = append(res, resolver.circularReferences[i])
 		}
 	}
-
 	return res
 }
 
@@ -93,40 +96,33 @@ func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularRef
 	return res
 }
 
+// GetJourneysTaken returns the number of journeys taken by the resolver
+func (resolver *Resolver) GetJourneysTaken() int {
+	return resolver.journeysTaken
+}
+
+// GetReferenceVisited returns the number of references visited by the resolver
+func (resolver *Resolver) GetReferenceVisited() int {
+	return resolver.referencesVisited
+}
+
+// GetIndexesVisited returns the number of indexes visited by the resolver
+func (resolver *Resolver) GetIndexesVisited() int {
+	return resolver.indexesVisited
+}
+
+//
+func (resolver *Resolver) GetRelativesSeen() int {
+	return resolver.relativesSeen
+}
+
 // Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved.
 // this data can get big, it results in a massive duplication of data. This is a destructive method and will permanently
 // re-organize the node tree. Make sure you have copied your original tree before running this (if you want to preserve
 // original data)
 func (resolver *Resolver) Resolve() []*ResolvingError {
-	mapped := resolver.specIndex.GetMappedReferencesSequenced()
-	mappedIndex := resolver.specIndex.GetMappedReferences()
 
-	for _, ref := range mapped {
+	visitIndex(resolver, resolver.specIndex)
-		seenReferences := make(map[string]bool)
-		var journey []*index.Reference
-		if ref != nil && ref.Reference != nil {
-			ref.Reference.Node.Content = resolver.VisitReference(ref.Reference, seenReferences, journey, true)
-		}
-	}
-
-	schemas := resolver.specIndex.GetAllComponentSchemas()
-	for s, schemaRef := range schemas {
-		if mappedIndex[s] == nil {
-			seenReferences := make(map[string]bool)
-			var journey []*index.Reference
-			schemaRef.Node.Content = resolver.VisitReference(schemaRef, seenReferences, journey, true)
-		}
-	}
-
-	// map everything
-	for _, sequenced := range resolver.specIndex.GetAllSequencedReferences() {
-		locatedDef := mappedIndex[sequenced.Definition]
-		if locatedDef != nil {
-			if !locatedDef.Circular && locatedDef.Seen {
-				sequenced.Node.Content = locatedDef.Node.Content
-			}
-		}
-	}
 
 	for _, circRef := range resolver.circularReferences {
 		// If the circular reference is not required, we can ignore it, as it's a terminable loop rather than an infinite one
@@ -181,12 +177,47 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError {
 	return resolver.resolvingErrors
 }
 
-// VisitReference will visit a reference as part of a journey and will return resolved nodes.
+func visitIndex(res *Resolver, idx *index.SpecIndex) {
-func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
+	mapped := idx.GetMappedReferencesSequenced()
-	if ref == nil {
+	mappedIndex := idx.GetMappedReferences()
-		panic("what?")
+	res.indexesVisited++
+
+	for _, ref := range mapped {
+		seenReferences := make(map[string]bool)
+		var journey []*index.Reference
+		res.journeysTaken++
+		if ref != nil && ref.Reference != nil {
+			ref.Reference.Node.Content = res.VisitReference(ref.Reference, seenReferences, journey, true)
+		}
 	}
 
+	schemas := idx.GetAllComponentSchemas()
+	for s, schemaRef := range schemas {
+		if mappedIndex[s] == nil {
+			seenReferences := make(map[string]bool)
+			var journey []*index.Reference
+			res.journeysTaken++
+			schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true)
+		}
+	}
+
+	// map everything
+	for _, sequenced := range idx.GetAllSequencedReferences() {
+		locatedDef := mappedIndex[sequenced.Definition]
+		if locatedDef != nil {
+			if !locatedDef.Circular && locatedDef.Seen {
+				sequenced.Node.Content = locatedDef.Node.Content
+			}
+		}
+	}
+	for _, c := range idx.GetChildren() {
+		visitIndex(res, c)
+	}
+}
+
+// VisitReference will visit a reference as part of a journey and will return resolved nodes.
+func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node {
+	resolver.referencesVisited++
 	if ref.Resolved || ref.Seen {
 		return ref.Node.Content
 	}
@@ -308,7 +339,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
 				ref := resolver.specIndex.SearchIndexForReference(value)
 
 				if ref == nil {
-					// TODO handle error, missing ref, can't resolve.
 					_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
 					err := &ResolvingError{
 						ErrorRef: fmt.Errorf("cannot resolve reference `%s`, it's missing", value),
@@ -414,6 +444,6 @@ func (resolver *Resolver) extractRelatives(node *yaml.Node,
 			}
 		}
 	}
-
+	resolver.relativesSeen += len(found)
 	return found
 }

resolver/resolver_test.go

@@ -170,19 +170,56 @@ components:
 	assert.Len(t, circ, 0)
 }
 
-// TODO: Test for remote and file references with the option switched on and off
+func TestResolver_ResolveComponents_Missing(t *testing.T) {
-func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
+	yml := `paths:
-	mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
+  /hey:
+    get:
+      responses:
+        "200":
+          $ref: '#/components/schemas/crackers'
+components:
+  schemas:
+    cheese:
+      description: cheese
+      properties:
+        thang:
+          $ref: '#/components/schemas/crackers' 
+    crackers:
+      description: crackers
+      properties:
+        butter:
+          $ref: 'go home, I am drunk'`
+
 	var rootNode yaml.Node
-	yaml.Unmarshal(mixedref, &rootNode)
+	yaml.Unmarshal([]byte(yml), &rootNode)
 
 	index := index.NewSpecIndex(&rootNode)
 
 	resolver := NewResolver(index)
 	assert.NotNil(t, resolver)
 
+	err := resolver.Resolve()
+	assert.Len(t, err, 1)
+	assert.Equal(t, "cannot resolve reference `go home, I am drunk`, it's missing: $go home, I am drunk [18:11]", err[0].Error())
+}
+
+func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
+	mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
+	var rootNode yaml.Node
+	yaml.Unmarshal(mixedref, &rootNode)
+
+	b := index.CreateOpenAPIIndexConfig()
+	idx := index.NewSpecIndexWithConfig(&rootNode, b)
+
+	resolver := NewResolver(idx)
+	assert.NotNil(t, resolver)
+
 	circ := resolver.Resolve()
-	assert.Len(t, circ, 10)
+	assert.Len(t, circ, 0)
+	assert.Equal(t, 5, resolver.GetIndexesVisited())
+	assert.Equal(t, 209, resolver.GetRelativesSeen())
+	assert.Equal(t, 35, resolver.GetJourneysTaken())
+	assert.Equal(t, 62, resolver.GetReferenceVisited())
 }
 
 func TestResolver_ResolveComponents_k8s(t *testing.T) {
@@ -211,7 +248,8 @@ func ExampleNewResolver() {
 	_ = yaml.Unmarshal(stripeBytes, &rootNode)
 
 	// create a new spec index (resolver depends on it)
-	index := index.NewSpecIndex(&rootNode)
+	indexConfig := index.CreateClosedAPIIndexConfig()
+	index := index.NewSpecIndexWithConfig(&rootNode, indexConfig)
 
 	// create a new resolver using the index.
 	resolver := NewResolver(index)