Add support for custom io.FS remote handler in index #85

A new configuration option is available to the index, a `RemoteHandler` of type  `fs.FS`. If set, will be used as an override to all other remote fetching code.

Signed-off-by: Dave Shanley <dave@quobix.com>

index/find_component.go

@@ -5,7 +5,7 @@ package index
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -99,7 +99,7 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) {
 		return
 	}
 	var body []byte
-	body, _ = ioutil.ReadAll(resp.Body)
+	body, _ = io.ReadAll(resp.Body)
 	d <- body
 	close(e)
 	close(d)
@@ -128,7 +128,28 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
 			if index.config != nil && index.config.RemoteURLHandler != nil {
 				getter = index.config.RemoteURLHandler
 			}
-			go getRemoteDoc(getter, uri, bc, ec)
+
+			// if we have a remote handler, use it instead of the default.
+			if index.config != nil && index.config.RemoteHandler != nil {
+				go func() {
+					remoteFS := index.config.RemoteHandler
+					remoteFile, rErr := remoteFS.Open(uri)
+					if rErr != nil {
+						e := fmt.Errorf("unable to open remote file: %s", rErr)
+						ec <- e
+						return
+					}
+					b, ioErr := io.ReadAll(remoteFile)
+					if ioErr != nil {
+						e := fmt.Errorf("unable to read remote file bytes: %s", ioErr)
+						ec <- e
+						return
+					}
+					bc <- b
+				}()
+			} else {
+				go getRemoteDoc(getter, uri, bc, ec)
+			}
 			select {
 			case v := <-bc:
 				body = v
@@ -137,16 +158,18 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
 				err = er
 				break
 			}
-			var remoteDoc yaml.Node
+			if len(body) > 0 {
-			er := yaml.Unmarshal(body, &remoteDoc)
+				var remoteDoc yaml.Node
-			if er != nil {
+				er := yaml.Unmarshal(body, &remoteDoc)
-				err = er
+				if er != nil {
-				d <- true
+					err = er
-				return
+					d <- true
-			}
+					return
-			parsedRemoteDocument = &remoteDoc
+				}
-			if index.config != nil {
+				parsedRemoteDocument = &remoteDoc
-				index.config.seenRemoteSources.Store(uri, &remoteDoc)
+				if index.config != nil {
+					index.config.seenRemoteSources.Store(uri, &remoteDoc)
+				}
 			}
 			d <- true
 		}(uri[0])

index/find_component_test.go

@@ -4,6 +4,10 @@
 package index
 
 import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -203,3 +207,148 @@ func TestGetRemoteDoc(t *testing.T) {
 		t.Errorf("Expected %v, got %v", expectedData, data)
 	}
 }
+
+type FS struct{}
+type FSBadOpen struct{}
+type FSBadRead struct{}
+
+type file struct {
+	name string
+	data string
+}
+
+type openFile struct {
+	f      *file
+	offset int64
+}
+
+func (f *openFile) Close() error               { return nil }
+func (f *openFile) Stat() (fs.FileInfo, error) { return nil, nil }
+func (f *openFile) Read(b []byte) (int, error) {
+	if f.offset >= int64(len(f.f.data)) {
+		return 0, io.EOF
+	}
+	if f.offset < 0 {
+		return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid}
+	}
+	n := copy(b, f.f.data[f.offset:])
+	f.offset += int64(n)
+	return n, nil
+}
+
+type badFileOpen struct{}
+
+func (f *badFileOpen) Close() error               { return errors.New("bad file close") }
+func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
+func (f *badFileOpen) Read(b []byte) (int, error) {
+	return 0, nil
+}
+
+type badFileRead struct {
+	f      *file
+	offset int64
+}
+
+func (f *badFileRead) Close() error               { return errors.New("bad file close") }
+func (f *badFileRead) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") }
+func (f *badFileRead) Read(b []byte) (int, error) {
+	return 0, fmt.Errorf("bad file read")
+}
+
+func (f FS) Open(name string) (fs.File, error) {
+
+	data := `type: string
+name: something
+in: query`
+
+	return &openFile{&file{"test.yaml", data}, 0}, nil
+}
+
+func (f FSBadOpen) Open(name string) (fs.File, error) {
+	return nil, errors.New("bad file open")
+}
+
+func (f FSBadRead) Open(name string) (fs.File, error) {
+	return &badFileRead{&file{}, 0}, nil
+}
+
+func TestSpecIndex_UseRemoteHandler(t *testing.T) {
+
+	spec := `openapi: 3.1.0
+info:
+  title: Test Remote Handler
+  version: 1.0.0
+paths:
+  /test:
+    get:
+      parameters:
+        - $ref: "https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"`
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(spec), &rootNode)
+
+	c := CreateOpenAPIIndexConfig()
+	c.RemoteHandler = FS{}
+
+	index := NewSpecIndexWithConfig(&rootNode, c)
+
+	// extract crs param from index
+	crsParam := index.GetMappedReferences()["https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"]
+	assert.NotNil(t, crsParam)
+	assert.True(t, crsParam.IsRemote)
+	assert.Equal(t, "string", crsParam.Node.Content[1].Value)
+	assert.Equal(t, "something", crsParam.Node.Content[3].Value)
+	assert.Equal(t, "query", crsParam.Node.Content[5].Value)
+}
+
+func TestSpecIndex_UseRemoteHandler_Error_Open(t *testing.T) {
+
+	spec := `openapi: 3.1.0
+info:
+  title: Test Remote Handler
+  version: 1.0.0
+paths:
+  /test:
+    get:
+      parameters:
+        - $ref: "https://-i-cannot-be-opened.com"`
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(spec), &rootNode)
+
+	c := CreateOpenAPIIndexConfig()
+	c.RemoteHandler = FSBadOpen{}
+	c.RemoteURLHandler = httpClient.Get
+
+	index := NewSpecIndexWithConfig(&rootNode, c)
+
+	assert.Len(t, index.GetReferenceIndexErrors(), 2)
+	assert.Equal(t, "unable to open remote file: bad file open", index.GetReferenceIndexErrors()[0].Error())
+	assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
+}
+
+func TestSpecIndex_UseRemoteHandler_Error_Read(t *testing.T) {
+
+	spec := `openapi: 3.1.0
+info:
+  title: Test Remote Handler
+  version: 1.0.0
+paths:
+  /test:
+    get:
+      parameters:
+        - $ref: "https://-i-cannot-be-opened.com"`
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(spec), &rootNode)
+
+	c := CreateOpenAPIIndexConfig()
+	c.RemoteHandler = FSBadRead{}
+	c.RemoteURLHandler = httpClient.Get
+
+	index := NewSpecIndexWithConfig(&rootNode, c)
+
+	assert.Len(t, index.GetReferenceIndexErrors(), 2)
+	assert.Equal(t, "unable to read remote file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
+	assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
+}

index/index_model.go

@@ -4,6 +4,7 @@
 package index
 
 import (
+	"io/fs"
 	"net/http"
 	"net/url"
 	"os"
@@ -66,6 +67,11 @@ type SpecIndexConfig struct {
 	// Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132
 	RemoteURLHandler func(url string) (*http.Response, error)
 
+	// RemoteHandler is a function that will be used to fetch remote documents, it trumps the RemoteURLHandler
+	// and will be used instead if it is set.
+	// Resolves[#85] https://github.com/pb33f/libopenapi/issues/85
+	RemoteHandler fs.FS
+
 	// If resolving locally, the BasePath will be the root from which relative references will be resolved from
 	BasePath string // set the Base Path for resolving relative references if the spec is exploded.