From 3b7cbacc447a8f630f55781ed3d23e564067e7dc Mon Sep 17 00:00:00 2001 From: Dave Shanley Date: Fri, 14 Jul 2023 15:13:16 -0400 Subject: [PATCH] Add support for custom `io.FS` remote handler in index #85 A new configuration option is available to the index, a `RemoteHandler` of type `fs.FS`. If set, will be used as an override to all other remote fetching code. Signed-off-by: Dave Shanley --- index/find_component.go | 49 +++++++++--- index/find_component_test.go | 149 +++++++++++++++++++++++++++++++++++ index/index_model.go | 6 ++ 3 files changed, 191 insertions(+), 13 deletions(-) diff --git a/index/find_component.go b/index/find_component.go index 2626389..0aa6881 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -5,7 +5,7 @@ package index import ( "fmt" - "io/ioutil" + "io" "net/http" "net/url" "os" @@ -99,7 +99,7 @@ func getRemoteDoc(g RemoteURLHandler, u string, d chan []byte, e chan error) { return } var body []byte - body, _ = ioutil.ReadAll(resp.Body) + body, _ = io.ReadAll(resp.Body) d <- body close(e) close(d) @@ -128,7 +128,28 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod if index.config != nil && index.config.RemoteURLHandler != nil { getter = index.config.RemoteURLHandler } - go getRemoteDoc(getter, uri, bc, ec) + + // if we have a remote handler, use it instead of the default. + if index.config != nil && index.config.RemoteHandler != nil { + go func() { + remoteFS := index.config.RemoteHandler + remoteFile, rErr := remoteFS.Open(uri) + if rErr != nil { + e := fmt.Errorf("unable to open remote file: %s", rErr) + ec <- e + return + } + b, ioErr := io.ReadAll(remoteFile) + if ioErr != nil { + e := fmt.Errorf("unable to read remote file bytes: %s", ioErr) + ec <- e + return + } + bc <- b + }() + } else { + go getRemoteDoc(getter, uri, bc, ec) + } select { case v := <-bc: body = v @@ -137,16 +158,18 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod err = er break } - var remoteDoc yaml.Node - er := yaml.Unmarshal(body, &remoteDoc) - if er != nil { - err = er - d <- true - return - } - parsedRemoteDocument = &remoteDoc - if index.config != nil { - index.config.seenRemoteSources.Store(uri, &remoteDoc) + if len(body) > 0 { + var remoteDoc yaml.Node + er := yaml.Unmarshal(body, &remoteDoc) + if er != nil { + err = er + d <- true + return + } + parsedRemoteDocument = &remoteDoc + if index.config != nil { + index.config.seenRemoteSources.Store(uri, &remoteDoc) + } } d <- true }(uri[0]) diff --git a/index/find_component_test.go b/index/find_component_test.go index 65a3051..5f50957 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -4,6 +4,10 @@ package index import ( + "errors" + "fmt" + "io" + "io/fs" "net/http" "net/http/httptest" "os" @@ -203,3 +207,148 @@ func TestGetRemoteDoc(t *testing.T) { t.Errorf("Expected %v, got %v", expectedData, data) } } + +type FS struct{} +type FSBadOpen struct{} +type FSBadRead struct{} + +type file struct { + name string + data string +} + +type openFile struct { + f *file + offset int64 +} + +func (f *openFile) Close() error { return nil } +func (f *openFile) Stat() (fs.FileInfo, error) { return nil, nil } +func (f *openFile) Read(b []byte) (int, error) { + if f.offset >= int64(len(f.f.data)) { + return 0, io.EOF + } + if f.offset < 0 { + return 0, &fs.PathError{Op: "read", Path: f.f.name, Err: fs.ErrInvalid} + } + n := copy(b, f.f.data[f.offset:]) + f.offset += int64(n) + return n, nil +} + +type badFileOpen struct{} + +func (f *badFileOpen) Close() error { return errors.New("bad file close") } +func (f *badFileOpen) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") } +func (f *badFileOpen) Read(b []byte) (int, error) { + return 0, nil +} + +type badFileRead struct { + f *file + offset int64 +} + +func (f *badFileRead) Close() error { return errors.New("bad file close") } +func (f *badFileRead) Stat() (fs.FileInfo, error) { return nil, errors.New("bad file stat") } +func (f *badFileRead) Read(b []byte) (int, error) { + return 0, fmt.Errorf("bad file read") +} + +func (f FS) Open(name string) (fs.File, error) { + + data := `type: string +name: something +in: query` + + return &openFile{&file{"test.yaml", data}, 0}, nil +} + +func (f FSBadOpen) Open(name string) (fs.File, error) { + return nil, errors.New("bad file open") +} + +func (f FSBadRead) Open(name string) (fs.File, error) { + return &badFileRead{&file{}, 0}, nil +} + +func TestSpecIndex_UseRemoteHandler(t *testing.T) { + + spec := `openapi: 3.1.0 +info: + title: Test Remote Handler + version: 1.0.0 +paths: + /test: + get: + parameters: + - $ref: "https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(spec), &rootNode) + + c := CreateOpenAPIIndexConfig() + c.RemoteHandler = FS{} + + index := NewSpecIndexWithConfig(&rootNode, c) + + // extract crs param from index + crsParam := index.GetMappedReferences()["https://i-dont-exist-but-it-does-not-matter.com/some-place/some-file.yaml"] + assert.NotNil(t, crsParam) + assert.True(t, crsParam.IsRemote) + assert.Equal(t, "string", crsParam.Node.Content[1].Value) + assert.Equal(t, "something", crsParam.Node.Content[3].Value) + assert.Equal(t, "query", crsParam.Node.Content[5].Value) +} + +func TestSpecIndex_UseRemoteHandler_Error_Open(t *testing.T) { + + spec := `openapi: 3.1.0 +info: + title: Test Remote Handler + version: 1.0.0 +paths: + /test: + get: + parameters: + - $ref: "https://-i-cannot-be-opened.com"` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(spec), &rootNode) + + c := CreateOpenAPIIndexConfig() + c.RemoteHandler = FSBadOpen{} + c.RemoteURLHandler = httpClient.Get + + index := NewSpecIndexWithConfig(&rootNode, c) + + assert.Len(t, index.GetReferenceIndexErrors(), 2) + assert.Equal(t, "unable to open remote file: bad file open", index.GetReferenceIndexErrors()[0].Error()) + assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error()) +} + +func TestSpecIndex_UseRemoteHandler_Error_Read(t *testing.T) { + + spec := `openapi: 3.1.0 +info: + title: Test Remote Handler + version: 1.0.0 +paths: + /test: + get: + parameters: + - $ref: "https://-i-cannot-be-opened.com"` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(spec), &rootNode) + + c := CreateOpenAPIIndexConfig() + c.RemoteHandler = FSBadRead{} + c.RemoteURLHandler = httpClient.Get + + index := NewSpecIndexWithConfig(&rootNode, c) + + assert.Len(t, index.GetReferenceIndexErrors(), 2) + assert.Equal(t, "unable to read remote file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error()) + assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error()) +} diff --git a/index/index_model.go b/index/index_model.go index d63d27b..975c576 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -4,6 +4,7 @@ package index import ( + "io/fs" "net/http" "net/url" "os" @@ -66,6 +67,11 @@ type SpecIndexConfig struct { // Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132 RemoteURLHandler func(url string) (*http.Response, error) + // RemoteHandler is a function that will be used to fetch remote documents, it trumps the RemoteURLHandler + // and will be used instead if it is set. + // Resolves[#85] https://github.com/pb33f/libopenapi/issues/85 + RemoteHandler fs.FS + // If resolving locally, the BasePath will be the root from which relative references will be resolved from BasePath string // set the Base Path for resolving relative references if the spec is exploded.