Tuned some glitches with v3 doc creation.

all covered now

Signed-off-by: quobix <dave@quobix.com>

datamodel/document_config.go

@@ -4,9 +4,9 @@
 package datamodel
 
 import (
+	"github.com/pb33f/libopenapi/utils"
 	"io/fs"
 	"log/slog"
-	"net/http"
 	"net/url"
 	"os"
 )
@@ -28,7 +28,7 @@ type DocumentConfiguration struct {
 	// will not be used, as there will be nothing to use it against.
 	//
 	// Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132
-	RemoteURLHandler func(url string) (*http.Response, error)
+	RemoteURLHandler utils.RemoteURLHandler
 
 	// If resolving locally, the BasePath will be the root from which relative references will be resolved from.
 	// It's usually the location of the root specification.
@@ -103,13 +103,3 @@ func NewDocumentConfiguration() *DocumentConfiguration {
 		})),
 	}
 }
-
-// Deprecated: use NewDocumentConfiguration instead.
-func NewOpenDocumentConfiguration() *DocumentConfiguration {
-	return NewDocumentConfiguration()
-}
-
-// Deprecated: use NewDocumentConfiguration instead.
-func NewClosedDocumentConfiguration() *DocumentConfiguration {
-	return NewDocumentConfiguration()
-}

datamodel/low/v3/create_document.go

@@ -51,12 +51,8 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 
 	// If basePath is provided, add a local filesystem to the rolodex.
 	if idxConfig.BasePath != "" {
-		var absError error
 		var cwd string
-		cwd, absError = filepath.Abs(config.BasePath)
-		if absError != nil {
-			return nil, absError
-		}
+		cwd, _ = filepath.Abs(config.BasePath)
 		// if a supplied local filesystem is provided, add it to the rolodex.
 		if config.LocalFS != nil {
 			rolodex.AddLocalFS(cwd, config.LocalFS)
@@ -68,34 +64,23 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 				DirFS:         os.DirFS(cwd),
 				FileFilters:   config.FileFilter,
 			}
+
 			fileFS, err := index.NewLocalFSWithConfig(&localFSConf)
 			if err != nil {
 				return nil, err
 			}
+
 			idxConfig.AllowFileLookup = true
 
 			// add the filesystem to the rolodex
 			rolodex.AddLocalFS(cwd, fileFS)
 		}
-
 	}
-
 	// if base url is provided, add a remote filesystem to the rolodex.
 	if idxConfig.BaseURL != nil {
 
-		// if a supplied remote filesystem is provided, add it to the rolodex.
-		if config.RemoteFS != nil {
-			if config.BaseURL == nil {
-				return nil, errors.New("cannot use remote filesystem without a BaseURL")
-			}
-			rolodex.AddRemoteFS(config.BaseURL.String(), config.RemoteFS)
-
-		} else {
 		// create a remote filesystem
-			remoteFS, fsErr := index.NewRemoteFSWithConfig(idxConfig)
-			if fsErr != nil {
-				return nil, fsErr
-			}
+		remoteFS, _ := index.NewRemoteFSWithConfig(idxConfig)
 		if config.RemoteURLHandler != nil {
 			remoteFS.RemoteHandlerFunc = config.RemoteURLHandler
 		}
@@ -104,7 +89,6 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur
 		// add to the rolodex
 		rolodex.AddRemoteFS(config.BaseURL.String(), remoteFS)
 	}
-	}
 
 	// index the rolodex
 	var errs []error

datamodel/low/v3/create_document_test.go

@@ -2,7 +2,10 @@ package v3
 
 import (
 	"fmt"
+	"github.com/pb33f/libopenapi/index"
 	"github.com/pb33f/libopenapi/utils"
+	"net/http"
+	"net/url"
 	"os"
 	"testing"
 
@@ -20,7 +23,7 @@ func initTest() {
 	info, _ := datamodel.ExtractSpecInfo(data)
 	var err error
 	// deprecated function test.
-	doc, err = CreateDocument(info)
+	doc, err = CreateDocumentFromConfig(info, datamodel.NewDocumentConfiguration())
 	if err != nil {
 		panic("broken something")
 	}
@@ -56,6 +59,123 @@ func TestCircularReferenceError(t *testing.T) {
 	assert.Len(t, utils.UnwrapErrors(err), 3)
 }
 
+func TestRolodexLocalFileSystem(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+
+	cf := datamodel.NewDocumentConfiguration()
+	cf.BasePath = "../../../test_specs"
+	cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.NoError(t, err)
+}
+
+func TestRolodexLocalFileSystem_ProvideNonRolodexFS(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+	baseDir := "../../../test_specs"
+
+	cf := datamodel.NewDocumentConfiguration()
+	cf.BasePath = baseDir
+	cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
+	cf.LocalFS = os.DirFS(baseDir)
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.Error(t, err)
+}
+
+func TestRolodexLocalFileSystem_ProvideRolodexFS(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+	baseDir := "../../../test_specs"
+	cf := datamodel.NewDocumentConfiguration()
+	cf.BasePath = baseDir
+	cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
+
+	localFS, lErr := index.NewLocalFSWithConfig(&index.LocalFSConfig{
+		BaseDirectory: baseDir,
+		DirFS:         os.DirFS(baseDir),
+		FileFilters:   cf.FileFilter,
+	})
+	cf.LocalFS = localFS
+
+	assert.NoError(t, lErr)
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.NoError(t, err)
+}
+
+func TestRolodexLocalFileSystem_BadPath(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+
+	cf := datamodel.NewDocumentConfiguration()
+	cf.BasePath = "/NOWHERE"
+	cf.FileFilter = []string{"first.yaml", "second.yaml", "third.yaml"}
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.Nil(t, lDoc)
+	assert.Error(t, err)
+}
+
+func TestRolodexRemoteFileSystem(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+
+	cf := datamodel.NewDocumentConfiguration()
+
+	baseUrl := "https://raw.githubusercontent.com/pb33f/libopenapi/main/test_specs"
+	u, _ := url.Parse(baseUrl)
+	cf.BaseURL = u
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.NoError(t, err)
+}
+
+func TestRolodexRemoteFileSystem_BadBase(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+
+	cf := datamodel.NewDocumentConfiguration()
+
+	baseUrl := "https://no-no-this-will-not-work-it-just-will-not-get-the-job-done-mate.com"
+	u, _ := url.Parse(baseUrl)
+	cf.BaseURL = u
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.Error(t, err)
+}
+
+func TestRolodexRemoteFileSystem_CustomRemote_NoBaseURL(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+
+	cf := datamodel.NewDocumentConfiguration()
+	cf.RemoteFS, _ = index.NewRemoteFSWithConfig(&index.SpecIndexConfig{})
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.Error(t, err)
+}
+
+func TestRolodexRemoteFileSystem_CustomHttpHandler(t *testing.T) {
+	data, _ := os.ReadFile("../../../test_specs/first.yaml")
+	info, _ := datamodel.ExtractSpecInfo(data)
+
+	cf := datamodel.NewDocumentConfiguration()
+	cf.RemoteURLHandler = http.Get
+	baseUrl := "https://no-no-this-will-not-work-it-just-will-not-get-the-job-done-mate.com"
+	u, _ := url.Parse(baseUrl)
+	cf.BaseURL = u
+
+	pizza := func(url string) (resp *http.Response, err error) {
+		return nil, nil
+	}
+	cf.RemoteURLHandler = pizza
+	lDoc, err := CreateDocumentFromConfig(info, cf)
+	assert.NotNil(t, lDoc)
+	assert.Error(t, err)
+}
+
 func TestCircularReference_IgnoreArray(t *testing.T) {
 	spec := `openapi: 3.1.0
 components:
@@ -76,8 +196,6 @@ components:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(spec))
 	circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:           false,
-		AllowRemoteReferences:         false,
 		IgnoreArrayCircularReferences: true,
 	})
 	assert.NotNil(t, circDoc)
@@ -104,8 +222,6 @@ components:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(spec))
 	circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:                 false,
-		AllowRemoteReferences:               false,
 		IgnorePolymorphicCircularReferences: true,
 	})
 	assert.NotNil(t, circDoc)
@@ -117,10 +233,7 @@ func BenchmarkCreateDocument_Stripe(b *testing.B) {
 	info, _ := datamodel.ExtractSpecInfo(data)
 
 	for i := 0; i < b.N; i++ {
-		_, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-			AllowFileReferences:   false,
-			AllowRemoteReferences: false,
-		})
+		_, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 		if err != nil {
 			panic("this should not error")
 		}
@@ -131,10 +244,7 @@ func BenchmarkCreateDocument_Petstore(b *testing.B) {
 	data, _ := os.ReadFile("../../../test_specs/petstorev3.json")
 	info, _ := datamodel.ExtractSpecInfo(data)
 	for i := 0; i < b.N; i++ {
-		_, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-			AllowFileReferences:   false,
-			AllowRemoteReferences: false,
-		})
+		_, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 		if err != nil {
 			panic("this should not error")
 		}
@@ -144,10 +254,7 @@ func BenchmarkCreateDocument_Petstore(b *testing.B) {
 func TestCreateDocumentStripe(t *testing.T) {
 	data, _ := os.ReadFile("../../../test_specs/stripe.yaml")
 	info, _ := datamodel.ExtractSpecInfo(data)
-	d, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	d, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Len(t, utils.UnwrapErrors(err), 3)
 
 	assert.Equal(t, "3.0.0", d.Version.Value)
@@ -211,10 +318,7 @@ webhooks:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Len(t, utils.UnwrapErrors(err), 1)
 }
 
@@ -590,10 +694,7 @@ components:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.NoError(t, err)
 
 	ob := doc.Components.Value.FindSchema("bork").Value
@@ -609,10 +710,7 @@ webhooks:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Equal(t, "flat map build failed: reference cannot be found: reference at line 4, column 5 is empty, it cannot be resolved",
 		err.Error())
 }
@@ -626,10 +724,7 @@ components:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Equal(t, "reference at line 5, column 7 is empty, it cannot be resolved", err.Error())
 }
 
@@ -641,10 +736,7 @@ paths:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Equal(t,
 		"path item build failed: cannot find reference:  at line 4, col 10", err.Error())
 }
@@ -656,10 +748,7 @@ tags:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Equal(t,
 		"object extraction failed: reference at line 3, column 5 is empty, it cannot be resolved", err.Error())
 }
@@ -671,10 +760,7 @@ security:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Equal(t,
 		"array build failed: reference cannot be found: reference at line 3, column 3 is empty, it cannot be resolved",
 		err.Error())
@@ -687,10 +773,7 @@ externalDocs:
 
 	info, _ := datamodel.ExtractSpecInfo([]byte(yml))
 	var err error
-	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	_, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 	assert.Equal(t, "object extraction failed: reference at line 3, column 3 is empty, it cannot be resolved", err.Error())
 }
 
@@ -702,10 +785,7 @@ func TestCreateDocument_YamlAnchor(t *testing.T) {
 	info, _ := datamodel.ExtractSpecInfo(anchorDocument)
 
 	// build low-level document model
-	document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 
 	if err != nil {
 		fmt.Printf("error: %s\n", err.Error())
@@ -755,10 +835,7 @@ func ExampleCreateDocument() {
 	info, _ := datamodel.ExtractSpecInfo(petstoreBytes)
 
 	// build low-level document model
-	document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{
-		AllowFileReferences:   false,
-		AllowRemoteReferences: false,
-	})
+	document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{})
 
 	if err != nil {
 		fmt.Printf("error: %s\n", err.Error())

index/extract_refs.go

@@ -283,7 +283,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 										if index.config.BaseURL != nil {
 
 											u := *index.config.BaseURL
-											abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0]))
+											abs := filepath.Join(u.Path, uri[0])
 											u.Path = abs
 											fullDefinitionPath = u.String()
 											componentName = uri[0]

index/rolodex_remote_loader.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/pb33f/libopenapi/datamodel"
+	"github.com/pb33f/libopenapi/utils"
 	"log/slog"
 	"runtime"
 
@@ -21,13 +22,11 @@ import (
 	"time"
 )
 
-type RemoteURLHandler = func(url string) (*http.Response, error)
-
 type RemoteFS struct {
 	indexConfig       *SpecIndexConfig
 	rootURL           string
 	rootURLParsed     *url.URL
-	RemoteHandlerFunc RemoteURLHandler
+	RemoteHandlerFunc utils.RemoteURLHandler
 	Files             syncmap.Map
 	ProcessingFiles   syncmap.Map
 	FetchTime         int64
@@ -176,6 +175,9 @@ const (
 )
 
 func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) {
+	if specIndexConfig == nil {
+		return nil, errors.New("no spec index config provided")
+	}
 	remoteRootURL := specIndexConfig.BaseURL
 	log := specIndexConfig.Logger
 	if log == nil {
@@ -217,7 +219,7 @@ func NewRemoteFSWithRootURL(rootURL string) (*RemoteFS, error) {
 	return NewRemoteFSWithConfig(config)
 }
 
-func (i *RemoteFS) SetRemoteHandlerFunc(handlerFunc RemoteURLHandler) {
+func (i *RemoteFS) SetRemoteHandlerFunc(handlerFunc utils.RemoteURLHandler) {
 	i.RemoteHandlerFunc = handlerFunc
 }
 

utils/utils.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"sort"
@@ -749,3 +750,5 @@ func CheckForMergeNodes(node *yaml.Node) {
 		}
 	}
 }
+
+type RemoteURLHandler = func(url string) (*http.Response, error)