Added more support for YAML merge nodes, anchors and aliases

And added deeper support for Aliases. Also added in local file handling through renamed `FSHandler` configuration property for the index.

Also re-ran `go fmt`

Signed-off-by: Dave Shanley <dave@quobix.com>
This commit is contained in:
Dave Shanley
2023-07-15 10:18:49 -04:00
committed by quobix
parent 3b7cbacc44
commit 25d8de9b0e
48 changed files with 925 additions and 531 deletions

View File

@@ -23,7 +23,7 @@ type Contact struct {
}
// Build is not implemented for Contact (there is nothing to build).
func (c *Contact) Build(root *yaml.Node, idx *index.SpecIndex) error {
func (c *Contact) Build(_ *yaml.Node, _ *index.SpecIndex) error {
c.Reference = new(low.Reference)
// not implemented.
return nil

View File

@@ -61,6 +61,8 @@ func (ex *Example) Hash() [32]byte {
// Build extracts extensions and example value
func (ex *Example) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
ex.Reference = new(low.Reference)
ex.Extensions = low.ExtractExtensions(root)
_, ln, vn := utils.FindKeyNodeFull(ValueLabel, root.Content)

View File

@@ -20,7 +20,7 @@ x-cake: hot`
var idxNode yaml.Node
mErr := yaml.Unmarshal([]byte(yml), &idxNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&idxNode)
idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig())
var n Example
err := low.BuildModel(idxNode.Content[0], &n)
@@ -46,7 +46,7 @@ x-cake: hot`
var idxNode yaml.Node
mErr := yaml.Unmarshal([]byte(yml), &idxNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&idxNode)
idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig())
var n Example
err := low.BuildModel(idxNode.Content[0], &n)
@@ -73,7 +73,7 @@ value:
var idxNode yaml.Node
mErr := yaml.Unmarshal([]byte(yml), &idxNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&idxNode)
idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig())
var n Example
err := low.BuildModel(idxNode.Content[0], &n)
@@ -104,7 +104,39 @@ value:
var idxNode yaml.Node
mErr := yaml.Unmarshal([]byte(yml), &idxNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&idxNode)
idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig())
var n Example
err := low.BuildModel(idxNode.Content[0], &n)
assert.NoError(t, err)
err = n.Build(idxNode.Content[0], idx)
assert.NoError(t, err)
assert.Equal(t, "hot", n.Summary.Value)
assert.Equal(t, "cakes", n.Description.Value)
if v, ok := n.Value.Value.([]interface{}); ok {
assert.Equal(t, "wow", v[0])
assert.Equal(t, "such array", v[1])
} else {
assert.Fail(t, "failed to decode correctly.")
}
}
func TestExample_Build_Success_MergeNode(t *testing.T) {
yml := `x-things: &things
summary: hot
description: cakes
value:
- wow
- such array
<<: *things`
var idxNode yaml.Node
mErr := yaml.Unmarshal([]byte(yml), &idxNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig())
var n Example
err := low.BuildModel(idxNode.Content[0], &n)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -33,6 +34,8 @@ func (ex *ExternalDoc) FindExtension(ext string) *low.ValueReference[any] {
// Build will extract extensions from the ExternalDoc instance.
func (ex *ExternalDoc) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
ex.Reference = new(low.Reference)
ex.Extensions = low.ExtractExtensions(root)
return nil

View File

@@ -6,6 +6,7 @@ package base
import (
"crypto/sha256"
"fmt"
"github.com/pb33f/libopenapi/utils"
"sort"
"strings"
@@ -45,6 +46,8 @@ func (i *Info) GetExtensions() map[low.KeyReference[string]]low.ValueReference[a
// Build will extract out the Contact and Info objects from the supplied root node.
func (i *Info) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
i.Reference = new(low.Reference)
i.Extensions = low.ExtractExtensions(root)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"strings"
)
@@ -25,6 +26,8 @@ type License struct {
// Build out a license, complain if both a URL and identifier are present as they are mutually exclusive
func (l *License) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
l.Reference = new(low.Reference)
if l.URL.Value != "" && l.Identifier.Value != "" {
return fmt.Errorf("license cannot have both a URL and an identifier, they are mutually exclusive")

View File

@@ -531,6 +531,8 @@ func (s *Schema) GetExtensions() map[low.KeyReference[string]]low.ValueReference
// - UnevaluatedProperties
// - Anchor
func (s *Schema) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
s.Reference = new(low.Reference)
if h, _, _ := utils.IsNodeRefValue(root); h {
ref, err := low.LocateRefNode(root, idx)

View File

@@ -80,6 +80,7 @@ func (sp *SchemaProxy) Schema() *Schema {
return sp.rendered
}
schema := new(Schema)
utils.CheckForMergeNodes(sp.vn)
err := schema.Build(sp.vn, sp.idx)
if err != nil {
sp.buildError = err

View File

@@ -73,3 +73,24 @@ func TestSchemaProxy_Build_HashInline(t *testing.T) {
assert.Equal(t, "6da88c34ba124c41f977db66a4fc5c1a951708d285c81bb0d47c3206f4c27ca8",
low.GenerateHashString(&sch))
}
func TestSchemaProxy_Build_UsingMergeNodes(t *testing.T) {
yml := `
x-common-definitions:
life_cycle_types: &life_cycle_types_def
type: string
enum: ["Onboarding", "Monitoring", "Re-Assessment"]
description: The type of life cycle
<<: *life_cycle_types_def`
var sch SchemaProxy
var idxNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &idxNode)
err := sch.Build(idxNode.Content[0], nil)
assert.NoError(t, err)
assert.Len(t, sch.Schema().Enum.Value, 3)
assert.Equal(t, "The type of life cycle", sch.Schema().Description.Value)
}

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -28,6 +29,8 @@ type SecurityRequirement struct {
// Build will extract security requirements from the node (the structure is odd, to be honest)
func (s *SecurityRequirement) Build(root *yaml.Node, _ *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
s.Reference = new(low.Reference)
var labelNode *yaml.Node
valueMap := make(map[low.KeyReference[string]]low.ValueReference[[]low.ValueReference[string]])

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -34,6 +35,8 @@ func (t *Tag) FindExtension(ext string) *low.ValueReference[any] {
// Build will extract extensions and external docs for the Tag.
func (t *Tag) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
t.Reference = new(low.Reference)
t.Extensions = low.ExtractExtensions(root)

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -31,6 +32,8 @@ type XML struct {
// Build will extract extensions from the XML instance.
func (x *XML) Build(root *yaml.Node, _ *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
x.Reference = new(low.Reference)
x.Extensions = low.ExtractExtensions(root)
return nil

View File

@@ -6,14 +6,13 @@ package low
import (
"crypto/sha256"
"fmt"
"reflect"
"strconv"
"strings"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
"gopkg.in/yaml.v3"
"reflect"
"strconv"
"strings"
)
// FindItemInMap accepts a string key and a collection of KeyReference[string] and ValueReference[T]. Every
@@ -86,14 +85,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
found[rv].Node.Column)
}
}
return found[rv].Node, nil
return utils.NodeAlias(found[rv].Node), nil
}
}
// perform a search for the reference in the index
foundRefs := idx.SearchIndexForReference(rv)
if len(foundRefs) > 0 {
return foundRefs[0].Node, nil
return utils.NodeAlias(foundRefs[0].Node), nil
}
// let's try something else to find our references.
@@ -106,7 +105,7 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
nodes, fErr := path.Find(idx.GetRootNode())
if fErr == nil {
if len(nodes) > 0 {
return nodes[0], nil
return utils.NodeAlias(nodes[0]), nil
}
}
}
@@ -123,6 +122,7 @@ func ExtractObjectRaw[T Buildable[N], N any](root *yaml.Node, idx *index.SpecInd
var circError error
var isReference bool
var referenceValue string
root = utils.NodeAlias(root)
if h, _, rv := utils.IsNodeRefValue(root); h {
ref, err := LocateRefNode(root, idx)
if ref != nil {
@@ -167,6 +167,7 @@ func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *in
var circError error
var isReference bool
var referenceValue string
root = utils.NodeAlias(root)
if rf, rl, refVal := utils.IsNodeRefValue(root); rf {
ref, err := LocateRefNode(root, idx)
if ref != nil {
@@ -251,6 +252,7 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
) {
var ln, vn *yaml.Node
var circError error
root = utils.NodeAlias(root)
if rf, rl, _ := utils.IsNodeRefValue(root); rf {
ref, err := LocateRefNode(root, idx)
if ref != nil {
@@ -370,7 +372,10 @@ func ExtractMapNoLookupExtensions[PT Buildable[N], N any](
if utils.IsNodeMap(root) {
var currentKey *yaml.Node
skip := false
for i, node := range root.Content {
rlen := len(root.Content)
for i := 0; i < rlen; i++ {
node := root.Content[i]
if !includeExtensions {
if strings.HasPrefix(strings.ToLower(node.Value), "x-") {
skip = true
@@ -386,6 +391,14 @@ func ExtractMapNoLookupExtensions[PT Buildable[N], N any](
continue
}
if currentKey.Tag == "!!merge" && currentKey.Value == "<<" {
root.Content = append(root.Content, utils.NodeAlias(node).Content...)
rlen = len(root.Content)
currentKey = nil
continue
}
node = utils.NodeAlias(node)
var isReference bool
var referenceValue string
// if value is a reference, we have to look it up in the index!
@@ -470,6 +483,7 @@ func ExtractMapExtensions[PT Buildable[N], N any](
var referenceValue string
var labelNode, valueNode *yaml.Node
var circError error
root = utils.NodeAlias(root)
if rf, rl, rv := utils.IsNodeRefValue(root); rf {
// locate reference in index.
ref, err := LocateRefNode(root, idx)
@@ -515,6 +529,7 @@ func ExtractMapExtensions[PT Buildable[N], N any](
buildMap := func(label *yaml.Node, value *yaml.Node, c chan mappingResult[PT], ec chan<- error, ref string) {
var n PT = new(N)
value = utils.NodeAlias(value)
_ = BuildModel(value, n)
err := n.Build(value, idx)
if err != nil {
@@ -544,6 +559,7 @@ func ExtractMapExtensions[PT Buildable[N], N any](
totalKeys := 0
for i, en := range valueNode.Content {
en = utils.NodeAlias(en)
referenceValue = ""
if i%2 == 0 {
currentLabelNode = en
@@ -620,6 +636,7 @@ func ExtractMap[PT Buildable[N], N any](
//
// int64, float64, bool, string
func ExtractExtensions(root *yaml.Node) map[KeyReference[string]]ValueReference[any] {
root = utils.NodeAlias(root)
extensions := utils.FindExtensionNodes(root.Content)
extensionMap := make(map[KeyReference[string]]ValueReference[any])
for _, ext := range extensions {

View File

@@ -23,6 +23,8 @@ func BuildModel(node *yaml.Node, model interface{}) error {
if node == nil {
return nil
}
node = utils.NodeAlias(node)
utils.CheckForMergeNodes(node)
if reflect.ValueOf(model).Type().Kind() != reflect.Pointer {
return fmt.Errorf("cannot build model on non-pointer: %v", reflect.ValueOf(model).Type().Kind())
@@ -51,6 +53,7 @@ func BuildModel(node *yaml.Node, model interface{}) error {
kind := field.Kind()
switch kind {
case reflect.Struct, reflect.Slice, reflect.Map, reflect.Pointer:
vn = utils.NodeAlias(vn)
err := SetField(&field, vn, kn)
if err != nil {
return err

View File

@@ -8,6 +8,7 @@ import (
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/datamodel/low/base"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -71,6 +72,8 @@ func (s *SecurityDefinitions) FindSecurityDefinition(securityDef string) *low.Va
// Build will extract all definitions into SchemaProxy instances.
func (d *Definitions) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
errorChan := make(chan error)
resultChan := make(chan definitionResult[*base.SchemaProxy])
var defLabel *yaml.Node

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -27,6 +28,8 @@ func (e *Examples) FindExample(name string) *low.ValueReference[any] {
// Build will extract all examples and will attempt to unmarshal content into a map or slice based on type.
func (e *Examples) Build(root *yaml.Node, _ *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
var keyNode, currNode *yaml.Node
var err error
e.Values = make(map[low.KeyReference[string]]low.ValueReference[any])

View File

@@ -52,6 +52,8 @@ func (h *Header) GetExtensions() map[low.KeyReference[string]]low.ValueReference
// Build will build out items, extensions and default value from the supplied node.
func (h *Header) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
h.Extensions = low.ExtractExtensions(root)
items, err := low.ExtractObject[*Items](ItemsLabel, root, idx)
if err != nil {

View File

@@ -103,6 +103,8 @@ func (i *Items) Hash() [32]byte {
// Build will build out items and default value.
func (i *Items) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
i.Extensions = low.ExtractExtensions(root)
items, iErr := low.ExtractObject[*Items](ItemsLabel, root, idx)
if iErr != nil {

View File

@@ -9,6 +9,7 @@ import (
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/datamodel/low/base"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -36,6 +37,8 @@ type Operation struct {
// Build will extract external docs, extensions, parameters, responses and security requirements.
func (o *Operation) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
o.Extensions = low.ExtractExtensions(root)
// extract externalDocs

View File

@@ -95,6 +95,8 @@ func (p *Parameter) GetExtensions() map[low.KeyReference[string]]low.ValueRefere
// Build will extract out extensions, schema, items and default value
func (p *Parameter) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
p.Extensions = low.ExtractExtensions(root)
sch, sErr := base.ExtractSchema(root, idx)
if sErr != nil {

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -47,6 +48,8 @@ func (p *PathItem) GetExtensions() map[low.KeyReference[string]]low.ValueReferen
// Build will extract extensions, parameters and operations for all methods. Every method is handled
// asynchronously, in order to keep things moving quickly for complex operations.
func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
p.Extensions = low.ExtractExtensions(root)
skip := false
var currentNode *yaml.Node
@@ -120,7 +123,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
wg.Add(1)
go low.BuildModelAsync(pathNode, &op, &wg, &errors)
low.BuildModelAsync(pathNode, &op, &wg, &errors)
opRef := low.NodeReference[*Operation]{
Value: &op,

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -51,6 +52,8 @@ func (p *Paths) FindExtension(ext string) *low.ValueReference[any] {
// Build will extract extensions and paths from node.
func (p *Paths) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
p.Extensions = low.ExtractExtensions(root)
skip := false
var currentNode *yaml.Node

View File

@@ -9,6 +9,7 @@ import (
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/datamodel/low/base"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -43,6 +44,8 @@ func (r *Response) FindHeader(hType string) *low.ValueReference[*Header] {
// Build will extract schema, extensions, examples and headers from node
func (r *Response) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
r.Extensions = low.ExtractExtensions(root)
s, err := base.ExtractSchema(root, idx)
if err != nil {

View File

@@ -28,6 +28,8 @@ func (r *Responses) GetExtensions() map[low.KeyReference[string]]low.ValueRefere
// Build will extract default value and extensions from node.
func (r *Responses) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
r.Extensions = low.ExtractExtensions(root)
if utils.IsNodeMap(root) {

View File

@@ -35,6 +35,8 @@ func (s *Scopes) FindScope(scope string) *low.ValueReference[string] {
// Build will extract scope values and extensions from node.
func (s *Scopes) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
s.Extensions = low.ExtractExtensions(root)
valueMap := make(map[low.KeyReference[string]]low.ValueReference[string])
if utils.IsNodeMap(root) {

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -38,6 +39,8 @@ func (ss *SecurityScheme) GetExtensions() map[low.KeyReference[string]]low.Value
// Build will extract extensions and scopes from the node.
func (ss *SecurityScheme) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
ss.Extensions = low.ExtractExtensions(root)
scopes, sErr := low.ExtractObject[*Scopes](ScopesLabel, root, idx)

View File

@@ -6,6 +6,7 @@ package v3
import (
"crypto/sha256"
"fmt"
"github.com/pb33f/libopenapi/utils"
"sort"
"strings"
@@ -39,6 +40,8 @@ func (cb *Callback) FindExpression(exp string) *low.ValueReference[*PathItem] {
// Build will extract extensions, expressions and PathItem objects for Callback
func (cb *Callback) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
cb.Reference = new(low.Reference)
cb.Extensions = low.ExtractExtensions(root)

View File

@@ -127,6 +127,8 @@ func (co *Components) FindCallback(callback string) *low.ValueReference[*Callbac
}
func (co *Components) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
co.Reference = new(low.Reference)
co.Extensions = low.ExtractExtensions(root)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"strings"
)
@@ -58,6 +59,8 @@ func (en *Encoding) Hash() [32]byte {
// Build will extract all Header objects from supplied node.
func (en *Encoding) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
en.Reference = new(low.Reference)
headers, hL, hN, err := low.ExtractMap[*Header](HeadersLabel, root, idx)
if err != nil {

View File

@@ -96,6 +96,8 @@ func (h *Header) Hash() [32]byte {
// Build will extract extensions, examples, schema and content/media types from node.
func (h *Header) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
h.Reference = new(low.Reference)
h.Extensions = low.ExtractExtensions(root)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -53,6 +54,8 @@ func (l *Link) FindExtension(ext string) *low.ValueReference[any] {
// Build will extract extensions and servers from the node.
func (l *Link) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
l.Reference = new(low.Reference)
l.Extensions = low.ExtractExtensions(root)
// extract server.

View File

@@ -55,6 +55,8 @@ func (mt *MediaType) GetAllExamples() map[low.KeyReference[string]]low.ValueRefe
// Build will extract examples, extensions, schema and encoding from node.
func (mt *MediaType) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
mt.Reference = new(low.Reference)
mt.Extensions = low.ExtractExtensions(root)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -36,6 +37,8 @@ func (o *OAuthFlows) FindExtension(ext string) *low.ValueReference[any] {
// Build will extract extensions and all OAuthFlow types from the supplied node.
func (o *OAuthFlows) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
o.Reference = new(low.Reference)
o.Extensions = low.ExtractExtensions(root)

View File

@@ -9,6 +9,7 @@ import (
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/datamodel/low/base"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -55,6 +56,8 @@ func (o *Operation) FindSecurityRequirement(name string) []low.ValueReference[st
// Build will extract external docs, parameters, request body, responses, callbacks, security and servers.
func (o *Operation) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
o.Reference = new(low.Reference)
o.Extensions = low.ExtractExtensions(root)

View File

@@ -59,6 +59,8 @@ func (p *Parameter) GetExtensions() map[low.KeyReference[string]]low.ValueRefere
// Build will extract examples, extensions and content/media types.
func (p *Parameter) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
p.Reference = new(low.Reference)
p.Extensions = low.ExtractExtensions(root)

View File

@@ -109,6 +109,8 @@ func (p *PathItem) GetExtensions() map[low.KeyReference[string]]low.ValueReferen
// Build extracts extensions, parameters, servers and each http method defined.
// everything is extracted asynchronously for speed.
func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
p.Reference = new(low.Reference)
p.Extensions = low.ExtractExtensions(root)
skip := false
@@ -232,7 +234,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
}
}
wg.Add(1)
go low.BuildModelAsync(pathNode, &op, &wg, &errors)
low.BuildModelAsync(pathNode, &op, &wg, &errors)
opRef := low.NodeReference[*Operation]{
Value: &op,

View File

@@ -59,6 +59,8 @@ func (p *Paths) GetExtensions() map[low.KeyReference[string]]low.ValueReference[
// Build will extract extensions and all PathItems. This happens asynchronously for speed.
func (p *Paths) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
p.Reference = new(low.Reference)
p.Extensions = low.ExtractExtensions(root)
skip := false

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -40,6 +41,8 @@ func (rb *RequestBody) FindContent(cType string) *low.ValueReference[*MediaType]
// Build will extract extensions and MediaType objects from the node.
func (rb *RequestBody) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
rb.Reference = new(low.Reference)
rb.Extensions = low.ExtractExtensions(root)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -54,6 +55,8 @@ func (r *Response) FindLink(hType string) *low.ValueReference[*Link] {
// Build will extract headers, extensions, content and links from node.
func (r *Response) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
r.Reference = new(low.Reference)
r.Extensions = low.ExtractExtensions(root)

View File

@@ -46,8 +46,10 @@ func (r *Responses) GetExtensions() map[low.KeyReference[string]]low.ValueRefere
// Build will extract default response and all Response objects for each code
func (r *Responses) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
r.Reference = new(low.Reference)
r.Extensions = low.ExtractExtensions(root)
utils.CheckForMergeNodes(root)
if utils.IsNodeMap(root) {
codes, err := low.ExtractMapNoLookup[*Response](root, idx)

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sort"
"strings"
@@ -48,6 +49,8 @@ func (ss *SecurityScheme) GetExtensions() map[low.KeyReference[string]]low.Value
// Build will extract OAuthFlows and extensions from the node.
func (ss *SecurityScheme) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
ss.Reference = new(low.Reference)
ss.Extensions = low.ExtractExtensions(root)

View File

@@ -35,6 +35,8 @@ func (s *Server) FindVariable(serverVar string) *low.ValueReference[*ServerVaria
// Build will extract server variables from the supplied node.
func (s *Server) Build(root *yaml.Node, idx *index.SpecIndex) error {
root = utils.NodeAlias(root)
utils.CheckForMergeNodes(root)
s.Reference = new(low.Reference)
s.Extensions = low.ExtractExtensions(root)
kn, vars := utils.FindKeyNode(VariablesLabel, root.Content)

View File

@@ -505,33 +505,34 @@ paths:
assert.Equal(t, d, strings.TrimSpace(string(rend)))
}
func TestDocument_RemoteWithoutBaseURL(t *testing.T) {
// This test will push the index to do try and locate remote references that use relative references
spec := `openapi: 3.0.2
info:
title: Test
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"`
config := datamodel.NewOpenDocumentConfiguration()
doc, err := NewDocumentWithConfiguration([]byte(spec), config)
if err != nil {
panic(err)
}
result, errs := doc.BuildV3Model()
if len(errs) > 0 {
panic(errs)
}
assert.Equal(t, "crs", result.Model.Paths.PathItems["/test"].Get.Parameters[0].Name)
}
// disabled for now as the host is timing out
//func TestDocument_RemoteWithoutBaseURL(t *testing.T) {
//
// // This test will push the index to do try and locate remote references that use relative references
// spec := `openapi: 3.0.2
//info:
// title: Test
// version: 1.0.0
//paths:
// /test:
// get:
// parameters:
// - $ref: "https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml#/components/parameters/crs"`
//
// config := datamodel.NewOpenDocumentConfiguration()
//
// doc, err := NewDocumentWithConfiguration([]byte(spec), config)
// if err != nil {
// panic(err)
// }
//
// result, errs := doc.BuildV3Model()
// if len(errs) > 0 {
// panic(errs)
// }
//
// assert.Equal(t, "crs", result.Model.Paths.PathItems["/test"].Get.Parameters[0].Name)
//}
func TestDocument_ExampleMap(t *testing.T) {
var d = `openapi: "3.1"

View File

@@ -130,9 +130,9 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
}
// if we have a remote handler, use it instead of the default.
if index.config != nil && index.config.RemoteHandler != nil {
if index.config != nil && index.config.FSHandler != nil {
go func() {
remoteFS := index.config.RemoteHandler
remoteFS := index.config.FSHandler
remoteFile, rErr := remoteFS.Open(uri)
if rErr != nil {
e := fmt.Errorf("unable to open remote file: %s", rErr)
@@ -220,10 +220,28 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
base := index.config.BasePath
fileToRead := filepath.Join(base, filePath, fileName)
var body []byte
var err error
// if we have an FS handler, use it instead of the default behavior
if index.config != nil && index.config.FSHandler != nil {
remoteFS := index.config.FSHandler
remoteFile, rErr := remoteFS.Open(fileToRead)
if rErr != nil {
e := fmt.Errorf("unable to open file: %s", rErr)
return nil, nil, e
}
body, err = io.ReadAll(remoteFile)
if err != nil {
e := fmt.Errorf("unable to read file bytes: %s", err)
return nil, nil, e
}
} else {
// try and read the file off the local file system, if it fails
// check for a baseURL and then ask our remote lookup function to go try and get it.
body, err := os.ReadFile(fileToRead)
body, err = os.ReadFile(fileToRead)
if err != nil {
@@ -244,7 +262,7 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
return nil, nil, err
}
}
}
var remoteDoc yaml.Node
err = yaml.Unmarshal(body, &remoteDoc)
if err != nil {

View File

@@ -288,7 +288,7 @@ paths:
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.RemoteHandler = FS{}
c.FSHandler = FS{}
index := NewSpecIndexWithConfig(&rootNode, c)
@@ -301,6 +301,35 @@ paths:
assert.Equal(t, "query", crsParam.Node.Content[5].Value)
}
func TestSpecIndex_UseFileHandler(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test Remote Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "some-file-that-does-not-exist.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FS{}
index := NewSpecIndexWithConfig(&rootNode, c)
// extract crs param from index
crsParam := index.GetMappedReferences()["some-file-that-does-not-exist.yaml"]
assert.NotNil(t, crsParam)
assert.True(t, crsParam.IsRemote)
assert.Equal(t, "string", crsParam.Node.Content[1].Value)
assert.Equal(t, "something", crsParam.Node.Content[3].Value)
assert.Equal(t, "query", crsParam.Node.Content[5].Value)
}
func TestSpecIndex_UseRemoteHandler_Error_Open(t *testing.T) {
spec := `openapi: 3.1.0
@@ -317,7 +346,7 @@ paths:
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.RemoteHandler = FSBadOpen{}
c.FSHandler = FSBadOpen{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
@@ -327,6 +356,32 @@ paths:
assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseFileHandler_Error_Open(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test File Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "I-can-never-be-opened.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FSBadOpen{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, "unable to open file: bad file open", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'I-can-never-be-opened.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseRemoteHandler_Error_Read(t *testing.T) {
spec := `openapi: 3.1.0
@@ -343,7 +398,7 @@ paths:
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.RemoteHandler = FSBadRead{}
c.FSHandler = FSBadRead{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
@@ -352,3 +407,29 @@ paths:
assert.Equal(t, "unable to read remote file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'https://-i-cannot-be-opened.com' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}
func TestSpecIndex_UseFileHandler_Error_Read(t *testing.T) {
spec := `openapi: 3.1.0
info:
title: Test File Handler
version: 1.0.0
paths:
/test:
get:
parameters:
- $ref: "I-am-impossible-to-open-forever.yaml"`
var rootNode yaml.Node
_ = yaml.Unmarshal([]byte(spec), &rootNode)
c := CreateOpenAPIIndexConfig()
c.FSHandler = FSBadRead{}
c.RemoteURLHandler = httpClient.Get
index := NewSpecIndexWithConfig(&rootNode, c)
assert.Len(t, index.GetReferenceIndexErrors(), 2)
assert.Equal(t, "unable to read file bytes: bad file read", index.GetReferenceIndexErrors()[0].Error())
assert.Equal(t, "component 'I-am-impossible-to-open-forever.yaml' does not exist in the specification", index.GetReferenceIndexErrors()[1].Error())
}

View File

@@ -67,10 +67,19 @@ type SpecIndexConfig struct {
// Resolves [#132]: https://github.com/pb33f/libopenapi/issues/132
RemoteURLHandler func(url string) (*http.Response, error)
// RemoteHandler is a function that will be used to fetch remote documents, it trumps the RemoteURLHandler
// and will be used instead if it is set.
// FSHandler is an entity that implements the `fs.FS` interface that will be used to fetch local or remote documents.
// This is useful if you want to use a custom file system handler, or if you want to use a custom http client or
// custom network implementation for a lookup.
//
// libopenapi will pass the path to the FSHandler, and it will be up to the handler to determine how to fetch
// the document. This is really useful if your application has a custom file system or uses a database for storing
// documents.
//
// Is the FSHandler is set, it will be used for all lookups, regardless of whether they are local or remote.
// it also overrides the RemoteURLHandler if set.
//
// Resolves[#85] https://github.com/pb33f/libopenapi/issues/85
RemoteHandler fs.FS
FSHandler fs.FS
// If resolving locally, the BasePath will be the root from which relative references will be resolved from
BasePath string // set the Base Path for resolving relative references if the spec is exploded.

View File

@@ -196,9 +196,9 @@ func FindFirstKeyNode(key string, nodes []*yaml.Node, depth int) (keyNode *yaml.
for i, v := range nodes {
if key != "" && key == v.Value {
if i+1 >= len(nodes) {
return v, nodes[i] // this is the node we need.
return v, NodeAlias(nodes[i]) // this is the node we need.
}
return v, nodes[i+1] // next node is what we need.
return v, NodeAlias(nodes[i+1]) // next node is what we need.
}
if len(v.Content) > 0 {
depth++
@@ -283,12 +283,12 @@ func FindKeyNodeFull(key string, nodes []*yaml.Node) (keyNode *yaml.Node, labelN
if key == v.Content[x].Value {
if IsNodeMap(v) {
if x+1 == len(v.Content) {
return v, v.Content[x], v.Content[x]
return v, v.Content[x], NodeAlias(v.Content[x])
}
return v, v.Content[x], v.Content[x+1]
return v, v.Content[x], NodeAlias(v.Content[x+1])
}
if IsNodeArray(v) {
return v, v.Content[x], v.Content[x]
return v, v.Content[x], NodeAlias(v.Content[x])
}
}
}
@@ -304,7 +304,7 @@ func FindKeyNodeFullTop(key string, nodes []*yaml.Node) (keyNode *yaml.Node, lab
continue
}
if i%2 == 0 && key == nodes[i].Value {
return nodes[i], nodes[i], nodes[i+1] // next node is what we need.
return nodes[i], nodes[i], NodeAlias(nodes[i+1]) // next node is what we need.
}
}
return nil, nil, nil
@@ -322,7 +322,7 @@ func FindExtensionNodes(nodes []*yaml.Node) []*ExtensionNode {
if i+1 < len(nodes) {
extensions = append(extensions, &ExtensionNode{
Key: v,
Value: nodes[i+1],
Value: NodeAlias(nodes[i+1]),
})
}
}
@@ -363,12 +363,38 @@ func IsNodeMap(node *yaml.Node) bool {
if node == nil {
return false
}
return node.Tag == "!!map"
n := NodeAlias(node)
return n.Tag == "!!map"
}
// IsNodeAlias checks if the node is an alias, and lifts out the anchor
func IsNodeAlias(node *yaml.Node) (*yaml.Node, bool) {
if node == nil {
return nil, false
}
if node.Kind == yaml.AliasNode {
node = node.Alias
return node, true
}
return node, false
}
// NodeAlias checks if the node is an alias, and lifts out the anchor
func NodeAlias(node *yaml.Node) *yaml.Node {
if node == nil {
return nil
}
if node.Kind == yaml.AliasNode {
node = node.Alias
return node
}
return node
}
// IsNodePolyMorphic will return true if the node contains polymorphic keys.
func IsNodePolyMorphic(node *yaml.Node) bool {
for i, v := range node.Content {
n := NodeAlias(node)
for i, v := range n.Content {
if i%2 == 0 {
if v.Value == "anyOf" || v.Value == "oneOf" || v.Value == "allOf" {
return true
@@ -383,7 +409,8 @@ func IsNodeArray(node *yaml.Node) bool {
if node == nil {
return false
}
return node.Tag == "!!seq"
n := NodeAlias(node)
return n.Tag == "!!seq"
}
// IsNodeStringValue checks if a node is a string value
@@ -391,7 +418,8 @@ func IsNodeStringValue(node *yaml.Node) bool {
if node == nil {
return false
}
return node.Tag == "!!str"
n := NodeAlias(node)
return n.Tag == "!!str"
}
// IsNodeIntValue will check if a node is an int value
@@ -399,7 +427,8 @@ func IsNodeIntValue(node *yaml.Node) bool {
if node == nil {
return false
}
return node.Tag == "!!int"
n := NodeAlias(node)
return n.Tag == "!!int"
}
// IsNodeFloatValue will check is a node is a float value.
@@ -407,7 +436,8 @@ func IsNodeFloatValue(node *yaml.Node) bool {
if node == nil {
return false
}
return node.Tag == "!!float"
n := NodeAlias(node)
return n.Tag == "!!float"
}
// IsNodeNumberValue will check if a node can be parsed as a float value.
@@ -423,18 +453,20 @@ func IsNodeBoolValue(node *yaml.Node) bool {
if node == nil {
return false
}
return node.Tag == "!!bool"
n := NodeAlias(node)
return n.Tag == "!!bool"
}
func IsNodeRefValue(node *yaml.Node) (bool, *yaml.Node, string) {
if node == nil {
return false, nil, ""
}
for i, r := range node.Content {
n := NodeAlias(node)
for i, r := range n.Content {
if i%2 == 0 {
if r.Value == "$ref" {
return true, r, node.Content[i+1].Value
return true, r, n.Content[i+1].Value
}
}
}
@@ -691,3 +723,26 @@ func DetermineWhitespaceLength(input string) int {
return 0
}
}
// CheckForMergeNodes will check the top level of the schema for merge nodes. If any are found, then the merged nodes
// will be appended to the end of the rest of the nodes in the schema.
// Note: this is a destructive operation, so the in-memory node structure will be modified
func CheckForMergeNodes(node *yaml.Node) {
if node == nil {
return
}
total := len(node.Content)
for i := 0; i < total; i++ {
mn := node.Content[i]
if i%2 == 0 {
if mn.Tag == "!!merge" {
an := node.Content[i+1].Alias
if an != nil {
node.Content = append(node.Content, an.Content...) // append the merged nodes
total = len(node.Content)
i += 2
}
}
}
}
}

View File

@@ -767,6 +767,69 @@ func TestIsNodeRefValue(t *testing.T) {
}
func TestIsNodeAlias(t *testing.T) {
yml := `things:
&anchorA
- Stuff
- Junk
thangs: *anchorA`
var node yaml.Node
_ = yaml.Unmarshal([]byte(yml), &node)
ref, a := IsNodeAlias(node.Content[0].Content[3])
assert.True(t, a)
assert.Len(t, ref.Content, 2)
}
func TestNodeAlias(t *testing.T) {
yml := `things:
&anchorA
- Stuff
- Junk
thangs: *anchorA`
var node yaml.Node
_ = yaml.Unmarshal([]byte(yml), &node)
ref := NodeAlias(node.Content[0].Content[3])
assert.Len(t, ref.Content, 2)
}
func TestCheckForMergeNodes(t *testing.T) {
yml := `x-common-definitions:
life_cycle_types: &life_cycle_types_def
type: string
enum: ["Onboarding", "Monitoring", "Re-Assessment"]
description: The type of life cycle
<<: *life_cycle_types_def`
var node yaml.Node
_ = yaml.Unmarshal([]byte(yml), &node)
mainNode := node.Content[0]
CheckForMergeNodes(mainNode)
_, _, enumVal := FindKeyNodeFullTop("enum", mainNode.Content)
_, _, descriptionVal := FindKeyNodeFullTop("description", mainNode.Content)
assert.Equal(t, "The type of life cycle", descriptionVal.Value)
assert.Len(t, enumVal.Content, 3)
}
func TestCheckForMergeNodes_Empty_NoPanic(t *testing.T) {
CheckForMergeNodes(nil)
}
func TestIsNodeRefValue_False(t *testing.T) {
f := &yaml.Node{