Added tag model build out with tests.

Working through patterns and re-applying them as I go, cleaning things up as I cook.

datamodel/low/3.0/document.go

@@ -11,7 +11,7 @@ type Document struct {
 	Paths        *Paths
 	Components   *Components
 	Security     []*SecurityRequirement
-    Tags         []*Tag
+	Tags         []low.NodeReference[*Tag]
 	ExternalDocs *ExternalDoc
 	Extensions   map[string]low.ObjectReference
 }

datamodel/low/3.0/external_doc.go

@@ -2,11 +2,9 @@ package v3
 
 import (
 	"github.com/pb33f/libopenapi/datamodel/low"
-    "gopkg.in/yaml.v3"
 )
 
 type ExternalDoc struct {
-    Node        *yaml.Node
 	Description low.NodeReference[string]
 	URL         low.NodeReference[string]
 	Extensions  map[string]low.ObjectReference

datamodel/low/3.0/server.go

@@ -7,14 +7,18 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	Variables = "variables"
+)
+
 type Server struct {
 	URL         low.NodeReference[string]
 	Description low.NodeReference[string]
-    Variables   low.NodeReference[*map[string]low.NodeReference[*ServerVariable]]
+	Variables   low.NodeReference[map[string]low.NodeReference[*ServerVariable]]
 }
 
 func (s *Server) Build(root *yaml.Node) error {
-    kn, vars := utils.FindKeyNode("variables", root.Content)
+	kn, vars := utils.FindKeyNode(Variables, root.Content)
 	if vars == nil {
 		return nil
 	}
@@ -39,10 +43,10 @@ func (s *Server) Build(root *yaml.Node) error {
 				Value:     &variable,
 			}
 		}
-        s.Variables = low.NodeReference[*map[string]low.NodeReference[*ServerVariable]]{
+		s.Variables = low.NodeReference[map[string]low.NodeReference[*ServerVariable]]{
 			KeyNode:   kn,
 			ValueNode: vars,
-            Value:     &variablesMap,
+			Value:     variablesMap,
 		}
 	}
 	return nil

datamodel/low/3.0/tag.go

@@ -1,14 +1,44 @@
 package v3
 
 import (
+	"github.com/pb33f/libopenapi/datamodel"
 	"github.com/pb33f/libopenapi/datamodel/low"
+	"github.com/pb33f/libopenapi/utils"
 	"gopkg.in/yaml.v3"
 )
 
+const (
+	Tags         = "tags"
+	ExternalDocs = "externalDocs"
+)
+
 type Tag struct {
-    Node         *yaml.Node
 	Name         low.NodeReference[string]
 	Description  low.NodeReference[string]
-    ExternalDocs ExternalDoc
+	ExternalDocs low.NodeReference[*ExternalDoc]
-    Extensions   map[string]low.ObjectReference
+	Extensions   map[low.NodeReference[string]]low.NodeReference[any]
+}
+
+func (t *Tag) Build(root *yaml.Node) error {
+	_, ln, exDocs := utils.FindKeyNodeFull(ExternalDocs, root.Content)
+
+	// extract extensions
+	extensionMap, err := datamodel.ExtractExtensions(root)
+	if err != nil {
+		return err
+	}
+	t.Extensions = extensionMap
+
+	// extract external docs
+	var externalDoc ExternalDoc
+	err = datamodel.BuildModel(exDocs, &externalDoc)
+	if err != nil {
+		return err
+	}
+	t.ExternalDocs = low.NodeReference[*ExternalDoc]{
+		Value:     &externalDoc,
+		KeyNode:   ln,
+		ValueNode: exDocs,
+	}
+	return nil
 }

datamodel/low/reference.go

@@ -17,7 +17,7 @@ type NodeReference[T any] struct {
 }
 
 type ObjectReference struct {
-    Value     map[string]interface{}
+	Value     interface{}
 	ValueNode *yaml.Node
 	KeyNode   *yaml.Node
 }

datamodel/model_builder.go

@@ -352,3 +352,60 @@ func BuildModelAsync(n *yaml.Node, model interface{}, lwg *sync.WaitGroup, error
 	}
 	lwg.Done()
 }
+
+func ExtractExtensions(root *yaml.Node) (map[low.NodeReference[string]]low.NodeReference[any], error) {
+	extensions := utils.FindExtensionNodes(root.Content)
+	extensionMap := make(map[low.NodeReference[string]]low.NodeReference[any])
+	for _, ext := range extensions {
+		if utils.IsNodeMap(ext.Value) {
+			var v interface{}
+			err := ext.Value.Decode(&v)
+			if err != nil {
+				return nil, err
+			}
+			extensionMap[low.NodeReference[string]{
+				Value:   ext.Key.Value,
+				KeyNode: ext.Key,
+			}] = low.NodeReference[any]{Value: v, KeyNode: ext.Key}
+		}
+		if utils.IsNodeStringValue(ext.Value) {
+			extensionMap[low.NodeReference[string]{
+				Value:   ext.Key.Value,
+				KeyNode: ext.Key,
+			}] = low.NodeReference[any]{Value: ext.Value.Value, ValueNode: ext.Value}
+		}
+		if utils.IsNodeFloatValue(ext.Value) {
+			fv, _ := strconv.ParseFloat(ext.Value.Value, 64)
+			extensionMap[low.NodeReference[string]{
+				Value:   ext.Key.Value,
+				KeyNode: ext.Key,
+			}] = low.NodeReference[any]{Value: fv, ValueNode: ext.Value}
+		}
+		if utils.IsNodeIntValue(ext.Value) {
+			iv, _ := strconv.ParseInt(ext.Value.Value, 10, 64)
+			extensionMap[low.NodeReference[string]{
+				Value:   ext.Key.Value,
+				KeyNode: ext.Key,
+			}] = low.NodeReference[any]{Value: iv, ValueNode: ext.Value}
+		}
+		if utils.IsNodeBoolValue(ext.Value) {
+			bv, _ := strconv.ParseBool(ext.Value.Value)
+			extensionMap[low.NodeReference[string]{
+				Value:   ext.Key.Value,
+				KeyNode: ext.Key,
+			}] = low.NodeReference[any]{Value: bv, ValueNode: ext.Value}
+		}
+		if utils.IsNodeArray(ext.Value) {
+			var v []interface{}
+			err := ext.Value.Decode(&v)
+			if err != nil {
+				return nil, err
+			}
+			extensionMap[low.NodeReference[string]{
+				Value:   ext.Key.Value,
+				KeyNode: ext.Key,
+			}] = low.NodeReference[any]{Value: v, ValueNode: ext.Value}
+		}
+	}
+	return extensionMap, nil
+}

openapi/create_document.go

@@ -5,39 +5,53 @@ import (
 	"github.com/pb33f/libopenapi/datamodel/low"
 	v3 "github.com/pb33f/libopenapi/datamodel/low/3.0"
 	"github.com/pb33f/libopenapi/utils"
+	"gopkg.in/yaml.v3"
+	"strconv"
+	"sync"
 )
 
-func CreateDocument(spec []byte) (*v3.Document, error) {
+const (
+	Info    = "info"
+	Servers = "servers"
+)
 
-    // extract details from spec
+func CreateDocument(info *datamodel.SpecInfo) (*v3.Document, error) {
-    info, err := datamodel.ExtractSpecInfo(spec)
-    if err != nil {
-        return nil, err
-    }
 
 	doc := v3.Document{Version: low.NodeReference[string]{Value: info.Version, ValueNode: info.RootNode}}
 
 	// build an index
 	//idx := index.NewSpecIndex(info.RootNode)
 	//datamodel.BuildModel(info.RootNode.Content[0], &doc)
+	var wg sync.WaitGroup
+	var errors []error
+	var runExtraction = func(info *datamodel.SpecInfo, doc *v3.Document,
+		runFunc func(i *datamodel.SpecInfo, d *v3.Document) error,
+		ers *[]error,
+		wg *sync.WaitGroup) {
 
-    // extract info
+		if er := runFunc(info, doc); er != nil {
-    extractErr := extractInfo(info, &doc)
+			*ers = append(*ers, er)
-    if extractErr != nil {
-        return nil, extractErr
 		}
 
-    // extract servers
+		wg.Done()
-    extractErr = extractServers(info, &doc)
+	}
-    if extractErr != nil {
+
-        return nil, extractErr
+	wg.Add(3)
+	go runExtraction(info, &doc, extractInfo, &errors, &wg)
+	go runExtraction(info, &doc, extractServers, &errors, &wg)
+	go runExtraction(info, &doc, extractTags, &errors, &wg)
+	wg.Wait()
+
+	// todo fix this.
+	if len(errors) > 0 {
+		return &doc, errors[0]
 	}
 
 	return &doc, nil
 }
 
 func extractInfo(info *datamodel.SpecInfo, doc *v3.Document) error {
-    _, ln, vn := utils.FindKeyNodeFull("info", info.RootNode.Content)
+	_, ln, vn := utils.FindKeyNodeFull(Info, info.RootNode.Content)
 	if vn != nil {
 		ir := v3.Info{}
 		err := datamodel.BuildModel(vn, &ir)
@@ -52,7 +66,7 @@ func extractInfo(info *datamodel.SpecInfo, doc *v3.Document) error {
 }
 
 func extractServers(info *datamodel.SpecInfo, doc *v3.Document) error {
-    _, ln, vn := utils.FindKeyNodeFull("servers", info.RootNode.Content)
+	_, ln, vn := utils.FindKeyNodeFull(Servers, info.RootNode.Content)
 	if vn != nil {
 		if utils.IsNodeArray(vn) {
 			var servers []low.NodeReference[*v3.Server]
@@ -76,3 +90,93 @@ func extractServers(info *datamodel.SpecInfo, doc *v3.Document) error {
 	}
 	return nil
 }
+
+func extractTags(info *datamodel.SpecInfo, doc *v3.Document) error {
+	_, ln, vn := utils.FindKeyNodeFull(v3.Tags, info.RootNode.Content)
+	if vn != nil {
+		if utils.IsNodeArray(vn) {
+			var tags []low.NodeReference[*v3.Tag]
+			for _, tagN := range vn.Content {
+				if utils.IsNodeMap(tagN) {
+					tag := v3.Tag{}
+					err := datamodel.BuildModel(tagN, &tag)
+					if err != nil {
+						return err
+					}
+					tag.Build(tagN)
+					tags = append(tags, low.NodeReference[*v3.Tag]{
+						Value:     &tag,
+						ValueNode: tagN,
+						KeyNode:   ln,
+					})
+				}
+			}
+			doc.Tags = tags
+		}
+	}
+	return nil
+}
+
+func ExtractExtensions(root *yaml.Node) (map[low.NodeReference[string]]low.NodeReference[any], error) {
+	extensions := utils.FindExtensionNodes(root.Content)
+	extensionMap := make(map[low.NodeReference[string]]low.NodeReference[any])
+	for _, ext := range extensions {
+		// this is an object, decode into an unknown map.
+		if utils.IsNodeMap(ext.Value) {
+			var v interface{}
+			err := ext.Value.Decode(&v)
+			if err != nil {
+				return nil, err
+			}
+			extensionMap[low.NodeReference[string]{
+				Value:     ext.Key.Value,
+				KeyNode:   ext.Key,
+				ValueNode: ext.Value,
+			}] = low.NodeReference[any]{Value: v, KeyNode: ext.Key}
+		}
+		if utils.IsNodeStringValue(ext.Value) {
+			extensionMap[low.NodeReference[string]{
+				Value:     ext.Key.Value,
+				KeyNode:   ext.Key,
+				ValueNode: ext.Value,
+			}] = low.NodeReference[any]{Value: ext.Value.Value, ValueNode: ext.Value}
+		}
+		if utils.IsNodeFloatValue(ext.Value) {
+			fv, _ := strconv.ParseFloat(ext.Value.Value, 64)
+			extensionMap[low.NodeReference[string]{
+				Value:     ext.Key.Value,
+				KeyNode:   ext.Key,
+				ValueNode: ext.Value,
+			}] = low.NodeReference[any]{Value: fv, ValueNode: ext.Value}
+		}
+		if utils.IsNodeIntValue(ext.Value) {
+			iv, _ := strconv.ParseInt(ext.Value.Value, 10, 64)
+			extensionMap[low.NodeReference[string]{
+				Value:     ext.Key.Value,
+				KeyNode:   ext.Key,
+				ValueNode: ext.Value,
+			}] = low.NodeReference[any]{Value: iv, ValueNode: ext.Value}
+		}
+		if utils.IsNodeBoolValue(ext.Value) {
+			bv, _ := strconv.ParseBool(ext.Value.Value)
+			extensionMap[low.NodeReference[string]{
+				Value:     ext.Key.Value,
+				KeyNode:   ext.Key,
+				ValueNode: ext.Value,
+			}] = low.NodeReference[any]{Value: bv, ValueNode: ext.Value}
+		}
+		if utils.IsNodeArray(ext.Value) {
+			var v []interface{}
+			err := ext.Value.Decode(&v)
+			if err != nil {
+				return nil, err
+			}
+			extensionMap[low.NodeReference[string]{
+				Value:     ext.Key.Value,
+				KeyNode:   ext.Key,
+				ValueNode: ext.Value,
+			}] = low.NodeReference[any]{Value: v, ValueNode: ext.Value}
+		}
+	}
+	return extensionMap, nil
+}

openapi/create_document_test.go

@@ -1,32 +1,109 @@
 package openapi
 
 import (
+    "github.com/pb33f/libopenapi/datamodel"
+    v3 "github.com/pb33f/libopenapi/datamodel/low/3.0"
     "github.com/stretchr/testify/assert"
     "io/ioutil"
     "testing"
 )
 
-func TestCreateDocument_NoData(t *testing.T) {
+var doc *v3.Document
-    doc, err := CreateDocument(nil)
+
-    assert.Nil(t, doc)
+func init() {
-    assert.Error(t, err)
+    data, _ := ioutil.ReadFile("../test_specs/burgershop.openapi.yaml")
+    info, _ := datamodel.ExtractSpecInfo(data)
+    doc, _ = CreateDocument(info)
+}
+
+func BenchmarkCreateDocument(b *testing.B) {
+    data, _ := ioutil.ReadFile("../test_specs/burgershop.openapi.yaml")
+    info, _ := datamodel.ExtractSpecInfo(data)
+    for i := 0; i < b.N; i++ {
+        doc, _ = CreateDocument(info)
+    }
 }
 
 func TestCreateDocument(t *testing.T) {
-    data, aErr := ioutil.ReadFile("../test_specs/burgershop.openapi.yaml")
-    assert.NoError(t, aErr)
-
-    doc, err := CreateDocument(data)
-    assert.NotNil(t, doc)
-    assert.NoError(t, err)
-
     assert.Equal(t, "3.0.1", doc.Version.Value)
     assert.Equal(t, "Burger Shop", doc.Info.Value.Title.Value)
     assert.NotEmpty(t, doc.Info.Value.Title.Value)
+}
+
+func TestCreateDocument_Info(t *testing.T) {
     assert.Equal(t, "https://pb33f.io", doc.Info.Value.TermsOfService.Value)
     assert.Equal(t, "pb33f", doc.Info.Value.Contact.Value.Name.Value)
     assert.Equal(t, "buckaroo@pb33f.io", doc.Info.Value.Contact.Value.Email.Value)
     assert.Equal(t, "https://pb33f.io", doc.Info.Value.Contact.Value.URL.Value)
+    assert.Equal(t, "pb33f", doc.Info.Value.License.Value.Name.Value)
+    assert.Equal(t, "https://pb33f.io/made-up", doc.Info.Value.License.Value.URL.Value)
+}
+
+func TestCreateDocument_Servers(t *testing.T) {
+    assert.Len(t, doc.Servers, 2)
+    server1 := doc.Servers[0]
+    server2 := doc.Servers[1]
+
+    // server 1
+    assert.Equal(t, "{scheme}://api.pb33f.io", server1.Value.URL.Value)
+    assert.NotEmpty(t, server1.Value.Description.Value)
+    assert.Len(t, server1.Value.Variables.Value, 1)
+    assert.Len(t, server1.Value.Variables.Value["scheme"].Value.Enum, 2)
+    assert.Equal(t, server1.Value.Variables.Value["scheme"].Value.Default.Value, "https")
+    assert.NotEmpty(t, server1.Value.Variables.Value["scheme"].Value.Description.Value)
+
+    // server 2
+    assert.Equal(t, "https://{domain}.{host}.com", server2.Value.URL.Value)
+    assert.NotEmpty(t, server2.Value.Description.Value)
+    assert.Len(t, server2.Value.Variables.Value, 2)
+    assert.Equal(t, server2.Value.Variables.Value["domain"].Value.Default.Value, "api")
+    assert.NotEmpty(t, server2.Value.Variables.Value["domain"].Value.Description.Value)
+    assert.NotEmpty(t, server2.Value.Variables.Value["host"].Value.Description.Value)
+    assert.Equal(t, server2.Value.Variables.Value["host"].Value.Default.Value, "pb33f.io")
     assert.Equal(t, "1.2", doc.Info.Value.Version.Value)
+}
+
+func TestCreateDocument_Tags(t *testing.T) {
+    assert.Len(t, doc.Tags, 2)
+
+    // tag1
+    assert.Equal(t, "Burgers", doc.Tags[0].Value.Name.Value)
+    assert.NotEmpty(t, doc.Tags[0].Value.Description.Value)
+    assert.NotNil(t, doc.Tags[0].Value.ExternalDocs.Value)
+    assert.Equal(t, "https://pb33f.io", doc.Tags[0].Value.ExternalDocs.Value.URL.Value)
+    assert.NotEmpty(t, doc.Tags[0].Value.ExternalDocs.Value.URL.Value)
+    assert.Len(t, doc.Tags[0].Value.Extensions, 7)
+
+    for key, extension := range doc.Tags[0].Value.Extensions {
+        switch key.Value {
+        case "x-internal-ting":
+            assert.Equal(t, "somethingSpecial", extension.Value)
+        case "x-internal-tong":
+            assert.Equal(t, int64(1), extension.Value)
+        case "x-internal-tang":
+            assert.Equal(t, 1.2, extension.Value)
+        case "x-internal-tung":
+            assert.Equal(t, true, extension.Value)
+        case "x-internal-arr":
+            assert.Len(t, extension.Value, 2)
+            assert.Equal(t, "one", extension.Value.([]interface{})[0].(string))
+        case "x-internal-arrmap":
+            assert.Len(t, extension.Value, 2)
+            assert.Equal(t, "now", extension.Value.([]interface{})[0].(map[string]interface{})["what"])
+        case "x-something-else":
+            // crazy times in the upside down. this API should be avoided for the higher up use cases.
+            // this is why we will need a higher level API to this model, this looks cool and all, but dude.
+            assert.Equal(t, "now?", extension.Value.(map[string]interface{})["ok"].([]interface{})[0].(map[string]interface{})["what"])
+        }
+
+    }
+
+    /// tag2
+    assert.Equal(t, "Dressing", doc.Tags[1].Value.Name.Value)
+    assert.NotEmpty(t, doc.Tags[1].Value.Description.Value)
+    assert.NotNil(t, doc.Tags[1].Value.ExternalDocs.Value)
+    assert.Equal(t, "https://pb33f.io", doc.Tags[1].Value.ExternalDocs.Value.URL.Value)
+    assert.NotEmpty(t, doc.Tags[1].Value.ExternalDocs.Value.URL.Value)
+    assert.Len(t, doc.Tags[1].Value.Extensions, 0)
 
 }

test_specs/burgershop.openapi.yaml

@@ -10,25 +10,38 @@ info:
     url: https://pb33f.io
   license:
     name: pb33f
-    url: https://quobix.com/made-up
+    url: https://pb33f.io/made-up
   version: "1.2"
 tags:
   - name: "Burgers"
     description: "All kinds of yummy burgers."
     externalDocs:
       description: "Find out more"
-      url: "https://quobix.com/"
+      url: "https://pb33f.io"
+    x-internal-ting: somethingSpecial
+    x-internal-tong: 1
+    x-internal-tang: 1.2
+    x-internal-tung: true
+    x-internal-arr:
+      - one
+      - two
+    x-internal-arrmap:
+      - what: now
+      - why: that
+    x-something-else:
+      ok:
+        - what: now?
   - name: "Dressing"
     description: "Variety of dressings: cheese, veggie, oil and a lot more"
     externalDocs:
       description: "Find out more information about our products)"
-      url: "https://quobix.com/"
+      url: "https://pb33f.io"
 servers:
-  - url: "{scheme}://api.quobix.com"
+  - url: "{scheme}://api.pb33f.io"
     description: "this is our main API server, for all fun API things."
     variables:
       scheme:
-        enum: [https]
+        enum: [https, wss]
         default: https
         description: this is a server variable for the scheme
   - url: "https://{domain}.{host}.com"
@@ -38,8 +51,8 @@ servers:
         default: "api"
         description: the default API domain is 'api'
       host:
-        default: "quobix.com"
+        default: "pb33f.io"
-        description: the default host for this API is 'quobix.com'
+        description: the default host for this API is 'pb33f.com'
 paths:
   /burgers:
     post:

utils/utils.go

@@ -262,6 +262,26 @@ func FindKeyNodeFull(key string, nodes []*yaml.Node) (keyNode *yaml.Node, labelN
 	return nil, nil, nil
 }
 
+type ExtensionNode struct {
+	Key   *yaml.Node
+	Value *yaml.Node
+}
+
+func FindExtensionNodes(nodes []*yaml.Node) []*ExtensionNode {
+	var extensions []*ExtensionNode
+	for i, v := range nodes {
+		if i%2 == 0 && strings.HasPrefix(v.Value, "x-") {
+			if i+1 < len(nodes) {
+				extensions = append(extensions, &ExtensionNode{
+					Key:   v,
+					Value: nodes[i+1],
+				})
+			}
+		}
+	}
+	return extensions
+}
+
 var ObjectLabel = "object"
 var IntegerLabel = "integer"
 var NumberLabel = "number"