Added resolver, models and model utils.

datamodel/model_utils.go

@@ -35,7 +35,7 @@ func ExtractSpecInfo(spec []byte) (*SpecInfo, error) {
     var parsedSpec yaml.Node
 
     specVersion := &SpecInfo{}
-    specVersion.jsonParsingChannel = make(chan bool)
+    specVersion.JsonParsingChannel = make(chan bool)
 
     // set original bytes
     specVersion.SpecBytes = &spec
@@ -85,8 +85,8 @@ func ExtractSpecInfo(spec []byte) (*SpecInfo, error) {
             spec.SpecJSONBytes = &bytes
             spec.SpecJSON = &jsonSpec
         }
-        spec.jsonParsingChannel <- true
+        spec.JsonParsingChannel <- true
-        close(spec.jsonParsingChannel)
+        close(spec.JsonParsingChannel)
     }
     // check for specific keys
     if openAPI3 != nil {

datamodel/spec_info.go

@@ -0,0 +1,28 @@
+package datamodel
+
+import (
+    "gopkg.in/yaml.v3"
+    "time"
+)
+
+// SpecInfo represents information about a supplied specification.
+type SpecInfo struct {
+    SpecType           string                  `json:"type"`
+    Version            string                  `json:"version"`
+    SpecFormat         string                  `json:"format"`
+    SpecFileType       string                  `json:"fileType"`
+    RootNode           *yaml.Node              `json:"-"`     // reference to the root node of the spec.
+    SpecBytes          *[]byte                 `json:"bytes"` // the original bytes
+    SpecJSONBytes      *[]byte                 `json:"-"`     // original bytes converted to JSON
+    SpecJSON           *map[string]interface{} `json:"-"`     // standard JSON map of original bytes
+    Error              error                   `json:"-"`     // something go wrong?
+    APISchema          string                  `json:"-"`     // API Schema for supplied spec type (2 or 3)
+    Generated          time.Time               `json:"-"`
+    JsonParsingChannel chan bool               `json:"-"`
+}
+
+// GetJSONParsingChannel returns a channel that will close once async JSON parsing is completed.
+// This is required as rules may start executing before we're even done reading in the spec to JSON.
+func (si SpecInfo) GetJSONParsingChannel() chan bool {
+    return si.JsonParsingChannel
+}

resolver/resolver.go

@@ -0,0 +1,229 @@
+// Copyright 2022 Dave Shanley / Quobix
+// SPDX-License-Identifier: MIT
+
+package resolver
+
+import (
+	"fmt"
+	"github.com/pb33f/libopenapi/index"
+	"github.com/pb33f/libopenapi/utils"
+	"gopkg.in/yaml.v3"
+	"strings"
+)
+
+// CircularReferenceResult contains a circular reference found when traversing the graph.
+type CircularReferenceResult struct {
+	Journey   []*index.Reference
+	Start     *index.Reference
+	LoopIndex int
+	LoopPoint *index.Reference
+}
+
+func (c *CircularReferenceResult) GenerateJourneyPath() string {
+	buf := strings.Builder{}
+	for i, ref := range c.Journey {
+		buf.WriteString(ref.Name)
+		if i+1 < len(c.Journey) {
+			buf.WriteString(" -> ")
+		}
+	}
+	return buf.String()
+}
+
+// ResolvingError represents an issue the resolver had trying to stitch the tree together.
+type ResolvingError struct {
+	Error error
+	Node  *yaml.Node
+	Path  string
+}
+
+// Resolver will use a *index.SpecIndex to stitch together a resolved root tree using all the discovered
+// references in the doc.
+type Resolver struct {
+	specIndex          *index.SpecIndex
+	resolvedRoot       *yaml.Node
+	resolvingErrors    []*ResolvingError
+	circularReferences []*CircularReferenceResult
+}
+
+// NewResolver will create a new resolver from a *index.SpecIndex
+func NewResolver(index *index.SpecIndex) *Resolver {
+	if index == nil {
+		return nil
+	}
+	return &Resolver{
+		specIndex:    index,
+		resolvedRoot: index.GetRootNode(),
+	}
+}
+
+// GetResolvingErrors returns all errors found during resolving
+func (resolver *Resolver) GetResolvingErrors() []*ResolvingError {
+	return resolver.resolvingErrors
+}
+
+// GetCircularErrors returns all errors found during resolving
+func (resolver *Resolver) GetCircularErrors() []*CircularReferenceResult {
+	return resolver.circularReferences
+}
+
+// Resolve will resolve the specification, everything that is not polymorphic and not circular, will be resolved.
+// this data can get big, it results in a massive duplication of data.
+func (resolver *Resolver) Resolve() []*ResolvingError {
+
+	mapped := resolver.specIndex.GetMappedReferencesSequenced()
+	mappedIndex := resolver.specIndex.GetMappedReferences()
+
+	for _, ref := range mapped {
+		seenReferences := make(map[string]bool)
+		var journey []*index.Reference
+		ref.Reference.Node.Content = resolver.VisitReference(ref.Reference, seenReferences, journey)
+	}
+
+	schemas := resolver.specIndex.GetAllSchemas()
+
+	for s, schemaRef := range schemas {
+		if mappedIndex[s] == nil {
+			seenReferences := make(map[string]bool)
+			var journey []*index.Reference
+			schemaRef.Node.Content = resolver.VisitReference(schemaRef, seenReferences, journey)
+		}
+	}
+
+	// map everything
+	for _, sequenced := range resolver.specIndex.GetAllSequencedReferences() {
+		locatedDef := mappedIndex[sequenced.Definition]
+		if locatedDef != nil {
+			if !locatedDef.Circular && locatedDef.Seen {
+				sequenced.Node.Content = locatedDef.Node.Content
+			}
+		}
+	}
+
+	for _, circRef := range resolver.circularReferences {
+		resolver.resolvingErrors = append(resolver.resolvingErrors, &ResolvingError{
+			Error: fmt.Errorf("Circular reference detected: %s", circRef.Start.Name),
+			Node:  circRef.LoopPoint.Node,
+			Path:  circRef.GenerateJourneyPath(),
+		})
+	}
+
+	return resolver.resolvingErrors
+}
+
+// VisitReference will visit a reference as part of a journey and will return resolved nodes.
+func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference) []*yaml.Node {
+
+	if ref.Resolved || ref.Seen {
+		return ref.Node.Content
+	}
+
+	journey = append(journey, ref)
+	relatives := resolver.extractRelatives(ref.Node, seen, journey)
+
+	seen = make(map[string]bool)
+
+	seen[ref.Definition] = true
+	for _, r := range relatives {
+
+		// check if we have seen this on the journey before, if so! it's circular
+		skip := false
+		for i, j := range journey {
+			if j.Definition == r.Definition {
+
+				foundDup := resolver.specIndex.GetMappedReferences()[r.Definition]
+
+				var circRef *CircularReferenceResult
+				if !foundDup.Circular {
+
+					loop := append(journey, foundDup)
+					circRef = &CircularReferenceResult{
+						Journey:   loop,
+						Start:     foundDup,
+						LoopIndex: i,
+						LoopPoint: foundDup,
+					}
+
+					foundDup.Seen = true
+					foundDup.Circular = true
+					resolver.circularReferences = append(resolver.circularReferences, circRef)
+
+				}
+				skip = true
+
+			}
+		}
+		if !skip {
+			original := resolver.specIndex.GetMappedReferences()[r.Definition]
+			resolved := resolver.VisitReference(original, seen, journey)
+			r.Node.Content = resolved // this is where we perform the actual resolving.
+			r.Seen = true
+			ref.Seen = true
+		}
+	}
+	ref.Resolved = true
+	ref.Seen = true
+
+	return ref.Node.Content
+}
+
+func (resolver *Resolver) extractRelatives(node *yaml.Node,
+	foundRelatives map[string]bool,
+	journey []*index.Reference) []*index.Reference {
+
+	var found []*index.Reference
+	if len(node.Content) > 0 {
+		for i, n := range node.Content {
+			if utils.IsNodeMap(n) || utils.IsNodeArray(n) {
+				found = append(found, resolver.extractRelatives(n, foundRelatives, journey)...)
+			}
+
+			if i%2 == 0 && n.Value == "$ref" {
+
+				if !utils.IsNodeStringValue(node.Content[i+1]) {
+					continue
+				}
+
+				value := node.Content[i+1].Value
+				ref := resolver.specIndex.GetMappedReferences()[value]
+
+				if ref == nil {
+					// TODO handle error, missing ref, can't resolve.
+					_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value)
+					err := &ResolvingError{
+						Error: fmt.Errorf("cannot resolve reference `%s`, it's missing", value),
+						Node:  n,
+						Path:  path,
+					}
+					resolver.resolvingErrors = append(resolver.resolvingErrors, err)
+					continue
+				}
+
+				r := &index.Reference{
+					Definition: value,
+					Name:       value,
+					Node:       node,
+				}
+
+				found = append(found, r)
+
+				foundRelatives[value] = true
+			}
+
+			if i%2 == 0 && n.Value != "$ref" && n.Value != "" {
+
+				if n.Value == "allOf" ||
+					n.Value == "oneOf" ||
+					n.Value == "anyOf" {
+
+					// TODO: track this.
+
+					break
+				}
+			}
+
+		}
+	}
+
+	return found
+}

resolver/resolver_test.go

@@ -0,0 +1,89 @@
+package resolver
+
+import (
+    "github.com/pb33f/libopenapi/index"
+    "github.com/stretchr/testify/assert"
+    "gopkg.in/yaml.v3"
+    "io/ioutil"
+    "testing"
+)
+
+func TestNewResolver(t *testing.T) {
+    assert.Nil(t, NewResolver(nil))
+}
+
+func Benchmark_ResolveDocumentStripe(b *testing.B) {
+    stripe, _ := ioutil.ReadFile("../test_specs/stripe.yaml")
+    for n := 0; n < b.N; n++ {
+        var rootNode yaml.Node
+        yaml.Unmarshal(stripe, &rootNode)
+        index := index.NewSpecIndex(&rootNode)
+        resolver := NewResolver(index)
+        resolver.Resolve()
+    }
+}
+
+func TestResolver_ResolveComponents_CircularSpec(t *testing.T) {
+
+    circular, _ := ioutil.ReadFile("../test_specs/circular-tests.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(circular, &rootNode)
+
+    index := index.NewSpecIndex(&rootNode)
+
+    resolver := NewResolver(index)
+    assert.NotNil(t, resolver)
+
+    circ := resolver.Resolve()
+    assert.Len(t, circ, 3)
+
+    _, err := yaml.Marshal(resolver.resolvedRoot)
+    assert.NoError(t, err)
+}
+
+func TestResolver_ResolveComponents_Stripe(t *testing.T) {
+
+    stripe, _ := ioutil.ReadFile("../test_specs/stripe.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(stripe, &rootNode)
+
+    index := index.NewSpecIndex(&rootNode)
+
+    resolver := NewResolver(index)
+    assert.NotNil(t, resolver)
+
+    circ := resolver.Resolve()
+    assert.Len(t, circ, 0)
+
+}
+
+func TestResolver_ResolveComponents_MixedRef(t *testing.T) {
+
+    mixedref, _ := ioutil.ReadFile("../test_specs/mixedref-burgershop.openapi.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(mixedref, &rootNode)
+
+    index := index.NewSpecIndex(&rootNode)
+
+    resolver := NewResolver(index)
+    assert.NotNil(t, resolver)
+
+    circ := resolver.Resolve()
+    assert.Len(t, circ, 2)
+
+}
+
+func TestResolver_ResolveComponents_k8s(t *testing.T) {
+
+    k8s, _ := ioutil.ReadFile("../test_specs/k8s.json")
+    var rootNode yaml.Node
+    yaml.Unmarshal(k8s, &rootNode)
+
+    index := index.NewSpecIndex(&rootNode)
+
+    resolver := NewResolver(index)
+    assert.NotNil(t, resolver)
+
+    circ := resolver.Resolve()
+    assert.Len(t, circ, 0)
+}

test_specs/circular-tests.yaml

@@ -0,0 +1,58 @@
+paths:
+  /burgers:
+    post:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/Nine'
+components:
+  schemas:
+    One:
+      description: "test one"
+      properties:
+        things:
+          "$ref": "#/components/schemas/Two"
+    Two:
+      decription: "test two"
+      properties:
+        testThing:
+          "$ref": "#/components/schemas/One"
+        anyOf:
+          - "$ref": "#/components/schemas/Four"
+    Three:
+      description: "test three"
+      properties:
+        tester:
+          "$ref": "#/components/schemas/Four"
+        bester:
+          "$ref": "#/components/schemas/Seven"
+        yester:
+          "$ref": "#/components/schemas/Seven"
+    Four:
+      desription: "test four"
+      properties:
+        lemons:
+          "$ref": "#/components/schemas/Nine"
+    Five:
+      properties:
+        rice:
+          "$ref": "#/components/schemas/Six"
+    Six:
+      properties:
+        mints:
+          "$ref": "#/components/schemas/Nine"
+    Seven:
+      properties:
+        wow:
+          "$ref": "#/components/schemas/Three"
+    Nine:
+      description: done.
+    Ten:
+      properties:
+        yeah:
+          "$ref": "#/components/schemas/Ten"
+
+
+
+

test_specs/mixedref-burgershop.openapi.yaml

@@ -0,0 +1,241 @@
+openapi: 3.0.1
+info:
+  title: Burger Shop
+  description: |
+    The best burger API at quobix. You can find the testiest burgers on the world
+  termsOfService: https://quobix.com
+  contact:
+    name: quobix
+  license:
+    name: Quobix
+  version: "1.2"
+tags:
+  - name: "pizza"
+    description: false
+    externalDocs:
+      description: "Find out more"
+      url: "https://quobix.com/"
+  - name: "Dressing"
+    description: "Variety of dressings: cheese, veggie, oil and a lot more"
+    externalDocs:
+      description: "Find out more information about our products)"
+      url: "https://quobix.com/"
+servers:
+  - url: https://quobix.com/api
+paths:
+  /burgers:
+    post:
+      operationId: createBurger
+      tags:
+        - "Meat"
+      summary:  Create a new burger
+      description: A new burger for our menu, yummy yum yum.
+      requestBody:
+        description: Give us the new burger!
+        content:
+          application/json:
+            schema:
+              $ref: 'https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml#/components/schemas/Burger'
+            examples:
+              pbjBurger:
+                summary: A horrible, nutty, sticky mess.
+                value:
+                  name: Peanut And Jelly
+                  numPatties: 3
+              cakeBurger:
+                summary: A sickly, sweet, atrocity
+                value:
+                  name: Chocolate Cake Burger
+                  numPatties: 5
+      responses:
+        "200":
+          description: A tasty burger for you to eat.
+          content:
+            application/json:
+              schema:
+                $ref: 'https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml#/components/schemas/Burger'
+              examples:
+                quarterPounder:
+                  summary: A juicy two handler sammich
+                  value:
+                    name: Quarter Pounder with Cheese
+                    numPatties: 1
+                filetOFish:
+                  summary: A tasty treat from the sea
+                  value:
+                    name: Filet-O-Fish
+                    numPatties: 1
+        "500":
+          description: Unexpected error creating a new burger. Sorry.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+              examples:
+                unexpectedError:
+                  summary: oh my goodness
+                  value:
+                    message: something went terribly wrong my friend, no new burger for you.
+        "422":
+          description: Unprocessable entity
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+              examples:
+                unexpectedError:
+                  summary: invalid request
+                  value:
+                    message: unable to accept this request, looks bad, missing something.
+  /burgers/{burgerId}:
+    get:
+      operationId: locateBurger
+      tags:
+        - "Meat"
+      summary: Search a burger by ID - returns the burger with that identifier
+      description: Look up a tasty burger take it and enjoy it
+      parameters:
+        - in: path
+          name: burgerId
+          schema:
+            type: string
+          example: big-mac
+          description: the name of the burger. use this to order your food
+          required: true
+      responses:
+        "200":
+          description: A tasty burger for you to eat. Wide variety of products to choose from
+          content:
+            application/json:
+              schema:
+                $ref: 'https://raw.githubusercontent.com/daveshanley/vacuum/main/model/test_files/burgershop.openapi.yaml#/components/schemas/Fries'
+              examples:
+                quarterPounder:
+                  summary: A juicy two handler sammich
+                  value:
+                    name: Quarter Pounder with Cheese
+                    numPatties: 1
+                filetOFish:
+                  summary: A tasty treat from the sea
+                  value:
+                    name: Filet-O-Fish
+                    numPatties: 1
+
+        "404":
+          description: Cannot find your burger. Sorry. We may have sold out of this type
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+              examples:
+                notFound:
+                  summary: burger missing
+                  value:
+                    message: can't find a burger with that ID, we may have sold out my friend.
+        "500":
+          description: Unexpected error. Sorry.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+              examples:
+                unexpectedError:
+                  summary: oh my stars
+                  value:
+                    message: something went terribly wrong my friend, burger location crashed!
+  /burgers/{burgerId}/dressings:
+    get:
+      operationId: listBurgerDressings
+      tags:
+        - "Dressing"
+      summary:  Get a list of all dressings available
+      description: Same as the summary, look up a tasty burger, by its ID - the burger identifier
+      parameters:
+        - in: path
+          name: burgerId
+          schema:
+            type: string
+          example: big-mac
+          description: the name of the our fantastic burger. You can pick a name from our menu
+          required: true
+      responses:
+        "200":
+          description: an array of
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: 'test_files/burgershop.openapi.yaml#/components/schemas/Dressing'
+        "404":
+          description: Cannot find your burger in which to list dressings. Sorry
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        "500":
+          description: Unexpected error listing dressings for burger. Sorry.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+  /dressings/{dressingId}:
+    get:
+      operationId: getDressing
+      tags:
+        - "Dressing"
+      summary:  Get a specific dressing - you can choose the dressing from our menu
+      description: Same as the summary, get a dressing, by its ID
+      parameters:
+        - in: path
+          name: dressingId
+          schema:
+            type: string
+          example: cheese
+          description: This is the unique identifier for the dressing items.
+          required: true
+      responses:
+        "404":
+          description: Cannot find your dressing, sorry.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+        "500":
+          description: Unexpected error getting a dressing. Sorry.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+  /dressings:
+    get:
+      operationId: getAllDressings
+      tags:
+        - "Dressing"
+      summary:  Get all dressings available in our store
+      description: Get all dressings and choose from them
+      responses:
+        "200":
+          description: an array of dressings
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: 'test_files/burgershop.openapi.yaml#/components/schemas/Dressing'
+        "500":
+          description: Unexpected error. Sorry.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'
+components:
+  schemas:
+    Error:
+      type: object
+      description: Error defining what went wrong when providing a specification. The message should help indicate the issue clearly.
+      properties:
+        message:
+          type: string
+          description: returns the error message if something wrong happens
+          example: No such burger as 'Big-Whopper'