Ported over universal utils and models from vacuum.

libopenapi.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/pb33f/libopenapi/libopenapi/utils"
+
+func main() {
+    utils.BuildPath("nope", []string{"one"})
+}

model/model_utils.go

@@ -0,0 +1,212 @@
+package model
+
+import (
+    _ "embed"
+    "encoding/json"
+    "errors"
+    "fmt"
+    "github.com/daveshanley/vacuum/utils"
+    "gopkg.in/yaml.v3"
+    "strings"
+)
+
+const (
+    OAS2  = "oas2"
+    OAS3  = "oas3"
+    OAS31 = "oas3_1"
+)
+
+//go:embed schemas/oas3-schema.json
+var OpenAPI3SchemaData string
+
+//go:embed schemas/swagger2-schema.json
+var OpenAPI2SchemaData string
+
+var OAS3_1Format = []string{OAS31}
+var OAS3Format = []string{OAS3}
+var OAS3AllFormat = []string{OAS3, OAS31}
+var OAS2Format = []string{OAS2}
+var AllFormats = []string{OAS3, OAS31, OAS2}
+
+// ExtractSpecInfo will look at a supplied OpenAPI specification, and return a *SpecInfo pointer, or an error
+// if the spec cannot be parsed correctly.
+func ExtractSpecInfo(spec []byte) (*SpecInfo, error) {
+
+    var parsedSpec yaml.Node
+
+    specVersion := &SpecInfo{}
+    specVersion.jsonParsingChannel = make(chan bool)
+
+    // set original bytes
+    specVersion.SpecBytes = &spec
+
+    runes := []rune(strings.TrimSpace(string(spec)))
+    if len(runes) <= 0 {
+        return specVersion, errors.New("there are no runes in the spec")
+    }
+
+    if runes[0] == '{' && runes[len(runes)-1] == '}' {
+        specVersion.SpecFileType = "json"
+    } else {
+        specVersion.SpecFileType = "yaml"
+    }
+
+    err := yaml.Unmarshal(spec, &parsedSpec)
+    if err != nil {
+        return nil, fmt.Errorf("unable to parse specification: %s", err.Error())
+    }
+
+    specVersion.RootNode = &parsedSpec
+
+    _, openAPI3 := utils.FindKeyNode(utils.OpenApi3, parsedSpec.Content)
+    _, openAPI2 := utils.FindKeyNode(utils.OpenApi2, parsedSpec.Content)
+    _, asyncAPI := utils.FindKeyNode(utils.AsyncApi, parsedSpec.Content)
+
+    parseJSON := func(bytes []byte, spec *SpecInfo) {
+        var jsonSpec map[string]interface{}
+
+        // no point in worrying about errors here, extract JSON friendly format.
+        // run in a separate thread, don't block.
+
+        if spec.SpecType == utils.OpenApi3 {
+            spec.APISchema = OpenAPI3SchemaData
+        }
+        if spec.SpecType == utils.OpenApi2 {
+            spec.APISchema = OpenAPI2SchemaData
+        }
+
+        if utils.IsYAML(string(bytes)) {
+            yaml.Unmarshal(bytes, &jsonSpec)
+            jsonData, _ := json.Marshal(jsonSpec)
+            spec.SpecJSONBytes = &jsonData
+            spec.SpecJSON = &jsonSpec
+        } else {
+            json.Unmarshal(bytes, &jsonSpec)
+            spec.SpecJSONBytes = &bytes
+            spec.SpecJSON = &jsonSpec
+        }
+        spec.jsonParsingChannel <- true
+        close(spec.jsonParsingChannel)
+    }
+    // check for specific keys
+    if openAPI3 != nil {
+        specVersion.SpecType = utils.OpenApi3
+        version, majorVersion := parseVersionTypeData(openAPI3.Value)
+
+        // parse JSON
+        go parseJSON(spec, specVersion)
+
+        // double check for the right version, people mix this up.
+        if majorVersion < 3 {
+            specVersion.Error = errors.New("spec is defined as an openapi spec, but is using a swagger (2.0), or unknown version")
+            return specVersion, specVersion.Error
+        }
+        specVersion.Version = version
+        specVersion.SpecFormat = OAS3
+    }
+    if openAPI2 != nil {
+        specVersion.SpecType = utils.OpenApi2
+        version, majorVersion := parseVersionTypeData(openAPI2.Value)
+
+        // parse JSON
+        go parseJSON(spec, specVersion)
+
+        // I am not certain this edge-case is very frequent, but let's make sure we handle it anyway.
+        if majorVersion > 2 {
+            specVersion.Error = errors.New("spec is defined as a swagger (openapi 2.0) spec, but is an openapi 3 or unknown version")
+            return specVersion, specVersion.Error
+        }
+        specVersion.Version = version
+        specVersion.SpecFormat = OAS2
+    }
+    if asyncAPI != nil {
+        specVersion.SpecType = utils.AsyncApi
+        version, majorVersion := parseVersionTypeData(asyncAPI.Value)
+
+        // parse JSON
+        go parseJSON(spec, specVersion)
+
+        // so far there is only 2 as a major release of AsyncAPI
+        if majorVersion > 2 {
+            specVersion.Error = errors.New("spec is defined as asyncapi, but has a major version that is invalid")
+            return specVersion, specVersion.Error
+        }
+        specVersion.Version = version
+        // TODO: format for AsyncAPI.
+
+    }
+
+    if specVersion.SpecType == "" {
+
+        // parse JSON
+        go parseJSON(spec, specVersion)
+
+        specVersion.Error = errors.New("spec type not supported by vacuum, sorry")
+        return specVersion, specVersion.Error
+    }
+
+    return specVersion, nil
+}
+
+func parseVersionTypeData(d interface{}) (string, int) {
+    r := []rune(strings.TrimSpace(fmt.Sprintf("%v", d)))
+    return string(r), int(r[0]) - '0'
+}
+
+// AreValuesCorrectlyTyped will look through an array of unknown values and check they match
+// against the supplied type as a string. The return value is empty if everything is OK, or it
+// contains failures in the form of a value as a key and a message as to why it's not valid
+func AreValuesCorrectlyTyped(valType string, values interface{}) map[string]string {
+    var arr []interface{}
+    if _, ok := values.([]interface{}); !ok {
+        return nil
+    }
+    arr = values.([]interface{})
+
+    results := make(map[string]string)
+    for _, v := range arr {
+        switch v.(type) {
+        case string:
+            if valType != "string" {
+                results[v.(string)] = fmt.Sprintf("enum value '%v' is a "+
+                    "string, but it's defined as a '%v'", v, valType)
+            }
+        case int64:
+            if valType != "integer" && valType != "number" {
+                results[fmt.Sprintf("%v", v)] = fmt.Sprintf("enum value '%v' is a "+
+                    "integer, but it's defined as a '%v'", v, valType)
+            }
+        case int:
+            if valType != "integer" && valType != "number" {
+                results[fmt.Sprintf("%v", v)] = fmt.Sprintf("enum value '%v' is a "+
+                    "integer, but it's defined as a '%v'", v, valType)
+            }
+        case float64:
+            if valType != "number" {
+                results[fmt.Sprintf("%v", v)] = fmt.Sprintf("enum value '%v' is a "+
+                    "number, but it's defined as a '%v'", v, valType)
+            }
+        case bool:
+            if valType != "boolean" {
+                results[fmt.Sprintf("%v", v)] = fmt.Sprintf("enum value '%v' is a "+
+                    "boolean, but it's defined as a '%v'", v, valType)
+            }
+        }
+    }
+    return results
+}
+
+// CheckEnumForDuplicates will check an array of nodes to check if there are any duplicates.
+func CheckEnumForDuplicates(seq []*yaml.Node) []*yaml.Node {
+    var res []*yaml.Node
+    seen := make(map[string]*yaml.Node)
+
+    for _, enum := range seq {
+        if seen[enum.Value] != nil {
+            res = append(res, enum)
+            continue
+        }
+        seen[enum.Value] = enum
+    }
+    return res
+}

model/model_utils_test.go

@@ -0,0 +1,234 @@
+package model
+
+import (
+    "github.com/daveshanley/vacuum/utils"
+    "github.com/stretchr/testify/assert"
+    "gopkg.in/yaml.v3"
+    "testing"
+)
+
+const (
+    // OpenApi3 is used by all OpenAPI 3+ docs
+    OpenApi3 = "openapi"
+
+    // OpenApi2 is used by all OpenAPI 2 docs, formerly known as swagger.
+    OpenApi2 = "swagger"
+
+    // AsyncApi is used by akk AsyncAPI docs, all versions.
+    AsyncApi = "asyncapi"
+)
+
+var goodJSON = `{"name":"kitty", "noises":["meow","purrrr","gggrrraaaaaooooww"]}`
+var badJSON = `{"name":"kitty, "noises":[{"meow","purrrr","gggrrraaaaaooooww"]}}`
+var goodYAML = `name: kitty
+noises:
+- meow
+- purrr
+- gggggrrraaaaaaaaaooooooowwwwwww
+`
+
+var badYAML = `name: kitty
+  noises:
+   - meow
+    - purrr
+    - gggggrrraaaaaaaaaooooooowwwwwww
+`
+
+var OpenApiWat = `openapi: 3.2
+info:
+  title: Test API, valid, but not quite valid 
+servers:
+  - url: https://quobix.com/api`
+
+var OpenApiFalse = `openapi: false
+info:
+  title: Test API version is a bool?
+servers:
+  - url: https://quobix.com/api`
+
+var OpenApiOne = `openapi: 1.0.1
+info:
+  title: Test API version is what version?
+servers:
+  - url: https://quobix.com/api`
+
+var OpenApi3Spec = `openapi: 3.0.1
+info:
+  title: Test API
+tags:
+  - name: "Test"
+  - name: "Test 2"
+servers:
+  - url: https://quobix.com/api`
+
+var OpenApi2Spec = `swagger: 2.0.1
+info:
+  title: Test API
+tags:
+  - name: "Test"
+servers:
+  - url: https://quobix.com/api`
+
+var OpenApi2SpecOdd = `swagger: 3.0.1
+info:
+  title: Test API
+tags:
+  - name: "Test"
+servers:
+  - url: https://quobix.com/api`
+
+var AsyncAPISpec = `asyncapi: 2.0.0
+info:
+  title: Hello world application
+  version: '0.1.0'
+channels:
+  hello:
+    publish:
+      message:
+        payload:
+          type: string
+          pattern: '^hello .+$'`
+
+var AsyncAPISpecOdd = `asyncapi: 3.0.0
+info:
+  title: Hello world application
+  version: '0.1.0'`
+
+func TestExtractSpecInfo_ValidJSON(t *testing.T) {
+    _, e := ExtractSpecInfo([]byte(goodJSON))
+    assert.Error(t, e)
+}
+
+func TestExtractSpecInfo_InvalidJSON(t *testing.T) {
+    _, e := ExtractSpecInfo([]byte(badJSON))
+    assert.Error(t, e)
+}
+
+func TestExtractSpecInfo_Nothing(t *testing.T) {
+    _, e := ExtractSpecInfo([]byte(""))
+    assert.Error(t, e)
+}
+
+func TestExtractSpecInfo_ValidYAML(t *testing.T) {
+    _, e := ExtractSpecInfo([]byte(goodYAML))
+    assert.Error(t, e)
+}
+
+func TestExtractSpecInfo_InvalidYAML(t *testing.T) {
+    _, e := ExtractSpecInfo([]byte(badYAML))
+    assert.Error(t, e)
+}
+
+func TestExtractSpecInfo_InvalidOpenAPIVersion(t *testing.T) {
+    _, e := ExtractSpecInfo([]byte(OpenApiOne))
+    assert.Error(t, e)
+}
+
+func TestExtractSpecInfo_OpenAPI3(t *testing.T) {
+
+    r, e := ExtractSpecInfo([]byte(OpenApi3Spec))
+    assert.Nil(t, e)
+    assert.Equal(t, utils.OpenApi3, r.SpecType)
+    assert.Equal(t, "3.0.1", r.Version)
+}
+
+func TestExtractSpecInfo_OpenAPIWat(t *testing.T) {
+
+    r, e := ExtractSpecInfo([]byte(OpenApiWat))
+    assert.Nil(t, e)
+    assert.Equal(t, OpenApi3, r.SpecType)
+    assert.Equal(t, "3.2", r.Version)
+}
+
+func TestExtractSpecInfo_OpenAPIFalse(t *testing.T) {
+
+    spec, e := ExtractSpecInfo([]byte(OpenApiFalse))
+    assert.NoError(t, e)
+    assert.Equal(t, "false", spec.Version)
+}
+
+func TestExtractSpecInfo_OpenAPI2(t *testing.T) {
+
+    r, e := ExtractSpecInfo([]byte(OpenApi2Spec))
+    assert.Nil(t, e)
+    assert.Equal(t, OpenApi2, r.SpecType)
+    assert.Equal(t, "2.0.1", r.Version)
+}
+
+func TestExtractSpecInfo_OpenAPI2_OddVersion(t *testing.T) {
+
+    _, e := ExtractSpecInfo([]byte(OpenApi2SpecOdd))
+    assert.NotNil(t, e)
+    assert.Equal(t,
+        "spec is defined as a swagger (openapi 2.0) spec, but is an openapi 3 or unknown version", e.Error())
+}
+
+func TestExtractSpecInfo_AsyncAPI(t *testing.T) {
+
+    r, e := ExtractSpecInfo([]byte(AsyncAPISpec))
+    assert.Nil(t, e)
+    assert.Equal(t, AsyncApi, r.SpecType)
+    assert.Equal(t, "2.0.0", r.Version)
+}
+
+func TestExtractSpecInfo_AsyncAPI_OddVersion(t *testing.T) {
+
+    _, e := ExtractSpecInfo([]byte(AsyncAPISpecOdd))
+    assert.NotNil(t, e)
+    assert.Equal(t,
+        "spec is defined as asyncapi, but has a major version that is invalid", e.Error())
+}
+
+func TestAreValuesCorrectlyTyped(t *testing.T) {
+
+    assert.Len(t, AreValuesCorrectlyTyped("string", []interface{}{"hi"}), 0)
+    assert.Len(t, AreValuesCorrectlyTyped("string", []interface{}{1}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("string", []interface{}{"nice", 123, int64(12345)}), 2)
+    assert.Len(t, AreValuesCorrectlyTyped("string", []interface{}{1.2, "burgers"}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("string", []interface{}{true, false, "what"}), 2)
+
+    assert.Len(t, AreValuesCorrectlyTyped("integer", []interface{}{1, 2, 3, 4}), 0)
+    assert.Len(t, AreValuesCorrectlyTyped("integer", []interface{}{"no way!"}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("integer", []interface{}{"nice", 123, int64(12345)}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("integer", []interface{}{999, 1.2, "burgers"}), 2)
+    assert.Len(t, AreValuesCorrectlyTyped("integer", []interface{}{true, false, "what"}), 3)
+
+    assert.Len(t, AreValuesCorrectlyTyped("number", []interface{}{1.2345}), 0)
+    assert.Len(t, AreValuesCorrectlyTyped("number", []interface{}{"no way!"}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("number", []interface{}{"nice", 123, 2.353}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("number", []interface{}{999, 1.2, "burgers"}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("number", []interface{}{true, false, "what"}), 3)
+
+    assert.Len(t, AreValuesCorrectlyTyped("boolean", []interface{}{true, false, true}), 0)
+    assert.Len(t, AreValuesCorrectlyTyped("boolean", []interface{}{"no way!"}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("boolean", []interface{}{"nice", 123, 2.353, true}), 3)
+    assert.Len(t, AreValuesCorrectlyTyped("boolean", []interface{}{true, true, "burgers"}), 1)
+    assert.Len(t, AreValuesCorrectlyTyped("boolean", []interface{}{true, false, "what", 1.2, 4}), 3)
+
+    assert.Nil(t, AreValuesCorrectlyTyped("boolean", []string{"hi"}))
+
+}
+
+func TestCheckEnumForDuplicates_Success(t *testing.T) {
+    yml := "- yes\n- no\n- crisps"
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+    assert.Len(t, CheckEnumForDuplicates(rootNode.Content[0].Content), 0)
+
+}
+
+func TestCheckEnumForDuplicates_Fail(t *testing.T) {
+    yml := "- yes\n- no\n- crisps\n- no"
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+    assert.Len(t, CheckEnumForDuplicates(rootNode.Content[0].Content), 1)
+
+}
+
+func TestCheckEnumForDuplicates_FailMultiple(t *testing.T) {
+    yml := "- yes\n- no\n- crisps\n- no\n- rice\n- yes\n- no"
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+    assert.Len(t, CheckEnumForDuplicates(rootNode.Content[0].Content), 3)
+}

model/reports/spectral.go

@@ -0,0 +1,23 @@
+package reports
+
+// SpectralReport represents a model that can be deserialized into a spectral compatible output.
+type SpectralReport struct {
+	Code     string   `json:"code" yaml:"code"`         // the rule that was run
+	Path     []string `json:"path" yaml:"path"`         // the path to the item, broken down into a slice
+	Message  string   `json:"message" yaml:"message"`   // the result message
+	Severity int      `json:"severity" yaml:"severity"` // the severity reported
+	Range    Range    `json:"range" yaml:"range"`       // the location of the issue in the spec.
+	Source   string   `json:"source" yaml:"source"`     // the source of the report.
+}
+
+// Range indicates the start and end of a report item
+type Range struct {
+	Start RangeItem `json:"start" yaml:"start"`
+	End   RangeItem `json:"end" yaml:"end"`
+}
+
+// RangeItem indicates the line and character of a range.
+type RangeItem struct {
+	Line int `json:"line" yaml:"line"`
+	Char int `json:"character" yaml:"character"`
+}

model/reports/statistics.go

@@ -0,0 +1,39 @@
+package reports
+
+// ReportStatistics represents statistics for an individual specification report.
+type ReportStatistics struct {
+	FilesizeKB         int                  `json:"filesizeKb,omitempty" yaml:"filesizeKb,omitempty"`
+	FilesizeBytes      int                  `json:"filesizeBytes,omitempty" yaml:"filesizeBytes,omitempty"`
+	SpecType           string               `json:"specType,omitempty" yaml:"specType,omitempty"`
+	SpecFormat         string               `json:"specFormat,omitempty" yaml:"specFormat,omitempty"`
+	Version            string               `json:"version,omitempty" yaml:"version,omitempty"`
+	References         int                  `json:"references,omitempty" yaml:"references,omitempty"`
+	ExternalDocs       int                  `json:"externalDocs,omitempty" yaml:"externalDocs,omitempty"`
+	Schemas            int                  `json:"schemas,omitempty" yaml:"schemas,omitempty"`
+	Parameters         int                  `json:"parameters,omitempty" yaml:"parameters,omitempty"`
+	Links              int                  `json:"links,omitempty" yaml:"links,omitempty"`
+	Paths              int                  `json:"paths,omitempty" yaml:"paths,omitempty"`
+	Operations         int                  `json:"operations,omitempty" yaml:"operations,omitempty"`
+	Tags               int                  `json:"tags,omitempty" yaml:"tags,omitempty"`
+	Examples           int                  `json:"examples,omitempty" yaml:"examples,omitempty"`
+	Enums              int                  `json:"enums,omitempty" yaml:"enums,omitempty"`
+	Security           int                  `json:"security,omitempty" yaml:"security,omitempty"`
+	OverallScore       int                  `json:"overallScore,omitempty" yaml:"overallScore,omitempty"`
+	TotalErrors        int                  `json:"totalErrors,omitempty" yaml:"totalErrors,omitempty"`
+	TotalWarnings      int                  `json:"totalWarnings,omitempty" yaml:"totalWarnings,omitempty"`
+	TotalInfo          int                  `json:"totalInfo,omitempty" yaml:"totalInfo,omitempty"`
+	TotalHints         int                  `json:"totalHints,omitempty" yaml:"totalHints,omitempty"`
+	CategoryStatistics []*CategoryStatistic `json:"categoryStatistics,omitempty" yaml:"categoryStatistics,omitempty"`
+}
+
+// CategoryStatistic represents the number of issues for a particular category
+type CategoryStatistic struct {
+	CategoryName string `json:"categoryName" yaml:"categoryName"`
+	CategoryId   string `json:"categoryId" yaml:"categoryId"`
+	NumIssues    int    `json:"numIssues" yaml:"numIssues"`
+	Score        int    `json:"score" yaml:"score"`
+	Warnings     int    `json:"warnings" yaml:"warnings"`
+	Errors       int    `json:"errors" yaml:"errors"`
+	Info         int    `json:"info" yaml:"info"`
+	Hints        int    `json:"hints" yaml:"hints"`
+}

model/spec.go

@@ -0,0 +1,35 @@
+package model
+
+import (
+	"gopkg.in/yaml.v3"
+	"time"
+)
+
+// SpecInfo represents information about a supplied specification.
+type SpecInfo struct {
+	SpecType           string                  `json:"type"`
+	Version            string                  `json:"version"`
+	SpecFormat         string                  `json:"format"`
+	SpecFileType       string                  `json:"fileType"`
+	RootNode           *yaml.Node              `json:"-"`     // reference to the root node of the spec.
+	SpecBytes          *[]byte                 `json:"bytes"` // the original bytes
+	SpecJSONBytes      *[]byte                 `json:"-"`     // original bytes converted to JSON
+	SpecJSON           *map[string]interface{} `json:"-"`     // standard JSON map of original bytes
+	Error              error                   `json:"-"`     // something go wrong?
+	APISchema          string                  `json:"-"`     // API Schema for supplied spec type (2 or 3)
+	Generated          time.Time               `json:"-"`
+	jsonParsingChannel chan bool
+}
+
+// SearchResult represents the position of a result in a specification.
+type SearchResult struct {
+	Key  string `json:"key"`
+	Line int    `json:"line"`
+	Col  int    `json:"col"`
+}
+
+// GetJSONParsingChannel returns a channel that will close once async JSON parsing is completed.
+// This is required as rules may start executing before we're even done reading in the spec to JSON.
+func (si SpecInfo) GetJSONParsingChannel() chan bool {
+	return si.jsonParsingChannel
+}

model/spec_index_test.go

@@ -0,0 +1,356 @@
+package model
+
+import (
+    "github.com/stretchr/testify/assert"
+    "gopkg.in/yaml.v3"
+    "io/ioutil"
+    "testing"
+)
+
+func TestSpecIndex_ExtractRefsStripe(t *testing.T) {
+
+    stripe, _ := ioutil.ReadFile("../test_specs/stripe.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(stripe, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 385)
+    assert.Len(t, index.allMappedRefs, 385)
+
+    combined := index.GetAllCombinedReferences()
+    assert.Equal(t, 537, len(combined))
+
+    assert.Len(t, index.rawSequencedRefs, 1972)
+    assert.Equal(t, 246, index.pathCount)
+    assert.Equal(t, 402, index.operationCount)
+    assert.Equal(t, 537, index.schemaCount)
+    assert.Equal(t, 0, index.globalTagsCount)
+    assert.Equal(t, 0, index.globalLinksCount)
+    assert.Equal(t, 0, index.componentParamCount)
+    assert.Equal(t, 143, index.operationParamCount)
+    assert.Equal(t, 88, index.componentsInlineParamDuplicateCount)
+    assert.Equal(t, 55, index.componentsInlineParamUniqueCount)
+    assert.Equal(t, 1516, index.enumCount)
+    assert.Len(t, index.GetAllEnums(), 1516)
+    assert.Len(t, index.GetPolyAllOfReferences(), 0)
+    assert.Len(t, index.GetPolyOneOfReferences(), 275)
+    assert.Len(t, index.GetPolyAnyOfReferences(), 553)
+    assert.NotNil(t, index.GetRootServersNode())
+    assert.Len(t, index.GetAllRootServers(), 1)
+
+}
+
+func TestSpecIndex_Asana(t *testing.T) {
+
+    asana, _ := ioutil.ReadFile("test_files/asana.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(asana, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 152)
+    assert.Len(t, index.allMappedRefs, 152)
+    combined := index.GetAllCombinedReferences()
+    assert.Equal(t, 171, len(combined))
+    assert.Equal(t, 118, index.pathCount)
+    assert.Equal(t, 152, index.operationCount)
+    assert.Equal(t, 135, index.schemaCount)
+    assert.Equal(t, 26, index.globalTagsCount)
+    assert.Equal(t, 0, index.globalLinksCount)
+    assert.Equal(t, 30, index.componentParamCount)
+    assert.Equal(t, 107, index.operationParamCount)
+    assert.Equal(t, 8, index.componentsInlineParamDuplicateCount)
+    assert.Equal(t, 69, index.componentsInlineParamUniqueCount)
+}
+
+func TestSpecIndex_k8s(t *testing.T) {
+
+    asana, _ := ioutil.ReadFile("test_files/k8s.json")
+    var rootNode yaml.Node
+    yaml.Unmarshal(asana, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 558)
+    assert.Len(t, index.allMappedRefs, 558)
+    combined := index.GetAllCombinedReferences()
+    assert.Equal(t, 563, len(combined))
+    assert.Equal(t, 436, index.pathCount)
+    assert.Equal(t, 853, index.operationCount)
+    assert.Equal(t, 563, index.schemaCount)
+    assert.Equal(t, 0, index.globalTagsCount)
+    assert.Equal(t, 58, index.operationTagsCount)
+    assert.Equal(t, 0, index.globalLinksCount)
+    assert.Equal(t, 0, index.componentParamCount)
+    assert.Equal(t, 36, index.operationParamCount)
+    assert.Equal(t, 26, index.componentsInlineParamDuplicateCount)
+    assert.Equal(t, 10, index.componentsInlineParamUniqueCount)
+    assert.Equal(t, 58, index.GetTotalTagsCount())
+    assert.Equal(t, 2524, index.GetRawReferenceCount())
+
+}
+
+func TestSpecIndex_PetstoreV2(t *testing.T) {
+
+    asana, _ := ioutil.ReadFile("test_files/petstorev2.json")
+    var rootNode yaml.Node
+    yaml.Unmarshal(asana, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 6)
+    assert.Len(t, index.allMappedRefs, 6)
+    assert.Equal(t, 14, index.pathCount)
+    assert.Equal(t, 20, index.operationCount)
+    assert.Equal(t, 6, index.schemaCount)
+    assert.Equal(t, 3, index.globalTagsCount)
+    assert.Equal(t, 3, index.operationTagsCount)
+    assert.Equal(t, 0, index.globalLinksCount)
+    assert.Equal(t, 1, index.componentParamCount)
+    assert.Equal(t, 1, index.GetComponentParameterCount())
+    assert.Equal(t, 11, index.operationParamCount)
+    assert.Equal(t, 5, index.componentsInlineParamDuplicateCount)
+    assert.Equal(t, 6, index.componentsInlineParamUniqueCount)
+    assert.Equal(t, 3, index.GetTotalTagsCount())
+}
+
+func TestSpecIndex_XSOAR(t *testing.T) {
+
+    xsoar, _ := ioutil.ReadFile("test_files/xsoar.json")
+    var rootNode yaml.Node
+    yaml.Unmarshal(xsoar, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+    assert.Len(t, index.allRefs, 209)
+    assert.Equal(t, 85, index.pathCount)
+    assert.Equal(t, 88, index.operationCount)
+    assert.Equal(t, 245, index.schemaCount)
+    assert.Equal(t, 207, len(index.allMappedRefs))
+    assert.Equal(t, 0, index.globalTagsCount)
+    assert.Equal(t, 0, index.operationTagsCount)
+    assert.Equal(t, 0, index.globalLinksCount)
+    assert.Len(t, index.GetRootSecurityReferences(), 1)
+    assert.NotNil(t, index.GetRootSecurityNode())
+}
+
+func TestSpecIndex_PetstoreV3(t *testing.T) {
+
+    asana, _ := ioutil.ReadFile("test_files/petstorev3.json")
+    var rootNode yaml.Node
+    yaml.Unmarshal(asana, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 7)
+    assert.Len(t, index.allMappedRefs, 7)
+    assert.Equal(t, 13, index.pathCount)
+    assert.Equal(t, 19, index.operationCount)
+    assert.Equal(t, 8, index.schemaCount)
+    assert.Equal(t, 3, index.globalTagsCount)
+    assert.Equal(t, 3, index.operationTagsCount)
+    assert.Equal(t, 0, index.globalLinksCount)
+    assert.Equal(t, 0, index.componentParamCount)
+    assert.Equal(t, 9, index.operationParamCount)
+    assert.Equal(t, 4, index.componentsInlineParamDuplicateCount)
+    assert.Equal(t, 5, index.componentsInlineParamUniqueCount)
+    assert.Equal(t, 3, index.GetTotalTagsCount())
+    assert.Equal(t, 90, index.GetAllDescriptionsCount())
+    assert.Equal(t, 19, index.GetAllSummariesCount())
+    assert.Len(t, index.GetAllDescriptions(), 90)
+    assert.Len(t, index.GetAllSummaries(), 19)
+
+}
+
+func TestSpecIndex_BurgerShop(t *testing.T) {
+
+    burgershop, _ := ioutil.ReadFile("test_files/burgershop.openapi.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(burgershop, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 5)
+    assert.Len(t, index.allMappedRefs, 5)
+    assert.Equal(t, 5, len(index.GetMappedReferences()))
+    assert.Equal(t, 5, len(index.GetMappedReferencesSequenced()))
+
+    assert.Equal(t, 5, index.pathCount)
+    assert.Equal(t, 5, index.GetPathCount())
+
+    assert.Equal(t, 5, len(index.GetAllSchemas()))
+
+    assert.Equal(t, 18, len(index.GetAllSequencedReferences()))
+    assert.NotNil(t, index.GetSchemasNode())
+    assert.Nil(t, index.GetParametersNode())
+
+    assert.Equal(t, 5, index.operationCount)
+    assert.Equal(t, 5, index.GetOperationCount())
+
+    assert.Equal(t, 5, index.schemaCount)
+    assert.Equal(t, 5, index.GetComponentSchemaCount())
+
+    assert.Equal(t, 2, index.globalTagsCount)
+    assert.Equal(t, 2, index.GetGlobalTagsCount())
+    assert.Equal(t, 2, index.GetTotalTagsCount())
+
+    assert.Equal(t, 2, index.operationTagsCount)
+    assert.Equal(t, 2, index.GetOperationTagsCount())
+
+    assert.Equal(t, 3, index.globalLinksCount)
+    assert.Equal(t, 3, index.GetGlobalLinksCount())
+
+    assert.Equal(t, 0, index.componentParamCount)
+    assert.Equal(t, 0, index.GetComponentParameterCount())
+
+    assert.Equal(t, 2, index.operationParamCount)
+    assert.Equal(t, 2, index.GetOperationsParameterCount())
+
+    assert.Equal(t, 1, index.componentsInlineParamDuplicateCount)
+    assert.Equal(t, 1, index.GetInlineDuplicateParamCount())
+
+    assert.Equal(t, 1, index.componentsInlineParamUniqueCount)
+    assert.Equal(t, 1, index.GetInlineUniqueParamCount())
+
+    assert.Equal(t, 0, len(index.GetAllRequestBodies()))
+    assert.NotNil(t, index.GetRootNode())
+    assert.NotNil(t, index.GetGlobalTagsNode())
+    assert.NotNil(t, index.GetPathsNode())
+    assert.NotNil(t, index.GetDiscoveredReferences())
+    assert.Equal(t, 0, len(index.GetPolyReferences()))
+    assert.NotNil(t, index.GetOperationParameterReferences())
+    assert.Equal(t, 0, len(index.GetAllSecuritySchemes()))
+    assert.Equal(t, 0, len(index.GetAllParameters()))
+    assert.Equal(t, 0, len(index.GetAllResponses()))
+    assert.Equal(t, 2, len(index.GetInlineOperationDuplicateParameters()))
+    assert.Equal(t, 0, len(index.GetReferencesWithSiblings()))
+    assert.Equal(t, 5, len(index.GetAllReferences()))
+    assert.Equal(t, 0, len(index.GetOperationParametersIndexErrors()))
+    assert.Equal(t, 5, len(index.GetAllPaths()))
+    assert.Equal(t, 5, len(index.GetOperationTags()))
+    assert.Equal(t, 3, len(index.GetAllParametersFromOperations()))
+
+}
+
+func TestSpecIndex_BurgerShop_AllTheComponents(t *testing.T) {
+
+    burgershop, _ := ioutil.ReadFile("test_files/all-the-components.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(burgershop, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Equal(t, 1, len(index.GetAllHeaders()))
+    assert.Equal(t, 1, len(index.GetAllLinks()))
+    assert.Equal(t, 1, len(index.GetAllCallbacks()))
+    assert.Equal(t, 1, len(index.GetAllExamples()))
+    assert.Equal(t, 1, len(index.GetAllResponses()))
+    assert.Equal(t, 2, len(index.GetAllRootServers()))
+    assert.Equal(t, 2, len(index.GetAllOperationsServers()))
+
+}
+
+func TestSpecIndex_SwaggerResponses(t *testing.T) {
+
+    yml := `swagger: 2.0
+responses:
+  niceResponse: 
+    description: hi`
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Equal(t, 1, len(index.GetAllResponses()))
+
+}
+
+func TestSpecIndex_NoNameParam(t *testing.T) {
+
+    yml := `paths:
+  /users/{id}:
+    parameters:
+    - in: path
+      name: id
+    - in: query
+    get:
+      parameters:
+        - in: path
+          name: id
+        - in: query`
+
+    var rootNode yaml.Node
+    yaml.Unmarshal([]byte(yml), &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Equal(t, 2, len(index.GetOperationParametersIndexErrors()))
+
+}
+
+func TestSpecIndex_NoRoot(t *testing.T) {
+
+    index := NewSpecIndex(nil)
+    refs := index.ExtractRefs(nil, nil, nil, 0, false, "")
+    docs := index.ExtractExternalDocuments(nil)
+    assert.Nil(t, docs)
+    assert.Nil(t, refs)
+    assert.Nil(t, index.FindComponent("nothing", nil))
+    assert.Equal(t, -1, index.GetOperationCount())
+    assert.Equal(t, -1, index.GetPathCount())
+    assert.Equal(t, -1, index.GetGlobalTagsCount())
+    assert.Equal(t, -1, index.GetOperationTagsCount())
+    assert.Equal(t, -1, index.GetTotalTagsCount())
+    assert.Equal(t, -1, index.GetOperationsParameterCount())
+    assert.Equal(t, -1, index.GetComponentParameterCount())
+    assert.Equal(t, -1, index.GetComponentSchemaCount())
+    assert.Equal(t, -1, index.GetGlobalLinksCount())
+
+}
+
+func TestSpecIndex_BurgerShopMixedRef(t *testing.T) {
+
+    spec, _ := ioutil.ReadFile("test_files/mixedref-burgershop.openapi.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(spec, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+
+    assert.Len(t, index.allRefs, 4)
+    assert.Len(t, index.allMappedRefs, 4)
+    assert.Equal(t, 5, index.GetPathCount())
+    assert.Equal(t, 5, index.GetOperationCount())
+    assert.Equal(t, 1, index.GetComponentSchemaCount())
+    assert.Equal(t, 2, index.GetGlobalTagsCount())
+    assert.Equal(t, 3, index.GetTotalTagsCount())
+    assert.Equal(t, 2, index.GetOperationTagsCount())
+    assert.Equal(t, 0, index.GetGlobalLinksCount())
+    assert.Equal(t, 0, index.GetComponentParameterCount())
+    assert.Equal(t, 2, index.GetOperationsParameterCount())
+    assert.Equal(t, 1, index.GetInlineDuplicateParamCount())
+    assert.Equal(t, 1, index.GetInlineUniqueParamCount())
+
+}
+
+func TestSpecIndex_TestEmptyBrokenReferences(t *testing.T) {
+
+    asana, _ := ioutil.ReadFile("test_files/badref-burgershop.openapi.yaml")
+    var rootNode yaml.Node
+    yaml.Unmarshal(asana, &rootNode)
+
+    index := NewSpecIndex(&rootNode)
+    assert.Equal(t, 5, index.GetPathCount())
+    assert.Equal(t, 5, index.GetOperationCount())
+    assert.Equal(t, 5, index.GetComponentSchemaCount())
+    assert.Equal(t, 2, index.GetGlobalTagsCount())
+    assert.Equal(t, 3, index.GetTotalTagsCount())
+    assert.Equal(t, 2, index.GetOperationTagsCount())
+    assert.Equal(t, 2, index.GetGlobalLinksCount())
+    assert.Equal(t, 0, index.GetComponentParameterCount())
+    assert.Equal(t, 2, index.GetOperationsParameterCount())
+    assert.Equal(t, 1, index.GetInlineDuplicateParamCount())
+    assert.Equal(t, 1, index.GetInlineUniqueParamCount())
+    assert.Len(t, index.refErrors, 7)
+
+}