fix: improves the ability to get all inline schemas and identify inline object schemas (#154)

* fix: improves the ability to get all inline schemas and identify inline object schemas

* fix

* fix

* fix

* fix

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/stretchr/testify v1.8.0
 	github.com/vmware-labs/yaml-jsonpath v0.3.2
+	golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
 	golang.org/x/sync v0.1.0
 	gopkg.in/yaml.v3 v3.0.1
 )

go.sum

@@ -64,6 +64,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA=
+golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -91,8 +93,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

index/extract_refs.go

@@ -6,9 +6,11 @@ package index
 import (
 	"errors"
 	"fmt"
-	"github.com/pb33f/libopenapi/utils"
-	"gopkg.in/yaml.v3"
 	"strings"
+
+	"github.com/pb33f/libopenapi/utils"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v3"
 )
 
 // ExtractRefs will return a deduplicated slice of references for every unique ref found in the document.
@@ -40,21 +42,26 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 			// check if we're dealing with an inline schema definition, that isn't part of an array
 			// (which means it's being used as a value in an array, and it's not a label)
 			// https://github.com/pb33f/libopenapi/issues/76
-			if i%2 == 0 && n.Value == "schema" && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
+			schemaContainingNodes := []string{"schema", "items", "additionalProperties", "contains", "not", "unevaluatedItems", "unevaluatedProperties"}
+			if i%2 == 0 && slices.Contains(schemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
+				ref := &Reference{
+					Node: node.Content[i+1],
+					Path: fmt.Sprintf("$.%s.%s", strings.Join(seenPath, "."), n.Value),
+				}
+
 				isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1])
 				if isRef {
 					// record this reference
-					ref := &Reference{
-						Node: node.Content[i+1],
-						Path: fmt.Sprintf("$.%s.schema", strings.Join(seenPath, ".")),
-					}
 					index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref)
 					continue
 				}
-				ref := &Reference{
+
-					Node: node.Content[i+1],
+				if n.Value == "additionalProperties" || n.Value == "unevaluatedProperties" {
-					Path: fmt.Sprintf("$.%s.schema", strings.Join(seenPath, ".")),
+					if utils.IsNodeBoolValue(node.Content[i+1]) {
+						continue
 					}
+				}
+
 				index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref)
 
 				// check if the schema is an object or an array,
@@ -67,14 +74,10 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 				}
 			}
 
-			// Perform the same check for all properties in an inline schema definition
+			// Perform the same check for all maps of schemas like properties and patternProperties
 			// https://github.com/pb33f/libopenapi/issues/76
-			if i%2 == 0 && n.Value == "properties" && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
+			mapOfSchemaContainingNodes := []string{"properties", "patternProperties"}
-				isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1])
+			if i%2 == 0 && slices.Contains(mapOfSchemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
-				if isRef {
-					continue
-				}
-
 				// for each property add it to our schema definitions
 				label := ""
 				for h, prop := range node.Content[i+1].Content {
@@ -83,16 +86,51 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 						label = prop.Value
 						continue
 					}
-
 					ref := &Reference{
 						Node: prop,
-						Path: fmt.Sprintf("$.%s.properties.%s", strings.Join(seenPath, "."), label),
+						Path: fmt.Sprintf("$.%s.%s.%s", strings.Join(seenPath, "."), n.Value, label),
+					}
+
+					isRef, _, _ := utils.IsNodeRefValue(prop)
+					if isRef {
+						// record this reference
+						index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref)
+						continue
+					}
+
+					index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref)
+
+					// check if the schema is an object or an array,
+					// and if so, add it to the list of inline schema object definitions.
+					k, v := utils.FindKeyNodeTop("type", prop.Content)
+					if k != nil && v != nil {
+						if v.Value == "object" || v.Value == "array" {
+							index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref)
+						}
+					}
+				}
+			}
+
+			// Perform the same check for all arrays of schemas like allOf, anyOf, oneOf
+			arrayOfSchemaContainingNodes := []string{"allOf", "anyOf", "oneOf", "prefixItems"}
+			if i%2 == 0 && slices.Contains(arrayOfSchemaContainingNodes, n.Value) && !utils.IsNodeArray(node) && (i+1 < len(node.Content)) {
+				// for each element in the array, add it to our schema definitions
+				for h, element := range node.Content[i+1].Content {
+					ref := &Reference{
+						Node: element,
+						Path: fmt.Sprintf("$.%s.%s[%d]", strings.Join(seenPath, "."), n.Value, h),
+					}
+
+					isRef, _, _ := utils.IsNodeRefValue(element)
+					if isRef { // record this reference
+						index.allRefSchemaDefinitions = append(index.allRefSchemaDefinitions, ref)
+						continue
 					}
 					index.allInlineSchemaDefinitions = append(index.allInlineSchemaDefinitions, ref)
 
 					// check if the schema is an object or an array,
 					// and if so, add it to the list of inline schema object definitions.
-					k, v := utils.FindKeyNodeTop("type", node.Content[i+1].Content)
+					k, v := utils.FindKeyNodeTop("type", element.Content)
 					if k != nil && v != nil {
 						if v.Value == "object" || v.Value == "array" {
 							index.allInlineSchemaObjectDefinitions = append(index.allInlineSchemaObjectDefinitions, ref)
@@ -342,7 +380,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 			if i < len(node.Content)-1 {
 				next := node.Content[i+1]
 
-				if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) {
+				if i%2 != 0 && next != nil && !utils.IsNodeArray(next) && !utils.IsNodeMap(next) && len(seenPath) > 0 {
 					seenPath = seenPath[:len(seenPath)-1]
 				}
 			}
@@ -366,7 +404,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Reference {
 	var found []*Reference
 
-	//run this async because when things get recursive, it can take a while
+	// run this async because when things get recursive, it can take a while
 	c := make(chan bool)
 
 	locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) {
@@ -419,7 +457,7 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
 	for r := range refsToCheck {
 		// expand our index of all mapped refs
 		go locate(refsToCheck[r], r, mappedRefsInSequence)
-		//locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
+		// locate(refsToCheck[r], r, mappedRefsInSequence) // used for sync testing.
 	}
 
 	completedRefs := 0

index/extract_refs_test.go

@@ -4,13 +4,13 @@
 package index
 
 import (
+	"testing"
+
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
-	"testing"
 )
 
 func TestSpecIndex_ExtractRefs_CheckDescriptionNotMap(t *testing.T) {
-
 	yml := `openapi: 3.1.0
 info:
   description: This is a description
@@ -37,7 +37,6 @@ paths:
 }
 
 func TestSpecIndex_ExtractRefs_CheckPropertiesForInlineSchema(t *testing.T) {
-
 	yml := `openapi: 3.1.0
 servers:
   - url: http://localhost:8080
@@ -54,14 +53,63 @@ paths:
                 properties:
                   test:
                     type: array
-                    items: true
+                    items:
+                      type: object
+                    prefixItems:
+                      - $ref: '#/components/schemas/Test'
+                    additionalProperties: false
+                    unevaluatedProperties: false
+components:
+  schemas:
+    Test:
+      type: object
+      additionalProperties:
+        type: string
+      contains:
+        type: string
+      not:
+        type: number
+      unevaluatedProperties:
+        type: boolean
+      patternProperties:
+        ^S_:
+          type: string
+        ^I_:
+          type: integer
+      prefixItems:
+        - type: string
+    AllOf:
+      allOf:
+        - type: object
+          properties:
+            test:
+              type: string
+        - type: object
+          properties:
+            test2:
+              type: string
+    AnyOf:
+      anyOf:
+        - type: object
+          properties:
+            test:
+              type: string
+        - type: object
+          properties:
+            test2:
+              type: string
+    OneOf:
+      oneOf:
+        - type: string
+        - type: number
+        - type: boolean
    `
 	var rootNode yaml.Node
 	_ = yaml.Unmarshal([]byte(yml), &rootNode)
 	c := CreateOpenAPIIndexConfig()
 	idx := NewSpecIndexWithConfig(&rootNode, c)
-	assert.Len(t, idx.allInlineSchemaDefinitions, 2)
+	assert.Len(t, idx.allInlineSchemaDefinitions, 21)
-	assert.Len(t, idx.allInlineSchemaObjectDefinitions, 1)
+	assert.Len(t, idx.allInlineSchemaObjectDefinitions, 7)
 }
 
 // https://github.com/pb33f/libopenapi/issues/112

index/spec_index_test.go

@@ -44,7 +44,7 @@ func TestSpecIndex_ExtractRefsStripe(t *testing.T) {
 	assert.Len(t, index.GetPolyAllOfReferences(), 0)
 	assert.Len(t, index.GetPolyOneOfReferences(), 275)
 	assert.Len(t, index.GetPolyAnyOfReferences(), 553)
-	assert.Len(t, index.GetAllReferenceSchemas(), 696)
+	assert.Len(t, index.GetAllReferenceSchemas(), 1972)
 	assert.NotNil(t, index.GetRootServersNode())
 	assert.Len(t, index.GetAllRootServers(), 1)
 
@@ -283,7 +283,7 @@ func TestSpecIndex_BurgerShop(t *testing.T) {
 	assert.Equal(t, 6, index.GetPathCount())
 
 	assert.Equal(t, 6, len(index.GetAllComponentSchemas()))
-	assert.Equal(t, 47, len(index.GetAllSchemas()))
+	assert.Equal(t, 56, len(index.GetAllSchemas()))
 
 	assert.Equal(t, 34, len(index.GetAllSequencedReferences()))
 	assert.NotNil(t, index.GetSchemasNode())
@@ -1096,10 +1096,10 @@ func ExampleNewSpecIndex() {
 	// 246 paths
 	// 402 operations
 	// 537 component schemas
-	// 696 reference schemas
+	// 1972 reference schemas
-	// 9798 inline schemas
+	// 11749 inline schemas
-	// 711 inline schemas that are objects or arrays
+	// 2612 inline schemas that are objects or arrays
-	// 11031 total schemas
+	// 14258 total schemas
 	// 1516 enums
 	// 828 polymorphic references
 }