Added some new getters to the index

Also added map conversion utilities based on reported vacuum error https://github.com/daveshanley/vacuum/issues/417
also prevented the bundler from inlining root references.

Signed-off-by: quobix <dave@quobix.com>

bundler/bundler.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pb33f/libopenapi/datamodel"
 	"github.com/pb33f/libopenapi/datamodel/high/v3"
 	"github.com/pb33f/libopenapi/index"
+	"strings"
 )
 
 // BundleBytes will take a byte slice of an OpenAPI specification and return a bundled version of it.
@@ -29,7 +30,7 @@ func BundleBytes(bytes []byte, configuration *datamodel.DocumentConfiguration) (
 	v3Doc, errs := doc.BuildV3Model()
 	err = errors.Join(errs...)
 
-	bundledBytes, e := BundleDocument(&v3Doc.Model)
+	bundledBytes, e := bundle(&v3Doc.Model, configuration.BundleInlineRefs)
 	return bundledBytes, errors.Join(err, e)
 }
 
@@ -43,14 +44,32 @@ func BundleBytes(bytes []byte, configuration *datamodel.DocumentConfiguration) (
 //
 // Circular references will not be resolved and will be skipped.
 func BundleDocument(model *v3.Document) ([]byte, error) {
+	return bundle(model, false)
+}
+
+func bundle(model *v3.Document, inline bool) ([]byte, error) {
 	rolodex := model.Rolodex
-	compress := func(idx *index.SpecIndex) {
+	compact := func(idx *index.SpecIndex, root bool) {
 		mappedReferences := idx.GetMappedReferences()
 		sequencedReferences := idx.GetRawReferencesSequenced()
 		for _, sequenced := range sequencedReferences {
 			mappedReference := mappedReferences[sequenced.FullDefinition]
+
+			//if we're in the root document, don't bundle anything.
+			refExp := strings.Split(sequenced.FullDefinition, "#/")
+			if len(refExp) == 2 {
+				if refExp[0] == "" {
+					if root && !inline {
+						idx.GetLogger().Debug("[bundler] skipping local root reference",
+							"ref", sequenced.Definition)
+						continue
+					}
+				}
+			}
+
 			if mappedReference != nil && !mappedReference.Circular {
 				sequenced.Node.Content = mappedReference.Node.Content
+				continue
 			}
 			if mappedReference != nil && mappedReference.Circular {
 				if idx.GetLogger() != nil {
@@ -62,9 +81,9 @@ func BundleDocument(model *v3.Document) ([]byte, error) {
 	}
 
 	indexes := rolodex.GetIndexes()
-	compress(rolodex.GetRootIndex())
 	for _, idx := range indexes {
-		compress(idx)
+		compact(idx, false)
 	}
+	compact(rolodex.GetRootIndex(), true)
 	return model.Render()
 }

bundler/bundler_test.go

@@ -5,7 +5,6 @@ package bundler
 
 import (
 	"bytes"
-	"encoding/json"
 	"github.com/pb33f/libopenapi"
 	"github.com/pb33f/libopenapi/datamodel"
 	"github.com/stretchr/testify/assert"
@@ -81,19 +80,15 @@ func TestBundleDocument_Circular(t *testing.T) {
 
 	bytes, e := BundleDocument(&v3Doc.Model)
 	assert.NoError(t, e)
-	assert.Len(t, bytes, 3069)
+	assert.Len(t, *doc.GetSpecInfo().SpecBytes, 1563)
+	assert.Len(t, bytes, 2016)
 
 	logEntries := strings.Split(byteBuf.String(), "\n")
-
+	if len(logEntries) == 1 && logEntries[0] == "" {
-	assert.Len(t, logEntries, 5)
+		logEntries = []string{}
-	for _, entry := range logEntries {
-		items := make(map[string]any)
-		if entry != "" {
-			_ = json.Unmarshal([]byte(entry), &items)
-			assert.Equal(t, "[bundler] skipping circular reference", items["msg"])
-		}
 	}
-	assert.NoError(t, e)
+
+	assert.Len(t, logEntries, 0)
 }
 
 func TestBundleBytes(t *testing.T) {
@@ -112,18 +107,102 @@ func TestBundleBytes(t *testing.T) {
 
 	bytes, e := BundleBytes(digi, config)
 	assert.Error(t, e)
-	assert.Len(t, bytes, 3069)
+	assert.Len(t, bytes, 2016)
 
 	logEntries := strings.Split(byteBuf.String(), "\n")
-
+	if len(logEntries) == 1 && logEntries[0] == "" {
-	assert.Len(t, logEntries, 5)
+		logEntries = []string{}
-	for _, entry := range logEntries {
-		items := make(map[string]any)
-		if entry != "" {
-			_ = json.Unmarshal([]byte(entry), &items)
-			assert.Equal(t, "[bundler] skipping circular reference", items["msg"])
-		}
 	}
+
+	assert.Len(t, logEntries, 0)
+}
+
+func TestBundleBytes_CircularArray(t *testing.T) {
+
+	digi := []byte(`openapi: 3.1.0
+info:
+  title: FailureCases
+  version: 0.1.0
+servers:
+  - url: http://localhost:35123
+    description: The default server.
+paths:
+  /test:
+    get:
+      responses:
+        '200':
+          description: OK
+components:
+  schemas:
+    Obj:
+      type: object
+      properties:
+        children:
+          type: array
+          items:
+            $ref: '#/components/schemas/Obj'
+      required:
+        - children`)
+
+	var logs []byte
+	byteBuf := bytes.NewBuffer(logs)
+
+	config := &datamodel.DocumentConfiguration{
+		ExtractRefsSequentially:       true,
+		IgnoreArrayCircularReferences: true,
+		Logger: slog.New(slog.NewJSONHandler(byteBuf, &slog.HandlerOptions{
+			Level: slog.LevelDebug,
+		})),
+	}
+
+	bytes, e := BundleBytes(digi, config)
+	assert.NoError(t, e)
+	assert.Len(t, bytes, 537)
+
+	logEntries := strings.Split(byteBuf.String(), "\n")
+	assert.Len(t, logEntries, 10)
+}
+
+func TestBundleBytes_CircularFile(t *testing.T) {
+
+	digi := []byte(`openapi: 3.1.0
+info:
+  title: FailureCases
+  version: 0.1.0
+servers:
+  - url: http://localhost:35123
+    description: The default server.
+paths:
+  /test:
+    get:
+      responses:
+        '200':
+          description: OK
+components:
+  schemas:
+    Obj:
+      type: object
+      properties:
+        children:
+          $ref: '../test_specs/circular-tests.yaml#/components/schemas/One'`)
+
+	var logs []byte
+	byteBuf := bytes.NewBuffer(logs)
+
+	config := &datamodel.DocumentConfiguration{
+		BasePath:                ".",
+		ExtractRefsSequentially: true,
+		Logger: slog.New(slog.NewJSONHandler(byteBuf, &slog.HandlerOptions{
+			Level: slog.LevelDebug,
+		})),
+	}
+
+	bytes, e := BundleBytes(digi, config)
+	assert.Error(t, e)
+	assert.Len(t, bytes, 458)
+
+	logEntries := strings.Split(byteBuf.String(), "\n")
+	assert.Len(t, logEntries, 13)
 }
 
 func TestBundleBytes_Bad(t *testing.T) {

datamodel/document_config.go

@@ -107,6 +107,10 @@ type DocumentConfiguration struct {
 	// This is a more thorough way of building the index, but it's slower. It's required building a document
 	// to be bundled.
 	ExtractRefsSequentially bool
+
+	// BundleInlineRefs is used by the bundler module. If set to true, all references will be inlined, including
+	// local references (to the root document) as well as all external references. This is false by default.
+	BundleInlineRefs bool
 }
 
 func NewDocumentConfiguration() *DocumentConfiguration {

datamodel/translate_test.go

@@ -1,6 +1,7 @@
 package datamodel_test
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -9,6 +10,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/pb33f/libopenapi/datamodel"
 	"github.com/pb33f/libopenapi/orderedmap"
@@ -486,41 +488,59 @@ func TestTranslatePipeline(t *testing.T) {
 			// context cancel.  Then the second item is aborted by this error
 			// handler.
 			t.Run("Error while waiting on worker", func(t *testing.T) {
-				const concurrency = 2
-				in := make(chan int)
-				out := make(chan string)
-				done := make(chan struct{})
-				var wg sync.WaitGroup
-				wg.Add(1) // input goroutine
 
-				// Send input.
+				// this test gets stuck sometimes, so it needs a hard limit.
-				go func() {
+
-					// Fill up worker pool with items.
+				ctx, c := context.WithTimeout(context.Background(), 5*time.Second)
-					for i := 0; i < concurrency; i++ {
+				defer c()
-						select {
+				doneChan := make(chan bool)
-						case in <- i:
+
-						case <-done:
+				go func(completedChan chan bool) {
+
+					const concurrency = 2
+					in := make(chan int)
+					out := make(chan string)
+					done := make(chan struct{})
+					var wg sync.WaitGroup
+					wg.Add(1) // input goroutine
+
+					// Send input.
+					go func() {
+						// Fill up worker pool with items.
+						for i := 0; i < concurrency; i++ {
+							select {
+							case in <- i:
+							case <-done:
+							}
 						}
-					}
+						wg.Done()
-					wg.Done()
+					}()
-				}()
 
-				// No need to capture output channel.
+					// No need to capture output channel.
 
-				var itemCount atomic.Int64
+					var itemCount atomic.Int64
-				err := datamodel.TranslatePipeline[int, string](in, out,
+					err := datamodel.TranslatePipeline[int, string](in, out,
-					func(value int) (string, error) {
+						func(value int) (string, error) {
-						counter := itemCount.Add(1)
+							counter := itemCount.Add(1)
-						// Cause error on first call.
+							// Cause error on first call.
-						if counter == 1 {
+							if counter == 1 {
-							return "", errors.New("Foobar")
+								return "", errors.New("Foobar")
-						}
+							}
-						return "", nil
+							return "", nil
-					},
+						},
-				)
+					)
-				close(done)
+					close(done)
-				wg.Wait()
+					wg.Wait()
-				require.Error(t, err)
+					require.Error(t, err)
+					doneChan <- true
+				}(doneChan)
+
+				select {
+				case <-ctx.Done():
+					t.Log("error waiting on worker test timed out")
+				case <-doneChan:
+					// test passed
+				}
 			})
 		})
 	}

index/extract_refs.go

@@ -401,7 +401,9 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
 			if i%2 == 0 && n.Value != "$ref" && n.Value != "" {
 
-				nodePath := fmt.Sprintf("$.%s", strings.Join(seenPath, "."))
+				loc := append(seenPath, n.Value)
+				definitionPath := fmt.Sprintf("#/%s", strings.Join(loc, "/"))
+				_, jsonPath := utils.ConvertComponentIdIntoFriendlyPathSearch(definitionPath)
 
 				// capture descriptions and summaries
 				if n.Value == "description" {
@@ -413,7 +415,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
 					ref := &DescriptionReference{
 						Content:   node.Content[i+1].Value,
-						Path:      nodePath,
+						Path:      jsonPath,
 						Node:      node.Content[i+1],
 						IsSummary: false,
 					}
@@ -434,7 +436,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 					}
 					ref := &DescriptionReference{
 						Content:   b.Value,
-						Path:      nodePath,
+						Path:      jsonPath,
 						Node:      b,
 						IsSummary: true,
 					}
@@ -477,7 +479,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
 											refs = append(refs, &Reference{
 												Definition: b.Content[k].Content[g].Content[r].Value,
-												Path:       fmt.Sprintf("%s.security[%d].%s[%d]", nodePath, k, secKey, r),
+												Path:       fmt.Sprintf("%s.security[%d].%s[%d]", jsonPath, k, secKey, r),
 												Node:       b.Content[k].Content[g].Content[r],
 											})
 
@@ -506,7 +508,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
 					if enumKeyValueNode != nil {
 						ref := &EnumReference{
-							Path:       nodePath,
+							Path:       jsonPath,
 							Node:       node.Content[i+1],
 							Type:       enumKeyValueNode,
 							SchemaNode: node,
@@ -536,7 +538,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 
 						if isObject {
 							index.allObjectsWithProperties = append(index.allObjectsWithProperties, &ObjectReference{
-								Path:       nodePath,
+								Path:       jsonPath,
 								Node:       node,
 								ParentNode: parent,
 							})
@@ -544,8 +546,8 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
 					}
 				}
 
-				//seenPath = append(seenPath, strings.ReplaceAll(n.Value, "/", "~1"))
+				seenPath = append(seenPath, strings.ReplaceAll(n.Value, "/", "~1"))
-				seenPath = append(seenPath, n.Value)
+				//seenPath = append(seenPath, n.Value)
 				prev = n.Value
 			}
 

index/index_model.go

@@ -16,13 +16,6 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-// Constants used to determine if resolving is local, file based or remote file based.
-const (
-	LocalResolve = iota
-	HttpResolve
-	FileResolve
-)
-
 // Reference is a wrapper around *yaml.Node results to make things more manageable when performing
 // algorithms on data models. the *yaml.Node def is just a bit too low level for tracking state.
 type Reference struct {
@@ -272,6 +265,8 @@ type SpecIndex struct {
 	componentLock                       sync.RWMutex
 	errorLock                           sync.RWMutex
 	circularReferences                  []*CircularReferenceResult // only available when the resolver has been used.
+	polyCircularReferences              []*CircularReferenceResult // only available when the resolver has been used.
+	arrayCircularReferences             []*CircularReferenceResult // only available when the resolver has been used.
 	allowCircularReferences             bool                       // decide if you want to error out, or allow circular references, default is false.
 	config                              *SpecIndexConfig           // configuration for the index
 	componentIndexChan                  chan bool
@@ -300,6 +295,10 @@ func (index *SpecIndex) SetCache(sync *syncmap.Map) {
 	index.cache = sync
 }
 
+func (index *SpecIndex) GetNodeMap() map[int]map[int]*yaml.Node {
+	return index.nodeMap
+}
+
 func (index *SpecIndex) GetCache() *syncmap.Map {
 	return index.cache
 }

index/resolver.go

@@ -229,6 +229,8 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError {
 	}
 	// update our index with any circular refs we found.
 	resolver.specIndex.SetCircularReferences(resolver.circularReferences)
+	resolver.specIndex.SetIgnoredArrayCircularReferences(resolver.ignoredArrayReferences)
+	resolver.specIndex.SetIgnoredPolymorphicCircularReferences(resolver.ignoredPolyReferences)
 	resolver.circChecked = true
 	return resolver.resolvingErrors
 }

index/spec_index.go

@@ -170,6 +170,28 @@ func (index *SpecIndex) GetCircularReferences() []*CircularReferenceResult {
 	return index.circularReferences
 }
 
+// SetIgnoredPolymorphicCircularReferences passes on any ignored poly circular refs captured using
+// `IgnorePolymorphicCircularReferences`
+func (index *SpecIndex) SetIgnoredPolymorphicCircularReferences(refs []*CircularReferenceResult) {
+	index.polyCircularReferences = refs
+}
+
+func (index *SpecIndex) SetIgnoredArrayCircularReferences(refs []*CircularReferenceResult) {
+	index.arrayCircularReferences = refs
+}
+
+// GetIgnoredPolymorphicCircularReferences will return any polymorphic circular references that were 'ignored' by
+// using the `IgnorePolymorphicCircularReferences` configuration option.
+func (index *SpecIndex) GetIgnoredPolymorphicCircularReferences() []*CircularReferenceResult {
+	return index.polyCircularReferences
+}
+
+// GetIgnoredArrayCircularReferences will return any array based circular references that were 'ignored' by
+// using the `IgnoreArrayCircularReferences` configuration option.
+func (index *SpecIndex) GetIgnoredArrayCircularReferences() []*CircularReferenceResult {
+	return index.arrayCircularReferences
+}
+
 // GetPathsNode returns document root node.
 func (index *SpecIndex) GetPathsNode() *yaml.Node {
 	return index.pathsNode

index/spec_index_test.go

@@ -56,6 +56,11 @@ func TestSpecIndex_GetCache(t *testing.T) {
 	loaded, ok = extCache.Load("test2")
 	assert.Nil(t, loaded)
 	assert.False(t, ok)
+
+	assert.Len(t, index.GetIgnoredPolymorphicCircularReferences(), 0)
+	assert.Len(t, index.GetIgnoredArrayCircularReferences(), 0)
+	assert.Equal(t, len(index.GetRawReferencesSequenced()), 42)
+	assert.Equal(t, len(index.GetNodeMap()), 824)
 }
 
 func TestSpecIndex_ExtractRefsStripe(t *testing.T) {
@@ -1535,3 +1540,77 @@ paths:
 	assert.Equal(t, "$.paths['/test2'].put", paths["/test2"]["put"].Path)
 	assert.Equal(t, 22, paths["/test2"]["put"].ParentNode.Line)
 }
+
+func TestSpecIndex_TestInlineSchemaPaths(t *testing.T) {
+
+	yml := `openapi: 3.1.0
+info:
+  title: Test
+  version: 0.0.1
+servers:
+  - url: http://localhost:35123
+paths:
+  /test:
+    get:
+      operationId: TestSomething
+      parameters:
+        - name: test
+          in: query
+          description: test param for duplicate inline schema
+          required: false
+          schema:
+            type: object
+            required:
+              - code
+              - message
+            properties:
+              code:
+                type: integer
+                format: int32
+              message:
+                type: string
+      responses:
+        '200':
+          description: OK
+        '5XX':
+          description: test response for slightly different inline schema
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - code
+                  - messages
+                properties:
+                  code:
+                    type: integer
+                    format: int32
+                  messages:
+                    type: string
+        default:
+          description: test response for duplicate inline schema
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - code
+                  - message
+                properties:
+                  code:
+                    type: integer
+                    format: int32
+                  message:
+                    type: string`
+
+	var rootNode yaml.Node
+	_ = yaml.Unmarshal([]byte(yml), &rootNode)
+
+	idx := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig())
+
+	schemas := idx.GetAllInlineSchemas()
+	assert.Equal(t, "$.paths['/test'].get.parameters.schema", schemas[0].Path)
+	assert.Equal(t, "$.paths['/test'].get.parameters.schema.properties.code", schemas[1].Path)
+	assert.Equal(t, "$.paths['/test'].get.parameters.schema.properties.message", schemas[2].Path)
+
+}

utils/utils.go

@@ -139,6 +139,18 @@ func ConvertInterfaceIntoStringMap(context interface{}) map[string]string {
 				if s, okB := n.(string); okB {
 					converted[k] = s
 				}
+				if s, okB := n.(float64); okB {
+					converted[k] = fmt.Sprint(s)
+				}
+				if s, okB := n.(bool); okB {
+					converted[k] = fmt.Sprint(s)
+				}
+				if s, okB := n.(int); okB {
+					converted[k] = fmt.Sprint(s)
+				}
+				if s, okB := n.(int64); okB {
+					converted[k] = fmt.Sprint(s)
+				}
 			}
 		}
 		if v, ok := context.(map[string]string); ok {

utils/utils_test.go

@@ -132,6 +132,42 @@ func TestConvertInterfaceIntoStringMap(t *testing.T) {
 	assert.Equal(t, "baby girl", parsed["melody"])
 }
 
+func TestConvertInterfaceIntoStringMap_Float64(t *testing.T) {
+	var d interface{}
+	n := make(map[string]interface{})
+	n["melody"] = 5.9
+	d = n
+	parsed := ConvertInterfaceIntoStringMap(d)
+	assert.Equal(t, "5.9", parsed["melody"])
+}
+
+func TestConvertInterfaceIntoStringMap_Bool(t *testing.T) {
+	var d interface{}
+	n := make(map[string]interface{})
+	n["melody"] = true
+	d = n
+	parsed := ConvertInterfaceIntoStringMap(d)
+	assert.Equal(t, "true", parsed["melody"])
+}
+
+func TestConvertInterfaceIntoStringMap_int64(t *testing.T) {
+	var d interface{}
+	n := make(map[string]interface{})
+	n["melody"] = int64(12345)
+	d = n
+	parsed := ConvertInterfaceIntoStringMap(d)
+	assert.Equal(t, "12345", parsed["melody"])
+}
+
+func TestConvertInterfaceIntoStringMap_int(t *testing.T) {
+	var d interface{}
+	n := make(map[string]interface{})
+	n["melody"] = 12345
+	d = n
+	parsed := ConvertInterfaceIntoStringMap(d)
+	assert.Equal(t, "12345", parsed["melody"])
+}
+
 func TestConvertInterfaceIntoStringMap_NoType(t *testing.T) {
 	var d interface{}
 	n := make(map[string]interface{})