diff --git a/datamodel/document_config.go b/datamodel/document_config.go index 7b7d113..205dd47 100644 --- a/datamodel/document_config.go +++ b/datamodel/document_config.go @@ -52,6 +52,11 @@ type DocumentConfiguration struct { // So if libopenapi is returning circular references for this use case, then this option should be enabled. // this is disabled by default, which means array circular references will be checked. IgnoreArrayCircularReferences bool + + // SkipCircularReferenceCheck will skip over checking for circular references. This is disabled by default, which + // means circular references will be checked. This is useful for developers building out models that should be + // indexed later on. + SkipCircularReferenceCheck bool } func NewOpenDocumentConfiguration() *DocumentConfiguration { diff --git a/datamodel/low/base/schema_test.go b/datamodel/low/base/schema_test.go index 014b970..07fb966 100644 --- a/datamodel/low/base/schema_test.go +++ b/datamodel/low/base/schema_test.go @@ -1,14 +1,13 @@ package base import ( - "github.com/pb33f/libopenapi/datamodel" - "github.com/pb33f/libopenapi/datamodel/low" - "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" - "github.com/pb33f/libopenapi/utils" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" - "testing" + "github.com/pb33f/libopenapi/datamodel" + "github.com/pb33f/libopenapi/datamodel/low" + "github.com/pb33f/libopenapi/index" + "github.com/pb33f/libopenapi/utils" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "testing" ) func test_get_schema_blob() string { @@ -901,7 +900,7 @@ func Test_Schema_RefMadnessIllegal_Circular(t *testing.T) { var idxNode yaml.Node _ = yaml.Unmarshal([]byte(yml), &idxNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -933,7 +932,7 @@ func Test_Schema_RefMadnessIllegal_Nonexist(t *testing.T) { var idxNode yaml.Node _ = yaml.Unmarshal([]byte(yml), &idxNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -1074,7 +1073,7 @@ func TestExtractSchema_CheckChildPropCircular(t *testing.T) { yml = `$ref: '#/components/schemas/Something'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index e6654d6..732eb0b 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -57,14 +57,14 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) { // if there are any external indexes being used by remote // documents, then we need to search through them also. - externalIndexes := idx.GetAllExternalIndexes() - if len(externalIndexes) > 0 { - var extCollection []func() map[string]*index.Reference - for _, extIndex := range externalIndexes { - extCollection = generateIndexCollection(extIndex) - collections = append(collections, extCollection...) - } - } + //externalIndexes := idx.GetAllExternalIndexes() + //if len(externalIndexes) > 0 { + // var extCollection []func() map[string]*index.Reference + // for _, extIndex := range externalIndexes { + // extCollection = generateIndexCollection(extIndex) + // collections = append(collections, extCollection...) + // } + //} var found map[string]*index.Reference for _, collection := range collections { @@ -501,6 +501,7 @@ func ExtractMapExtensions[PT Buildable[N], N any]( } } else { _, labelNode, valueNode = utils.FindKeyNodeFull(label, root.Content) + valueNode = utils.NodeAlias(valueNode) if valueNode != nil { if h, _, rvt := utils.IsNodeRefValue(valueNode); h { ref, err := LocateRefNode(valueNode, idx) diff --git a/datamodel/low/extraction_functions_test.go b/datamodel/low/extraction_functions_test.go index 926773d..2b6670b 100644 --- a/datamodel/low/extraction_functions_test.go +++ b/datamodel/low/extraction_functions_test.go @@ -4,16 +4,15 @@ package low import ( - "crypto/sha256" - "fmt" - "os" - "strings" - "testing" + "crypto/sha256" + "fmt" + "os" + "strings" + "testing" - "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" + "github.com/pb33f/libopenapi/index" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestFindItemInMap(t *testing.T) { @@ -234,7 +233,7 @@ func TestExtractObject_DoubleRef_Circular(t *testing.T) { idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) // circular references are detected by the resolver, so lets run it! - resolv := resolver.NewResolver(idx) + resolv := index.NewResolver(idx) assert.Len(t, resolv.CheckForCircularReferences(), 1) yml = `tags: @@ -264,7 +263,7 @@ func TestExtractObject_DoubleRef_Circular_Fail(t *testing.T) { idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) // circular references are detected by the resolver, so lets run it! - resolv := resolver.NewResolver(idx) + resolv := index.NewResolver(idx) assert.Len(t, resolv.CheckForCircularReferences(), 1) yml = `tags: @@ -294,7 +293,7 @@ func TestExtractObject_DoubleRef_Circular_Direct(t *testing.T) { idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) // circular references are detected by the resolver, so lets run it! - resolv := resolver.NewResolver(idx) + resolv := index.NewResolver(idx) assert.Len(t, resolv.CheckForCircularReferences(), 1) yml = `$ref: '#/components/schemas/pizza'` @@ -324,7 +323,7 @@ func TestExtractObject_DoubleRef_Circular_Direct_Fail(t *testing.T) { idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) // circular references are detected by the resolver, so lets run it! - resolv := resolver.NewResolver(idx) + resolv := index.NewResolver(idx) assert.Len(t, resolv.CheckForCircularReferences(), 1) yml = `$ref: '#/components/schemas/why-did-westworld-have-to-end-so-poorly-ffs'` @@ -449,7 +448,7 @@ func TestExtractObject_PathIsCircular(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -485,7 +484,7 @@ func TestExtractObject_PathIsCircular_IgnoreErrors(t *testing.T) { // disable circular ref checking. idx.SetAllowCircularReferenceResolving(true) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -563,7 +562,7 @@ func TestExtractObjectRaw_Ref_Circular(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -735,7 +734,7 @@ func TestExtractArray_Ref_Circular(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -763,7 +762,7 @@ func TestExtractArray_Ref_Bad(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -791,7 +790,7 @@ func TestExtractArray_Ref_Nested(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -820,7 +819,7 @@ func TestExtractArray_Ref_Nested_Circular(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -876,7 +875,7 @@ func TestExtractArray_Ref_Nested_CircularFlat(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -1165,7 +1164,7 @@ func TestExtractMapFlatNoLookup_Ref_Circular(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -1386,7 +1385,7 @@ func TestExtractMapFlat_DoubleRef_Circles(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -1445,7 +1444,7 @@ func TestExtractMapFlat_Ref_Circ_Error(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -1474,7 +1473,7 @@ func TestExtractMapFlat_Ref_Nested_Circ_Error(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -1556,7 +1555,7 @@ func TestExtractMapFlat_Ref_Bad(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateClosedAPIIndexConfig()) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) diff --git a/datamodel/low/reference_test.go b/datamodel/low/reference_test.go index 2a33f2e..8ef9861 100644 --- a/datamodel/low/reference_test.go +++ b/datamodel/low/reference_test.go @@ -4,16 +4,15 @@ package low import ( - "crypto/sha256" - "fmt" - "github.com/pb33f/libopenapi/utils" - "strings" - "testing" + "crypto/sha256" + "fmt" + "github.com/pb33f/libopenapi/utils" + "strings" + "testing" - "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" + "github.com/pb33f/libopenapi/index" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestNodeReference_IsEmpty(t *testing.T) { @@ -124,7 +123,7 @@ func TestIsCircular_LookupFromJourney(t *testing.T) { yml = `$ref: '#/components/schemas/Something'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -157,7 +156,7 @@ func TestIsCircular_LookupFromJourney_Optional(t *testing.T) { yml = `$ref: '#/components/schemas/Something'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) @@ -193,7 +192,7 @@ func TestIsCircular_LookupFromLoopPoint(t *testing.T) { yml = `$ref: '#/components/schemas/Nothing'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -225,7 +224,7 @@ func TestIsCircular_LookupFromLoopPoint_Optional(t *testing.T) { yml = `$ref: '#/components/schemas/Nothing'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) @@ -262,7 +261,7 @@ func TestIsCircular_FromRefLookup(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndex(&iNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -298,7 +297,7 @@ func TestIsCircular_FromRefLookup_Optional(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndex(&iNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) @@ -346,7 +345,7 @@ func TestGetCircularReferenceResult_FromJourney(t *testing.T) { yml = `$ref: '#/components/schemas/Something'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -380,7 +379,7 @@ func TestGetCircularReferenceResult_FromJourney_Optional(t *testing.T) { yml = `$ref: '#/components/schemas/Something'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) @@ -418,7 +417,7 @@ func TestGetCircularReferenceResult_FromLoopPoint(t *testing.T) { yml = `$ref: '#/components/schemas/Nothing'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -452,7 +451,7 @@ func TestGetCircularReferenceResult_FromLoopPoint_Optional(t *testing.T) { yml = `$ref: '#/components/schemas/Nothing'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) @@ -490,7 +489,7 @@ func TestGetCircularReferenceResult_FromMappedRef(t *testing.T) { yml = `$ref: '#/components/schemas/Nothing'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -522,7 +521,7 @@ func TestGetCircularReferenceResult_FromMappedRef_Optional(t *testing.T) { yml = `$ref: '#/components/schemas/Nothing'` - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) @@ -545,7 +544,7 @@ func TestGetCircularReferenceResult_NothingFound(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndex(&iNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 0) diff --git a/datamodel/low/v2/swagger.go b/datamodel/low/v2/swagger.go index 872c8de..394977c 100644 --- a/datamodel/low/v2/swagger.go +++ b/datamodel/low/v2/swagger.go @@ -16,7 +16,6 @@ import ( "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/datamodel/low/base" "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" "gopkg.in/yaml.v3" ) @@ -165,7 +164,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur doc.ExternalDocs = extDocs // create resolver and check for circular references. - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) resolvingErrors := resolve.CheckForCircularReferences() if len(resolvingErrors) > 0 { diff --git a/datamodel/low/v3/create_document_test.go b/datamodel/low/v3/create_document_test.go index 636b23b..0f58a1c 100644 --- a/datamodel/low/v3/create_document_test.go +++ b/datamodel/low/v3/create_document_test.go @@ -2,6 +2,7 @@ package v3 import ( "fmt" + "github.com/pb33f/libopenapi/utils" "os" "testing" @@ -17,7 +18,7 @@ func initTest() { } data, _ := os.ReadFile("../../../test_specs/burgershop.openapi.yaml") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error // deprecated function test. doc, err = CreateDocument(info) if err != nil { @@ -29,10 +30,7 @@ func BenchmarkCreateDocument(b *testing.B) { data, _ := os.ReadFile("../../../test_specs/burgershop.openapi.yaml") info, _ := datamodel.ExtractSpecInfo(data) for i := 0; i < b.N; i++ { - doc, _ = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ - AllowFileReferences: false, - AllowRemoteReferences: false, - }) + doc, _ = CreateDocumentFromConfig(info, datamodel.NewClosedDocumentConfiguration()) } } @@ -40,28 +38,9 @@ func BenchmarkCreateDocument_Circular(b *testing.B) { data, _ := os.ReadFile("../../../test_specs/circular-tests.yaml") info, _ := datamodel.ExtractSpecInfo(data) for i := 0; i < b.N; i++ { - _, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ - AllowFileReferences: false, - AllowRemoteReferences: false, - }) - if err != nil { - panic("this should not error") - } - } -} - -func BenchmarkCreateDocument_k8s(b *testing.B) { - data, _ := os.ReadFile("../../../test_specs/k8s.json") - info, _ := datamodel.ExtractSpecInfo(data) - - for i := 0; i < b.N; i++ { - - _, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ - AllowFileReferences: false, - AllowRemoteReferences: false, - }) - if err != nil { - panic("this should not error") + _, err := CreateDocumentFromConfig(info, datamodel.NewClosedDocumentConfiguration()) + if err == nil { + panic("this should error, it has circular references") } } } @@ -69,12 +48,12 @@ func BenchmarkCreateDocument_k8s(b *testing.B) { func TestCircularReferenceError(t *testing.T) { data, _ := os.ReadFile("../../../test_specs/circular-tests.yaml") info, _ := datamodel.ExtractSpecInfo(data) - circDoc, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ - AllowFileReferences: false, - AllowRemoteReferences: false, - }) + circDoc, err := CreateDocumentFromConfig(info, datamodel.NewClosedDocumentConfiguration()) + assert.NotNil(t, circDoc) - assert.Len(t, err, 3) + assert.Error(t, err) + + assert.Len(t, utils.UnwrapErrors(err), 3) } func TestCircularReference_IgnoreArray(t *testing.T) { @@ -102,7 +81,7 @@ components: IgnoreArrayCircularReferences: true, }) assert.NotNil(t, circDoc) - assert.Len(t, err, 0) + assert.Len(t, utils.UnwrapErrors(err), 0) } func TestCircularReference_IgnorePoly(t *testing.T) { @@ -130,7 +109,7 @@ components: IgnorePolymorphicCircularReferences: true, }) assert.NotNil(t, circDoc) - assert.Len(t, err, 0) + assert.Len(t, utils.UnwrapErrors(err), 0) } func BenchmarkCreateDocument_Stripe(b *testing.B) { @@ -231,7 +210,7 @@ func TestCreateDocument_WebHooks_Error(t *testing.T) { $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, @@ -610,12 +589,12 @@ components: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 0) + assert.NoError(t, err) ob := doc.Components.Value.FindSchema("bork").Value ob.Schema() @@ -629,12 +608,13 @@ webhooks: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error doc, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Equal(t, "flat map build failed: reference cannot be found: reference '' at line 4, column 5 was not found", + err.Error()) } func TestCreateDocument_Components_Error_Extract(t *testing.T) { @@ -645,12 +625,12 @@ components: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Equal(t, "reference '' at line 5, column 7 was not found", err.Error()) } func TestCreateDocument_Paths_Errors(t *testing.T) { @@ -660,12 +640,13 @@ paths: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Equal(t, + "path item build failed: cannot find reference: at line 4, col 10", err.Error()) } func TestCreateDocument_Tags_Errors(t *testing.T) { @@ -674,12 +655,13 @@ tags: - $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Equal(t, + "object extraction failed: reference '' at line 3, column 5 was not found", err.Error()) } func TestCreateDocument_Security_Error(t *testing.T) { @@ -688,12 +670,14 @@ security: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Equal(t, + "array build failed: reference cannot be found: reference '' at line 3, column 3 was not found", + err.Error()) } func TestCreateDocument_ExternalDoc_Error(t *testing.T) { @@ -702,12 +686,12 @@ externalDocs: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error _, err = CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Equal(t, "object extraction failed: reference '' at line 3, column 3 was not found", err.Error()) } func TestCreateDocument_YamlAnchor(t *testing.T) { @@ -718,16 +702,13 @@ func TestCreateDocument_YamlAnchor(t *testing.T) { info, _ := datamodel.ExtractSpecInfo(anchorDocument) // build low-level document model - document, errors := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ + document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - // if something went wrong, a slice of errors is returned - if len(errors) > 0 { - for i := range errors { - fmt.Printf("error: %s\n", errors[i].Error()) - } + if err != nil { + fmt.Printf("error: %s\n", err.Error()) panic("cannot build document") } @@ -759,8 +740,9 @@ func TestCreateDocument_YamlAnchor(t *testing.T) { assert.NotNil(t, jsonGet) // Should this work? It doesn't - // postJsonType := examplePath.GetValue().Post.GetValue().RequestBody.GetValue().FindContent("application/json") - // assert.NotNil(t, postJsonType) + // update from quobix 10/14/2023: It does now! + postJsonType := examplePath.GetValue().Post.GetValue().RequestBody.GetValue().FindContent("application/json") + assert.NotNil(t, postJsonType) } func ExampleCreateDocument() { @@ -773,16 +755,13 @@ func ExampleCreateDocument() { info, _ := datamodel.ExtractSpecInfo(petstoreBytes) // build low-level document model - document, errors := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ + document, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, }) - // if something went wrong, a slice of errors is returned - if len(errors) > 0 { - for i := range errors { - fmt.Printf("error: %s\n", errors[i].Error()) - } + if err != nil { + fmt.Printf("error: %s\n", err.Error()) panic("cannot build document") } diff --git a/datamodel/low/v3/examples_test.go b/datamodel/low/v3/examples_test.go index 7894976..0686008 100644 --- a/datamodel/low/v3/examples_test.go +++ b/datamodel/low/v3/examples_test.go @@ -21,13 +21,11 @@ func Example_createLowLevelOpenAPIDocument() { info, _ := datamodel.ExtractSpecInfo(petstoreBytes) // build low-level document model - document, errors := CreateDocument(info) + document, errs := CreateDocument(info) // if something went wrong, a slice of errors is returned - if len(errors) > 0 { - for i := range errors { - fmt.Printf("error: %s\n", errors[i].Error()) - } + if errs != nil { + fmt.Printf("error: %s\n", errs.Error()) panic("cannot build document") } diff --git a/datamodel/low/v3/paths_test.go b/datamodel/low/v3/paths_test.go index ce4ee6e..9ab63e6 100644 --- a/datamodel/low/v3/paths_test.go +++ b/datamodel/low/v3/paths_test.go @@ -9,7 +9,6 @@ import ( "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" ) @@ -358,7 +357,7 @@ func TestPath_Build_Using_CircularRef(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndex(&idxNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) @@ -394,7 +393,7 @@ func TestPath_Build_Using_CircularRefWithOp(t *testing.T) { assert.NoError(t, mErr) idx := index.NewSpecIndex(&idxNode) - resolve := resolver.NewResolver(idx) + resolve := index.NewResolver(idx) errs := resolve.CheckForCircularReferences() assert.Len(t, errs, 1) diff --git a/datamodel/spec_info.go b/datamodel/spec_info.go index e185d49..cdbfb11 100644 --- a/datamodel/spec_info.go +++ b/datamodel/spec_info.go @@ -55,21 +55,21 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro var parsedSpec yaml.Node - specVersion := &SpecInfo{} - specVersion.JsonParsingChannel = make(chan bool) + specInfo := &SpecInfo{} + specInfo.JsonParsingChannel = make(chan bool) // set original bytes - specVersion.SpecBytes = &spec + specInfo.SpecBytes = &spec runes := []rune(strings.TrimSpace(string(spec))) if len(runes) <= 0 { - return specVersion, errors.New("there is nothing in the spec, it's empty - so there is nothing to be done") + return specInfo, errors.New("there is nothing in the spec, it's empty - so there is nothing to be done") } if runes[0] == '{' && runes[len(runes)-1] == '}' { - specVersion.SpecFileType = JSONFileType + specInfo.SpecFileType = JSONFileType } else { - specVersion.SpecFileType = YAMLFileType + specInfo.SpecFileType = YAMLFileType } err := yaml.Unmarshal(spec, &parsedSpec) @@ -77,7 +77,7 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro return nil, fmt.Errorf("unable to parse specification: %s", err.Error()) } - specVersion.RootNode = &parsedSpec + specInfo.RootNode = &parsedSpec _, openAPI3 := utils.FindKeyNode(utils.OpenApi3, parsedSpec.Content) _, openAPI2 := utils.FindKeyNode(utils.OpenApi2, parsedSpec.Content) @@ -122,17 +122,17 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro return nil, versionError } - specVersion.SpecType = utils.OpenApi3 - specVersion.Version = version - specVersion.SpecFormat = OAS3 + specInfo.SpecType = utils.OpenApi3 + specInfo.Version = version + specInfo.SpecFormat = OAS3 // parse JSON - parseJSON(spec, specVersion, &parsedSpec) + parseJSON(spec, specInfo, &parsedSpec) // double check for the right version, people mix this up. if majorVersion < 3 { - specVersion.Error = errors.New("spec is defined as an openapi spec, but is using a swagger (2.0), or unknown version") - return specVersion, specVersion.Error + specInfo.Error = errors.New("spec is defined as an openapi spec, but is using a swagger (2.0), or unknown version") + return specInfo, specInfo.Error } } @@ -142,17 +142,17 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro return nil, versionError } - specVersion.SpecType = utils.OpenApi2 - specVersion.Version = version - specVersion.SpecFormat = OAS2 + specInfo.SpecType = utils.OpenApi2 + specInfo.Version = version + specInfo.SpecFormat = OAS2 // parse JSON - parseJSON(spec, specVersion, &parsedSpec) + parseJSON(spec, specInfo, &parsedSpec) // I am not certain this edge-case is very frequent, but let's make sure we handle it anyway. if majorVersion > 2 { - specVersion.Error = errors.New("spec is defined as a swagger (openapi 2.0) spec, but is an openapi 3 or unknown version") - return specVersion, specVersion.Error + specInfo.Error = errors.New("spec is defined as a swagger (openapi 2.0) spec, but is an openapi 3 or unknown version") + return specInfo, specInfo.Error } } if asyncAPI != nil { @@ -161,45 +161,45 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro return nil, versionErr } - specVersion.SpecType = utils.AsyncApi - specVersion.Version = version + specInfo.SpecType = utils.AsyncApi + specInfo.Version = version // TODO: format for AsyncAPI. // parse JSON - parseJSON(spec, specVersion, &parsedSpec) + parseJSON(spec, specInfo, &parsedSpec) // so far there is only 2 as a major release of AsyncAPI if majorVersion > 2 { - specVersion.Error = errors.New("spec is defined as asyncapi, but has a major version that is invalid") - return specVersion, specVersion.Error + specInfo.Error = errors.New("spec is defined as asyncapi, but has a major version that is invalid") + return specInfo, specInfo.Error } } - if specVersion.SpecType == "" { + if specInfo.SpecType == "" { // parse JSON - parseJSON(spec, specVersion, &parsedSpec) - specVersion.Error = errors.New("spec type not supported by libopenapi, sorry") - return specVersion, specVersion.Error + parseJSON(spec, specInfo, &parsedSpec) + specInfo.Error = errors.New("spec type not supported by libopenapi, sorry") + return specInfo, specInfo.Error } } else { var jsonSpec map[string]interface{} if utils.IsYAML(string(spec)) { _ = parsedSpec.Decode(&jsonSpec) b, _ := json.Marshal(&jsonSpec) - specVersion.SpecJSONBytes = &b - specVersion.SpecJSON = &jsonSpec + specInfo.SpecJSONBytes = &b + specInfo.SpecJSON = &jsonSpec } else { _ = json.Unmarshal(spec, &jsonSpec) - specVersion.SpecJSONBytes = &spec - specVersion.SpecJSON = &jsonSpec + specInfo.SpecJSONBytes = &spec + specInfo.SpecJSON = &jsonSpec } - close(specVersion.JsonParsingChannel) // this needs removing at some point + close(specInfo.JsonParsingChannel) // this needs removing at some point } // detect the original whitespace indentation - specVersion.OriginalIndentation = utils.DetermineWhitespaceLength(string(spec)) + specInfo.OriginalIndentation = utils.DetermineWhitespaceLength(string(spec)) - return specVersion, nil + return specInfo, nil } diff --git a/document.go b/document.go index 0783fee..2211d62 100644 --- a/document.go +++ b/document.go @@ -24,7 +24,6 @@ import ( v3high "github.com/pb33f/libopenapi/datamodel/high/v3" v2low "github.com/pb33f/libopenapi/datamodel/low/v2" v3low "github.com/pb33f/libopenapi/datamodel/low/v3" - "github.com/pb33f/libopenapi/resolver" "github.com/pb33f/libopenapi/utils" what_changed "github.com/pb33f/libopenapi/what-changed" "github.com/pb33f/libopenapi/what-changed/model" @@ -44,6 +43,10 @@ type Document interface { // allowing remote or local references, as well as a BaseURL to allow for relative file references. SetConfiguration(configuration *datamodel.DocumentConfiguration) + // GetConfiguration will return the configuration for the document. This allows for finer grained control over + // allowing remote or local references, as well as a BaseURL to allow for relative file references. + GetConfiguration() *datamodel.DocumentConfiguration + // BuildV2Model will build out a Swagger (version 2) model from the specification used to create the document // If there are any issues, then no model will be returned, instead a slice of errors will explain all the // problems that occurred. This method will only support version 2 specifications and will throw an error for @@ -166,6 +169,10 @@ func (d *document) GetSpecInfo() *datamodel.SpecInfo { return d.info } +func (d *document) GetConfiguration() *datamodel.DocumentConfiguration { + return d.config +} + func (d *document) SetConfiguration(configuration *datamodel.DocumentConfiguration) { d.config = configuration } @@ -254,7 +261,7 @@ func (d *document) BuildV2Model() (*DocumentModel[v2high.Swagger], []error) { // Do not short-circuit on circular reference errors, so the client // has the option of ignoring them. for _, err := range errors { - if refErr, ok := err.(*resolver.ResolvingError); ok { + if refErr, ok := err.(*index.ResolvingError); ok { if refErr.CircularReference == nil { return nil, errors } @@ -297,7 +304,7 @@ func (d *document) BuildV3Model() (*DocumentModel[v3high.Document], []error) { // Do not short-circuit on circular reference errors, so the client // has the option of ignoring them. for _, err := range errors { - if refErr, ok := err.(*resolver.ResolvingError); ok { + if refErr, ok := err.(*index.ResolvingError); ok { if refErr.CircularReference == nil { return nil, errors } diff --git a/document_examples_test.go b/document_examples_test.go index 04131ff..c6182bc 100644 --- a/document_examples_test.go +++ b/document_examples_test.go @@ -6,6 +6,7 @@ package libopenapi import ( "fmt" "github.com/pb33f/libopenapi/datamodel" + "github.com/pb33f/libopenapi/index" "net/url" "os" "strings" @@ -15,7 +16,6 @@ import ( v3high "github.com/pb33f/libopenapi/datamodel/high/v3" low "github.com/pb33f/libopenapi/datamodel/low/base" v3 "github.com/pb33f/libopenapi/datamodel/low/v3" - "github.com/pb33f/libopenapi/resolver" "github.com/pb33f/libopenapi/utils" "github.com/stretchr/testify/assert" ) @@ -433,7 +433,7 @@ components: // resolving error is a pointer to *resolver.ResolvingError // which provides access to rich details about the error. - circularReference := resolvingError.(*resolver.ResolvingError).CircularReference + circularReference := resolvingError.(*index.ResolvingError).CircularReference // capture the journey with all details var buf strings.Builder diff --git a/resolver/resolver.go b/index/resolver.go similarity index 77% rename from resolver/resolver.go rename to index/resolver.go index ad191e7..5070b02 100644 --- a/resolver/resolver.go +++ b/index/resolver.go @@ -1,12 +1,12 @@ // Copyright 2022 Dave Shanley / Quobix // SPDX-License-Identifier: MIT -package resolver +package index import ( "fmt" + "strings" - "github.com/pb33f/libopenapi/index" "github.com/pb33f/libopenapi/utils" "gopkg.in/yaml.v3" ) @@ -23,7 +23,7 @@ type ResolvingError struct { Path string // CircularReference is set if the error is a reference to the circular reference. - CircularReference *index.CircularReferenceResult + CircularReference *CircularReferenceResult } func (r *ResolvingError) Error() string { @@ -34,20 +34,22 @@ func (r *ResolvingError) Error() string { // Resolver will use a *index.SpecIndex to stitch together a resolved root tree using all the discovered // references in the doc. type Resolver struct { - specIndex *index.SpecIndex - resolvedRoot *yaml.Node - resolvingErrors []*ResolvingError - circularReferences []*index.CircularReferenceResult - referencesVisited int - indexesVisited int - journeysTaken int - relativesSeen int - ignorePoly bool - ignoreArray bool + specIndex *SpecIndex + resolvedRoot *yaml.Node + resolvingErrors []*ResolvingError + circularReferences []*CircularReferenceResult + ignoredPolyReferences []*CircularReferenceResult + ignoredArrayReferences []*CircularReferenceResult + referencesVisited int + indexesVisited int + journeysTaken int + relativesSeen int + IgnorePoly bool + IgnoreArray bool } // NewResolver will create a new resolver from a *index.SpecIndex -func NewResolver(index *index.SpecIndex) *Resolver { +func NewResolver(index *SpecIndex) *Resolver { if index == nil { return nil } @@ -63,13 +65,13 @@ func (resolver *Resolver) GetResolvingErrors() []*ResolvingError { } // GetCircularErrors returns all circular reference errors found. -func (resolver *Resolver) GetCircularErrors() []*index.CircularReferenceResult { +func (resolver *Resolver) GetCircularErrors() []*CircularReferenceResult { return resolver.circularReferences } // GetPolymorphicCircularErrors returns all circular errors that stem from polymorphism -func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularReferenceResult { - var res []*index.CircularReferenceResult +func (resolver *Resolver) GetPolymorphicCircularErrors() []*CircularReferenceResult { + var res []*CircularReferenceResult for i := range resolver.circularReferences { if !resolver.circularReferences[i].IsInfiniteLoop { continue @@ -83,8 +85,8 @@ func (resolver *Resolver) GetPolymorphicCircularErrors() []*index.CircularRefere } // GetNonPolymorphicCircularErrors returns all circular errors that DO NOT stem from polymorphism -func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularReferenceResult { - var res []*index.CircularReferenceResult +func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*CircularReferenceResult { + var res []*CircularReferenceResult for i := range resolver.circularReferences { if !resolver.circularReferences[i].IsInfiniteLoop { continue @@ -100,13 +102,13 @@ func (resolver *Resolver) GetNonPolymorphicCircularErrors() []*index.CircularRef // IgnorePolymorphicCircularReferences will ignore any circular references that are polymorphic (oneOf, anyOf, allOf) // This must be set before any resolving is done. func (resolver *Resolver) IgnorePolymorphicCircularReferences() { - resolver.ignorePoly = true + resolver.IgnorePoly = true } // IgnoreArrayCircularReferences will ignore any circular references that stem from arrays. This must be set before // any resolving is done. func (resolver *Resolver) IgnoreArrayCircularReferences() { - resolver.ignoreArray = true + resolver.IgnoreArray = true } // GetJourneysTaken returns the number of journeys taken by the resolver @@ -174,13 +176,13 @@ func (resolver *Resolver) CheckForCircularReferences() []*ResolvingError { return resolver.resolvingErrors } -func visitIndexWithoutDamagingIt(res *Resolver, idx *index.SpecIndex) { +func visitIndexWithoutDamagingIt(res *Resolver, idx *SpecIndex) { mapped := idx.GetMappedReferencesSequenced() mappedIndex := idx.GetMappedReferences() res.indexesVisited++ for _, ref := range mapped { seenReferences := make(map[string]bool) - var journey []*index.Reference + var journey []*Reference res.journeysTaken++ res.VisitReference(ref.Reference, seenReferences, journey, false) } @@ -188,24 +190,24 @@ func visitIndexWithoutDamagingIt(res *Resolver, idx *index.SpecIndex) { for s, schemaRef := range schemas { if mappedIndex[s] == nil { seenReferences := make(map[string]bool) - var journey []*index.Reference + var journey []*Reference res.journeysTaken++ res.VisitReference(schemaRef, seenReferences, journey, false) } } - for _, c := range idx.GetChildren() { - visitIndexWithoutDamagingIt(res, c) - } + //for _, c := range idx.GetChildren() { + // visitIndexWithoutDamagingIt(res, c) + //} } -func visitIndex(res *Resolver, idx *index.SpecIndex) { +func visitIndex(res *Resolver, idx *SpecIndex) { mapped := idx.GetMappedReferencesSequenced() mappedIndex := idx.GetMappedReferences() res.indexesVisited++ for _, ref := range mapped { seenReferences := make(map[string]bool) - var journey []*index.Reference + var journey []*Reference res.journeysTaken++ if ref != nil && ref.Reference != nil { ref.Reference.Node.Content = res.VisitReference(ref.Reference, seenReferences, journey, true) @@ -216,7 +218,7 @@ func visitIndex(res *Resolver, idx *index.SpecIndex) { for s, schemaRef := range schemas { if mappedIndex[s] == nil { seenReferences := make(map[string]bool) - var journey []*index.Reference + var journey []*Reference res.journeysTaken++ schemaRef.Node.Content = res.VisitReference(schemaRef, seenReferences, journey, true) } @@ -231,13 +233,13 @@ func visitIndex(res *Resolver, idx *index.SpecIndex) { } } } - for _, c := range idx.GetChildren() { - visitIndex(res, c) - } + //for _, c := range idx.GetChildren() { + // visitIndex(res, c) + //} } // VisitReference will visit a reference as part of a journey and will return resolved nodes. -func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]bool, journey []*index.Reference, resolve bool) []*yaml.Node { +func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { resolver.referencesVisited++ if ref.Resolved || ref.Seen { return ref.Node.Content @@ -255,13 +257,13 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b for i, j := range journey { if j.Definition == r.Definition { - var foundDup *index.Reference + var foundDup *Reference foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition) if len(foundRefs) > 0 { foundDup = foundRefs[0] } - var circRef *index.CircularReferenceResult + var circRef *CircularReferenceResult if !foundDup.Circular { loop := append(journey, foundDup) @@ -272,7 +274,7 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b if r.ParentNodeSchemaType == "array" { isArray = true } - circRef = &index.CircularReferenceResult{ + circRef = &CircularReferenceResult{ Journey: loop, Start: foundDup, LoopIndex: i, @@ -280,7 +282,14 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b IsArrayResult: isArray, IsInfiniteLoop: isInfiniteLoop, } - resolver.circularReferences = append(resolver.circularReferences, circRef) + + if resolver.IgnoreArray && isArray { + fmt.Printf("Ignored: %s\n", circRef.GenerateJourneyPath()) + resolver.ignoredArrayReferences = append(resolver.ignoredArrayReferences, circRef) + } else { + fmt.Printf("Not Ignored: %s\n", circRef.GenerateJourneyPath()) + resolver.circularReferences = append(resolver.circularReferences, circRef) + } foundDup.Seen = true foundDup.Circular = true @@ -290,13 +299,13 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b } if !skip { - var original *index.Reference + var original *Reference foundRefs := resolver.specIndex.SearchIndexForReference(r.Definition) if len(foundRefs) > 0 { original = foundRefs[0] } resolved := resolver.VisitReference(original, seen, journey, resolve) - if resolve { + if resolve && !original.Circular { r.Node.Content = resolved // this is where we perform the actual resolving. } r.Seen = true @@ -309,7 +318,7 @@ func (resolver *Resolver) VisitReference(ref *index.Reference, seen map[string]b return ref.Node.Content } -func (resolver *Resolver) isInfiniteCircularDependency(ref *index.Reference, visitedDefinitions map[string]bool, initialRef *index.Reference) (bool, map[string]bool) { +func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDefinitions map[string]bool, initialRef *Reference) (bool, map[string]bool) { if ref == nil { return false, visitedDefinitions } @@ -342,38 +351,41 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *index.Reference, vis func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, foundRelatives map[string]bool, - journey []*index.Reference, resolve bool) []*index.Reference { + journey []*Reference, resolve bool) []*Reference { if len(journey) > 100 { return nil } - var found []*index.Reference + var found []*Reference + //var ignoredPoly []*index.Reference + //var ignoredArray []*index.Reference + if len(node.Content) > 0 { for i, n := range node.Content { if utils.IsNodeMap(n) || utils.IsNodeArray(n) { - var anyvn, allvn, onevn, arrayTypevn *yaml.Node + //var anyvn, allvn, onevn, arrayTypevn *yaml.Node // extract polymorphic references if len(n.Content) > 1 { - _, anyvn = utils.FindKeyNodeTop("anyOf", n.Content) - _, allvn = utils.FindKeyNodeTop("allOf", n.Content) - _, onevn = utils.FindKeyNodeTop("oneOf", n.Content) - _, arrayTypevn = utils.FindKeyNodeTop("type", n.Content) - } - if anyvn != nil || allvn != nil || onevn != nil { - if resolver.ignorePoly { - continue - } - } - if arrayTypevn != nil { - if arrayTypevn.Value == "array" { - if resolver.ignoreArray { - continue - } - } + //_, anyvn = utils.FindKeyNodeTop("anyOf", n.Content) + //_, allvn = utils.FindKeyNodeTop("allOf", n.Content) + //_, onevn = utils.FindKeyNodeTop("oneOf", n.Content) + //_, arrayTypevn = utils.FindKeyNodeTop("type", n.Content) } + //if anyvn != nil || allvn != nil || onevn != nil { + // if resolver.IgnorePoly { + // ignoredPoly = append(ignoredPoly, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...) + // } + //} + //if arrayTypevn != nil { + // if arrayTypevn.Value == "array" { + // if resolver.IgnoreArray { + // ignoredArray = append(ignoredArray, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...) + // } + // } + //} found = append(found, resolver.extractRelatives(n, node, foundRelatives, journey, resolve)...) } @@ -409,7 +421,7 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, } } - r := &index.Reference{ + r := &Reference{ Definition: value, Name: value, Node: node, @@ -447,7 +459,7 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, resolver.VisitReference(ref, foundRelatives, journey, resolve) } else { loop := append(journey, ref) - circRef := &index.CircularReferenceResult{ + circRef := &CircularReferenceResult{ Journey: loop, Start: ref, LoopIndex: i, @@ -458,7 +470,11 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, ref.Seen = true ref.Circular = true - resolver.circularReferences = append(resolver.circularReferences, circRef) + if resolver.IgnorePoly { + resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) + } else { + resolver.circularReferences = append(resolver.circularReferences, circRef) + } } } } @@ -472,6 +488,11 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, v := node.Content[i+1].Content[q] if utils.IsNodeMap(v) { if d, _, l := utils.IsNodeRefValue(v); d { + strangs := strings.Split(l, "/#") + if len(strangs) == 2 { + fmt.Println("wank") + } + ref := resolver.specIndex.GetMappedReferences()[l] if ref != nil && !ref.Circular { circ := false @@ -485,7 +506,8 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, resolver.VisitReference(ref, foundRelatives, journey, resolve) } else { loop := append(journey, ref) - circRef := &index.CircularReferenceResult{ + + circRef := &CircularReferenceResult{ Journey: loop, Start: ref, LoopIndex: i, @@ -496,7 +518,11 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, ref.Seen = true ref.Circular = true - resolver.circularReferences = append(resolver.circularReferences, circRef) + if resolver.IgnorePoly { + resolver.ignoredPolyReferences = append(resolver.ignoredPolyReferences, circRef) + } else { + resolver.circularReferences = append(resolver.circularReferences, circRef) + } } } } @@ -509,6 +535,8 @@ func (resolver *Resolver) extractRelatives(node, parent *yaml.Node, } } } + //resolver.ignoredPolyReferences = ignoredPoly + resolver.relativesSeen += len(found) return found } diff --git a/resolver/resolver_test.go b/index/resolver_test.go similarity index 87% rename from resolver/resolver_test.go rename to index/resolver_test.go index 9fe2d92..6fda4ea 100644 --- a/resolver/resolver_test.go +++ b/index/resolver_test.go @@ -1,4 +1,4 @@ -package resolver +package index import ( "errors" @@ -7,8 +7,7 @@ import ( "os" "testing" - "github.com/pb33f/libopenapi/index" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" ) @@ -21,7 +20,7 @@ func Benchmark_ResolveDocumentStripe(b *testing.B) { for n := 0; n < b.N; n++ { var rootNode yaml.Node _ = yaml.Unmarshal(stripe, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) resolver.Resolve() } @@ -32,7 +31,7 @@ func TestResolver_ResolveComponents_CircularSpec(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -49,7 +48,7 @@ func TestResolver_CheckForCircularReferences(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -83,7 +82,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -118,7 +117,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -154,7 +153,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -190,7 +189,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -226,7 +225,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -262,7 +261,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -296,7 +295,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -318,7 +317,7 @@ func TestResolver_CheckForCircularReferences_DigitalOcean(t *testing.T) { baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") - idx := index.NewSpecIndexWithConfig(&rootNode, &index.SpecIndexConfig{ + idx := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ AllowRemoteLookup: true, AllowFileLookup: true, BaseURL: baseURL, @@ -341,7 +340,7 @@ func TestResolver_CircularReferencesRequiredValid(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -358,7 +357,7 @@ func TestResolver_CircularReferencesRequiredInvalid(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(circular, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -371,11 +370,11 @@ func TestResolver_CircularReferencesRequiredInvalid(t *testing.T) { } func TestResolver_DeepJourney(t *testing.T) { - var journey []*index.Reference + var journey []*Reference for f := 0; f < 200; f++ { journey = append(journey, nil) } - idx := index.NewSpecIndexWithConfig(nil, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(nil, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.Nil(t, resolver.extractRelatives(nil, nil, nil, journey, false)) } @@ -385,7 +384,7 @@ func TestResolver_ResolveComponents_Stripe(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(stripe, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -402,7 +401,7 @@ func TestResolver_ResolveComponents_BurgerShop(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(mixedref, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -435,7 +434,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal([]byte(yml), &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -462,7 +461,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal([]byte(yml), &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -498,7 +497,7 @@ components: var rootNode yaml.Node _ = yaml.Unmarshal([]byte(yml), &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -513,8 +512,8 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(mixedref, &rootNode) - b := index.CreateOpenAPIIndexConfig() - idx := index.NewSpecIndexWithConfig(&rootNode, b) + b := CreateOpenAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, b) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -534,7 +533,7 @@ func TestResolver_ResolveComponents_k8s(t *testing.T) { var rootNode yaml.Node _ = yaml.Unmarshal(k8s, &rootNode) - idx := index.NewSpecIndexWithConfig(&rootNode, index.CreateClosedAPIIndexConfig()) + idx := NewSpecIndexWithConfig(&rootNode, CreateClosedAPIIndexConfig()) resolver := NewResolver(idx) assert.NotNil(t, resolver) @@ -555,8 +554,8 @@ func ExampleNewResolver() { _ = yaml.Unmarshal(stripeBytes, &rootNode) // create a new spec index (resolver depends on it) - indexConfig := index.CreateClosedAPIIndexConfig() - idx := index.NewSpecIndexWithConfig(&rootNode, indexConfig) + indexConfig := CreateClosedAPIIndexConfig() + idx := NewSpecIndexWithConfig(&rootNode, indexConfig) // create a new resolver using the index. resolver := NewResolver(idx) @@ -581,7 +580,7 @@ func ExampleResolvingError() { Column: 21, }, Path: "#/definitions/JeSuisUneErreur", - CircularReference: &index.CircularReferenceResult{}, + CircularReference: &CircularReferenceResult{}, } fmt.Printf("%s", re.Error()) diff --git a/index/rolodex.go b/index/rolodex.go index a15b30e..f208ab1 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -57,6 +57,8 @@ type Rolodex struct { indexConfig *SpecIndexConfig indexingDuration time.Duration indexes []*SpecIndex + rootIndex *SpecIndex + caughtErrors []error } type rolodexFile struct { @@ -204,10 +206,22 @@ func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex { return r } +func (r *Rolodex) GetIndexingDuration() time.Duration { + return r.indexingDuration +} + +func (r *Rolodex) GetRootIndex() *SpecIndex { + return r.rootIndex +} + func (r *Rolodex) GetIndexes() []*SpecIndex { return r.indexes } +func (r *Rolodex) GetCaughtErrors() []error { + return r.caughtErrors +} + func (r *Rolodex) AddLocalFS(baseDir string, fileSystem fs.FS) { absBaseDir, _ := filepath.Abs(baseDir) r.localFS[absBaseDir] = fileSystem @@ -246,6 +260,23 @@ func (r *Rolodex) IndexTheRolodex() error { copiedConfig.SpecAbsolutePath = fullPath copiedConfig.AvoidBuildIndex = true // we will build out everything in two steps. idx, err := idxFile.Index(&copiedConfig) + + // for each index, we need a resolver + resolver := NewResolver(idx) + idx.resolver = resolver + + // check if the config has been set to ignore circular references in arrays and polymorphic schemas + if copiedConfig.IgnoreArrayCircularReferences { + resolver.IgnoreArrayCircularReferences() + } + if copiedConfig.IgnorePolymorphicCircularReferences { + resolver.IgnorePolymorphicCircularReferences() + } + resolvingErrors := resolver.CheckForCircularReferences() + for e := range resolvingErrors { + caughtErrors = append(caughtErrors, resolvingErrors[e]) + } + if err != nil { errChan <- err } @@ -295,10 +326,29 @@ func (r *Rolodex) IndexTheRolodex() error { // now that we have indexed all the files, we can build the index. for _, idx := range indexBuildQueue { idx.BuildIndex() + } r.indexes = indexBuildQueue + + // indexed and built every supporting file, we can build the root index (our entry point) + index := NewSpecIndexWithConfig(r.indexConfig.SpecInfo.RootNode, r.indexConfig) + resolver := NewResolver(index) + if r.indexConfig.IgnoreArrayCircularReferences { + resolver.IgnoreArrayCircularReferences() + } + if r.indexConfig.IgnorePolymorphicCircularReferences { + resolver.IgnorePolymorphicCircularReferences() + } + index.resolver = resolver + resolvingErrors := resolver.CheckForCircularReferences() + for e := range resolvingErrors { + caughtErrors = append(caughtErrors, resolvingErrors[e]) + } + + r.rootIndex = index r.indexingDuration = time.Now().Sub(started) r.indexed = true + r.caughtErrors = caughtErrors return errors.Join(caughtErrors...) } @@ -363,6 +413,7 @@ func (r *Rolodex) Open(location string) (RolodexFile, error) { data: bytes, fullPath: fileLookup, lastModified: s.ModTime(), + index: r.rootIndex, } break } diff --git a/index/rolodex_test_data/dir1/components.yaml b/index/rolodex_test_data/dir1/components.yaml index 8d521ee..dd14d4c 100644 --- a/index/rolodex_test_data/dir1/components.yaml +++ b/index/rolodex_test_data/dir1/components.yaml @@ -1,13 +1,15 @@ openapi: 3.1.0 info: - title: Rolodex Test Data + title: Dir1 Test Components version: 1.0.0 components: schemas: - Ding: + GlobalComponent: type: object - description: A thing that does nothing. Ding a ling! + description: Dir1 Global Component properties: message: type: string - description: I am pointless. Ding Ding! \ No newline at end of file + description: I am pointless, but I am global dir1. + SomeUtil: + $ref: "utils/utils.yaml" \ No newline at end of file diff --git a/index/rolodex_test_data/dir1/subdir1/shared.yaml b/index/rolodex_test_data/dir1/subdir1/shared.yaml index e69de29..d70a69a 100644 --- a/index/rolodex_test_data/dir1/subdir1/shared.yaml +++ b/index/rolodex_test_data/dir1/subdir1/shared.yaml @@ -0,0 +1,15 @@ +openapi: 3.1.0 +info: + title: Dir1 Shared Components + version: 1.0.0 +components: + schemas: + SharedComponent: + type: object + description: Dir1 Shared Component + properties: + message: + type: string + description: I am pointless, but I am shared dir1. + SomeUtil: + $ref: "../utils/utils.yaml" \ No newline at end of file diff --git a/index/rolodex_test_data/dir1/utils/utils.yaml b/index/rolodex_test_data/dir1/utils/utils.yaml index e69de29..59b33bc 100644 --- a/index/rolodex_test_data/dir1/utils/utils.yaml +++ b/index/rolodex_test_data/dir1/utils/utils.yaml @@ -0,0 +1,11 @@ +type: object +description: I am a utility for dir1 +properties: + message: + type: string + description: I am pointless dir1. + properties: + link: + $ref: "../components.yaml#/components/schemas/GlobalComponent" + shared: + $ref: '../shared/shared.yaml#/components/schemas/SharedComponent' \ No newline at end of file diff --git a/index/rolodex_test_data/dir2/components.yaml b/index/rolodex_test_data/dir2/components.yaml index dd14d4c..f41d4aa 100644 --- a/index/rolodex_test_data/dir2/components.yaml +++ b/index/rolodex_test_data/dir2/components.yaml @@ -1,15 +1,15 @@ openapi: 3.1.0 info: - title: Dir1 Test Components + title: Dir2 Test Components version: 1.0.0 components: schemas: GlobalComponent: type: object - description: Dir1 Global Component + description: Dir2 Global Component properties: message: type: string - description: I am pointless, but I am global dir1. + description: I am pointless, but I am global dir2. SomeUtil: $ref: "utils/utils.yaml" \ No newline at end of file diff --git a/index/rolodex_test_data/dir2/subdir2/shared.yaml b/index/rolodex_test_data/dir2/subdir2/shared.yaml index e69de29..3c33657 100644 --- a/index/rolodex_test_data/dir2/subdir2/shared.yaml +++ b/index/rolodex_test_data/dir2/subdir2/shared.yaml @@ -0,0 +1,15 @@ +openapi: 3.1.0 +info: + title: Dir2 Shared Components + version: 1.0.0 +components: + schemas: + SharedComponent: + type: object + description: Dir2 Shared Component + properties: + message: + type: string + description: I am pointless, but I am shared dir2. + SomeUtil: + $ref: "../utils/utils.yaml" \ No newline at end of file diff --git a/index/rolodex_test_data/dir2/utils/utils.yaml b/index/rolodex_test_data/dir2/utils/utils.yaml index e69de29..471b4c0 100644 --- a/index/rolodex_test_data/dir2/utils/utils.yaml +++ b/index/rolodex_test_data/dir2/utils/utils.yaml @@ -0,0 +1,11 @@ +type: object +description: I am a utility for dir2 +properties: + message: + type: string + description: I am pointless dir2. + properties: + link: + $ref: "../components.yaml#/components/schemas/GlobalComponent" + shared: + $ref: '../shared/shared.yaml#/components/schemas/SharedComponent' \ No newline at end of file diff --git a/index/rolodex_test_data/doc2.yaml b/index/rolodex_test_data/doc2.yaml index e69de29..10586ec 100644 --- a/index/rolodex_test_data/doc2.yaml +++ b/index/rolodex_test_data/doc2.yaml @@ -0,0 +1,50 @@ +openapi: 3.1.0 +info: + title: Rolodex Test Data + version: 1.0.0 +paths: + /one/local: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Thing' + /one/file: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: 'components.yaml#/components/schemas/Ding' + /nested/files1: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: 'dir1/components.yaml#/components/schemas/GlobalComponent' + /nested/files2: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: 'dir2/components.yaml#/components/schemas/GlobalComponent' +components: + schemas: + Thing: + type: object + description: A thing that does nothing. + properties: + message: + type: string + description: I am pointless. \ No newline at end of file diff --git a/index/search_index.go b/index/search_index.go index 0c8f698..2586d08 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -7,26 +7,36 @@ package index // and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes // extracted when parsing the OpenAPI Spec. func (index *SpecIndex) SearchIndexForReference(ref string) []*Reference { + if r, ok := index.allMappedRefs[ref]; ok { return []*Reference{r} } - for c := range index.children { - found := goFindMeSomething(index.children[c], ref) - if found != nil { - return found - } - } + + // TODO: look in the rolodex. + return nil + + //if r, ok := index.allMappedRefs[ref]; ok { + // return []*Reference{r} + //} + //for c := range index.children { + // found := goFindMeSomething(index.children[c], ref) + // if found != nil { + // return found + // } + //} + //return nil } func (index *SpecIndex) SearchAncestryForSeenURI(uri string) *SpecIndex { - if index.parentIndex == nil { - return nil - } - if index.uri[0] != uri { - return index.parentIndex.SearchAncestryForSeenURI(uri) - } - return index + //if index.parentIndex == nil { + // return nil + //} + //if index.uri[0] != uri { + // return index.parentIndex.SearchAncestryForSeenURI(uri) + //} + //return index + return nil } func goFindMeSomething(i *SpecIndex, ref string) []*Reference { diff --git a/index/spec_index.go b/index/spec_index.go index 6d78f37..53c6aba 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -13,14 +13,15 @@ package index import ( + "context" "fmt" "sort" "strings" "sync" + "time" "github.com/pb33f/libopenapi/utils" "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" - "golang.org/x/sync/syncmap" "gopkg.in/yaml.v3" ) @@ -29,13 +30,15 @@ import ( // how the index is set up. func NewSpecIndexWithConfig(rootNode *yaml.Node, config *SpecIndexConfig) *SpecIndex { index := new(SpecIndex) - if config != nil && config.seenRemoteSources == nil { - config.seenRemoteSources = &syncmap.Map{} - } - config.remoteLock = &sync.Mutex{} + //if config != nil && config.seenRemoteSources == nil { + // config.seenRemoteSources = &syncmap.Map{} + //} + //config.remoteLock = &sync.Mutex{} index.config = config - index.parentIndex = config.ParentIndex + index.rolodex = config.Rolodex + //index.parentIndex = config.ParentIndex index.uri = config.uri + index.specAbsolutePath = config.SpecAbsolutePath if rootNode == nil || len(rootNode.Content) <= 0 { return index } @@ -89,10 +92,10 @@ func createNewIndex(rootNode *yaml.Node, index *SpecIndex, avoidBuildOut bool) * } // do a copy! - index.config.seenRemoteSources.Range(func(k, v any) bool { - index.seenRemoteSources[k.(string)] = v.(*yaml.Node) - return true - }) + //index.config.seenRemoteSources.Range(func(k, v any) bool { + // index.seenRemoteSources[k.(string)] = v.(*yaml.Node) + // return true + //}) return index } @@ -618,13 +621,34 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { return index.globalCallbacksCount } - // index.pathRefsLock.Lock() + index.pathRefsLock.RLock() for path, p := range index.pathRefs { for _, m := range p { // look through method for callbacks callbacks, _ := yamlpath.NewPath("$..callbacks") - res, _ := callbacks.Find(m.Node) + // Channel used to receive the result from doSomething function + ch := make(chan string, 1) + + // Create a context with a timeout of 5 seconds + ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + + var res []*yaml.Node + + doSomething := func(ctx context.Context, ch chan<- string) { + res, _ = callbacks.Find(m.Node) + ch <- m.Definition + } + + // Start the doSomething function + go doSomething(ctxTimeout, ch) + + select { + case <-ctxTimeout.Done(): + fmt.Printf("Callback %d: Context cancelled: %v\n", m.Node.Line, ctxTimeout.Err()) + case <-ch: + } if len(res) > 0 { for _, callback := range res[0].Content { @@ -650,7 +674,7 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { } } } - // index.pathRefsLock.Unlock() + index.pathRefsLock.RUnlock() return index.globalCallbacksCount } @@ -670,7 +694,29 @@ func (index *SpecIndex) GetGlobalLinksCount() int { // look through method for links links, _ := yamlpath.NewPath("$..links") - res, _ := links.Find(m.Node) + + // Channel used to receive the result from doSomething function + ch := make(chan string, 1) + + // Create a context with a timeout of 5 seconds + ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + + var res []*yaml.Node + + doSomething := func(ctx context.Context, ch chan<- string) { + res, _ = links.Find(m.Node) + ch <- m.Definition + } + + // Start the doSomething function + go doSomething(ctxTimeout, ch) + + select { + case <-ctxTimeout.Done(): + fmt.Printf("Global links %d ref: Context cancelled: %v\n", m.Node.Line, ctxTimeout.Err()) + case <-ch: + } if len(res) > 0 { for _, link := range res[0].Content { @@ -928,6 +974,8 @@ func (index *SpecIndex) GetOperationCount() int { opCount := 0 + locatedPathRefs := make(map[string]map[string]*Reference) + for x, p := range index.pathsNode.Content { if x%2 == 0 { @@ -950,6 +998,7 @@ func (index *SpecIndex) GetOperationCount() int { } } if valid { + fmt.Sprint(p) ref := &Reference{ Definition: m.Value, Name: m.Value, @@ -957,12 +1006,12 @@ func (index *SpecIndex) GetOperationCount() int { Path: fmt.Sprintf("$.paths.%s.%s", p.Value, m.Value), ParentNode: m, } - index.pathRefsLock.Lock() - if index.pathRefs[p.Value] == nil { - index.pathRefs[p.Value] = make(map[string]*Reference) + //index.pathRefsLock.Lock() + if locatedPathRefs[p.Value] == nil { + locatedPathRefs[p.Value] = make(map[string]*Reference) } - index.pathRefs[p.Value][ref.Name] = ref - index.pathRefsLock.Unlock() + locatedPathRefs[p.Value][ref.Name] = ref + //index.pathRefsLock.Unlock() // update opCount++ } @@ -970,7 +1019,11 @@ func (index *SpecIndex) GetOperationCount() int { } } } - + index.pathRefsLock.Lock() + for k, v := range locatedPathRefs { + index.pathRefs[k] = v + } + index.pathRefsLock.Unlock() index.operationCount = opCount return opCount } @@ -1188,13 +1241,13 @@ func (index *SpecIndex) GetAllSummariesCount() int { // CheckForSeenRemoteSource will check to see if we have already seen this remote source and return it, // to avoid making duplicate remote calls for document data. -func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) { - if index.config == nil || index.config.seenRemoteSources == nil { - return false, nil - } - j, _ := index.config.seenRemoteSources.Load(url) - if j != nil { - return true, j.(*yaml.Node) - } - return false, nil -} +//func (index *SpecIndex) CheckForSeenRemoteSource(url string) (bool, *yaml.Node) { +// if index.config == nil || index.config.seenRemoteSources == nil { +// return false, nil +// } +// j, _ := index.config.seenRemoteSources.Load(url) +// if j != nil { +// return true, j.(*yaml.Node) +// } +// return false, nil +//} diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 4224f2e..de235da 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -92,9 +92,9 @@ func TestSpecIndex_DigitalOcean(t *testing.T) { baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - AllowRemoteLookup: true, - AllowFileLookup: true, + BaseURL: baseURL, + //AllowRemoteLookup: true, + //AllowFileLookup: true, }) assert.Len(t, index.GetAllExternalIndexes(), 291) @@ -163,9 +163,9 @@ func TestSpecIndex_BaseURLError(t *testing.T) { // anything. baseURL, _ := url.Parse("https://githerbs.com/fresh/herbs/for/you") index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - BaseURL: baseURL, - AllowRemoteLookup: true, - AllowFileLookup: true, + BaseURL: baseURL, + //AllowRemoteLookup: true, + //AllowFileLookup: true, }) assert.Len(t, index.GetAllExternalIndexes(), 0) @@ -441,9 +441,9 @@ func TestSpecIndex_BurgerShopMixedRef(t *testing.T) { cwd, _ := os.Getwd() index := NewSpecIndexWithConfig(&rootNode, &SpecIndexConfig{ - AllowRemoteLookup: true, - AllowFileLookup: true, - BasePath: cwd, + //AllowRemoteLookup: true, + // AllowFileLookup: true, + BasePath: cwd, }) assert.Len(t, index.allRefs, 5) @@ -630,7 +630,7 @@ func TestSpecIndex_TestPathsNodeAsArray(t *testing.T) { _ = yaml.Unmarshal([]byte(yml), &rootNode) index := NewSpecIndexWithConfig(&rootNode, CreateOpenAPIIndexConfig()) - assert.Nil(t, index.performExternalLookup(nil, "unknown", nil, nil)) + assert.Nil(t, index.performExternalLookup(nil)) } func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_Error(t *testing.T) { @@ -734,7 +734,7 @@ paths: func TestSpecIndex_lookupRemoteReference_SeenSourceSimulation_BadJSON(t *testing.T) { index := NewSpecIndexWithConfig(nil, &SpecIndexConfig{ - AllowRemoteLookup: true, + //AllowRemoteLookup: true, }) index.seenRemoteSources = make(map[string]*yaml.Node) a, b, err := index.lookupRemoteReference("https://google.com//logos/doodles/2022/labor-day-2022-6753651837109490.3-l.png#/hey") diff --git a/index/utility_methods.go b/index/utility_methods.go index 8803872..109b48d 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -28,14 +28,14 @@ func (index *SpecIndex) extractDefinitionsAndSchemas(schemasNode *yaml.Node, pat Node: schema, Path: fmt.Sprintf("$.components.schemas.%s", name), ParentNode: schemasNode, - RequiredRefProperties: index.extractDefinitionRequiredRefProperties(schemasNode, map[string][]string{}), + RequiredRefProperties: extractDefinitionRequiredRefProperties(schemasNode, map[string][]string{}), } index.allComponentSchemaDefinitions[def] = ref } } // extractDefinitionRequiredRefProperties goes through the direct properties of a schema and extracts the map of required definitions from within it -func (index *SpecIndex) extractDefinitionRequiredRefProperties(schemaNode *yaml.Node, reqRefProps map[string][]string) map[string][]string { +func extractDefinitionRequiredRefProperties(schemaNode *yaml.Node, reqRefProps map[string][]string) map[string][]string { if schemaNode == nil { return reqRefProps } @@ -70,7 +70,7 @@ func (index *SpecIndex) extractDefinitionRequiredRefProperties(schemaNode *yaml. // Check to see if the current property is directly embedded within the current schema, and handle its properties if so _, paramPropertiesMapNode := utils.FindKeyNodeTop("properties", param.Content) if paramPropertiesMapNode != nil { - reqRefProps = index.extractDefinitionRequiredRefProperties(param, reqRefProps) + reqRefProps = extractDefinitionRequiredRefProperties(param, reqRefProps) } // Check to see if the current property is polymorphic, and dive into that model if so @@ -78,7 +78,7 @@ func (index *SpecIndex) extractDefinitionRequiredRefProperties(schemaNode *yaml. _, ofNode := utils.FindKeyNodeTop(key, param.Content) if ofNode != nil { for _, ofNodeItem := range ofNode.Content { - reqRefProps = index.extractRequiredReferenceProperties(ofNodeItem, name, reqRefProps) + reqRefProps = extractRequiredReferenceProperties(ofNodeItem, name, reqRefProps) } } } @@ -91,14 +91,14 @@ func (index *SpecIndex) extractDefinitionRequiredRefProperties(schemaNode *yaml. continue } - reqRefProps = index.extractRequiredReferenceProperties(requiredPropDefNode, requiredPropertyNode.Value, reqRefProps) + reqRefProps = extractRequiredReferenceProperties(requiredPropDefNode, requiredPropertyNode.Value, reqRefProps) } return reqRefProps } // extractRequiredReferenceProperties returns a map of definition names to the property or properties which reference it within a node -func (index *SpecIndex) extractRequiredReferenceProperties(requiredPropDefNode *yaml.Node, propName string, reqRefProps map[string][]string) map[string][]string { +func extractRequiredReferenceProperties(requiredPropDefNode *yaml.Node, propName string, reqRefProps map[string][]string) map[string][]string { isRef, _, defPath := utils.IsNodeRefValue(requiredPropDefNode) if !isRef { _, defItems := utils.FindKeyNodeTop("items", requiredPropDefNode.Content) diff --git a/index/utility_methods_test.go b/index/utility_methods_test.go index aec4506..b8cad43 100644 --- a/index/utility_methods_test.go +++ b/index/utility_methods_test.go @@ -49,7 +49,5 @@ func TestGenerateCleanSpecConfigBaseURL_HttpStrip(t *testing.T) { } func TestSpecIndex_extractDefinitionRequiredRefProperties(t *testing.T) { - c := CreateOpenAPIIndexConfig() - idx := NewSpecIndexWithConfig(nil, c) - assert.Nil(t, idx.extractDefinitionRequiredRefProperties(nil, nil)) + assert.Nil(t, extractDefinitionRequiredRefProperties(nil, nil)) } diff --git a/utils/unwrap_errors.go b/utils/unwrap_errors.go new file mode 100644 index 0000000..7d1f83a --- /dev/null +++ b/utils/unwrap_errors.go @@ -0,0 +1,11 @@ +// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package utils + +func UnwrapErrors(err error) []error { + if err == nil { + return []error{} + } + return err.(interface{ Unwrap() []error }).Unwrap() +} diff --git a/what-changed/model/components_test.go b/what-changed/model/components_test.go index 4853b66..99544ba 100644 --- a/what-changed/model/components_test.go +++ b/what-changed/model/components_test.go @@ -4,14 +4,13 @@ package model import ( - "github.com/pb33f/libopenapi/datamodel/low" - v2 "github.com/pb33f/libopenapi/datamodel/low/v2" - "github.com/pb33f/libopenapi/datamodel/low/v3" - "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/resolver" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" - "testing" + "github.com/pb33f/libopenapi/datamodel/low" + v2 "github.com/pb33f/libopenapi/datamodel/low/v2" + "github.com/pb33f/libopenapi/datamodel/low/v3" + "github.com/pb33f/libopenapi/index" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "testing" ) func TestCompareComponents_Swagger_Definitions_Equal(t *testing.T) { @@ -677,8 +676,8 @@ func TestCompareComponents_OpenAPI_Responses_FullBuild_CircularRef(t *testing.T) idx2 := index.NewSpecIndex(&rNode) // resolver required to check circular refs. - re1 := resolver.NewResolver(idx) - re2 := resolver.NewResolver(idx2) + re1 := index.NewResolver(idx) + re2 := index.NewResolver(idx2) re1.CheckForCircularReferences() re2.CheckForCircularReferences()