diff --git a/datamodel/high/v3/callback_test.go b/datamodel/high/v3/callback_test.go index b6a748b..406a23c 100644 --- a/datamodel/high/v3/callback_test.go +++ b/datamodel/high/v3/callback_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "github.com/pb33f/libopenapi/datamodel/low" v3 "github.com/pb33f/libopenapi/datamodel/low/v3" "github.com/pb33f/libopenapi/index" @@ -65,7 +66,7 @@ func TestCallback_MarshalYAML(t *testing.T) { var n v3.Callback _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewCallback(&n) diff --git a/datamodel/high/v3/components_test.go b/datamodel/high/v3/components_test.go index 5be13d4..4231087 100644 --- a/datamodel/high/v3/components_test.go +++ b/datamodel/high/v3/components_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "github.com/pb33f/libopenapi/datamodel/low" v3 "github.com/pb33f/libopenapi/datamodel/low/v3" "github.com/pb33f/libopenapi/index" @@ -46,7 +47,7 @@ func TestComponents_MarshalYAML(t *testing.T) { var n v3.Components _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(idxNode.Content[0], idx) + _ = n.Build(context.Background(), idxNode.Content[0], idx) r := NewComponents(&n) diff --git a/datamodel/high/v3/document_test.go b/datamodel/high/v3/document_test.go index bb09a4d..ebc9c61 100644 --- a/datamodel/high/v3/document_test.go +++ b/datamodel/high/v3/document_test.go @@ -5,16 +5,22 @@ package v3 import ( "fmt" - "net/url" - "os" - "strings" - "testing" - "github.com/pb33f/libopenapi/datamodel" v2 "github.com/pb33f/libopenapi/datamodel/high/v2" lowv2 "github.com/pb33f/libopenapi/datamodel/low/v2" lowv3 "github.com/pb33f/libopenapi/datamodel/low/v3" + "github.com/pb33f/libopenapi/utils" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" ) var lowDoc *lowv3.Document @@ -22,7 +28,7 @@ var lowDoc *lowv3.Document func initTest() { data, _ := os.ReadFile("../../../test_specs/burgershop.openapi.yaml") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: true, AllowRemoteReferences: true, @@ -382,9 +388,9 @@ func testBurgerShop(t *testing.T, h *Document, checkLines bool) { func TestStripeAsDoc(t *testing.T) { data, _ := os.ReadFile("../../../test_specs/stripe.yaml") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, datamodel.NewOpenDocumentConfiguration()) - assert.Len(t, err, 3) + assert.Len(t, utils.UnwrapErrors(err), 3) d := NewDocument(lowDoc) assert.NotNil(t, d) } @@ -402,7 +408,7 @@ func TestK8sAsDoc(t *testing.T) { func TestAsanaAsDoc(t *testing.T) { data, _ := os.ReadFile("../../../test_specs/asana.yaml") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, datamodel.NewOpenDocumentConfiguration()) if err != nil { panic("broken something") @@ -412,10 +418,51 @@ func TestAsanaAsDoc(t *testing.T) { assert.Equal(t, 118, len(d.Paths.PathItems)) } +func TestDigitalOceanAsDocViaCheckout(t *testing.T) { + + // this is a full checkout of the digitalocean API repo. + tmp, _ := os.MkdirTemp("", "openapi") + cmd := exec.Command("git", "clone", "https://github.com/digitalocean/openapi", tmp) + defer os.RemoveAll(filepath.Join(tmp, "openapi")) + + err := cmd.Run() + if err != nil { + log.Fatalf("cmd.Run() failed with %s\n", err) + } + + spec, _ := filepath.Abs(filepath.Join(tmp, "specification", "DigitalOcean-public.v2.yaml")) + doLocal, _ := os.ReadFile(spec) + + var rootNode yaml.Node + _ = yaml.Unmarshal(doLocal, &rootNode) + + basePath := filepath.Join(tmp, "specification") + + data, _ := os.ReadFile("../../../test_specs/digitalocean.yaml") + info, _ := datamodel.ExtractSpecInfo(data) + + config := datamodel.DocumentConfiguration{ + AllowFileReferences: true, + AllowRemoteReferences: true, + BasePath: basePath, + } + + lowDoc, err = lowv3.CreateDocumentFromConfig(info, &config) + if err != nil { + er := utils.UnwrapErrors(err) + for e := range er { + fmt.Println(er[e]) + } + } + d := NewDocument(lowDoc) + assert.NotNil(t, d) + assert.Equal(t, 183, len(d.Paths.PathItems)) +} + func TestDigitalOceanAsDocFromSHA(t *testing.T) { data, _ := os.ReadFile("../../../test_specs/digitalocean.yaml") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/82e1d558e15a59edc1d47d2c5544e7138f5b3cbf/specification") config := datamodel.DocumentConfiguration{ @@ -424,12 +471,53 @@ func TestDigitalOceanAsDocFromSHA(t *testing.T) { BaseURL: baseURL, } + if os.Getenv("GITHUB_TOKEN") != "" { + client := &http.Client{ + Timeout: time.Second * 60, + } + config.RemoteURLHandler = func(url string) (*http.Response, error) { + request, _ := http.NewRequest(http.MethodGet, url, nil) + request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", os.Getenv("GITHUB_TOKEN"))) + return client.Do(request) + } + } + + lowDoc, err = lowv3.CreateDocumentFromConfig(info, &config) + assert.Len(t, utils.UnwrapErrors(err), 3) // there are 3 404's in this release of the API. + d := NewDocument(lowDoc) + assert.NotNil(t, d) + assert.Equal(t, 183, len(d.Paths.PathItems)) +} + +func TestDigitalOceanAsDocFromMain(t *testing.T) { + data, _ := os.ReadFile("../../../test_specs/digitalocean.yaml") + info, _ := datamodel.ExtractSpecInfo(data) + var err error + + baseURL, _ := url.Parse("https://raw.githubusercontent.com/digitalocean/openapi/main/specification") + config := datamodel.DocumentConfiguration{ + AllowFileReferences: true, + AllowRemoteReferences: true, + BaseURL: baseURL, + } + + if os.Getenv("GITHUB_TOKEN") != "" { + client := &http.Client{ + Timeout: time.Second * 60, + } + config.RemoteURLHandler = func(url string) (*http.Response, error) { + request, _ := http.NewRequest(http.MethodGet, url, nil) + request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", os.Getenv("GITHUB_TOKEN"))) + return client.Do(request) + } + } + lowDoc, err = lowv3.CreateDocumentFromConfig(info, &config) if err != nil { - for e := range err { - fmt.Println(err[e]) + er := utils.UnwrapErrors(err) + for e := range er { + fmt.Printf("Reported Error: %s\n", er[e]) } - panic("broken something") } d := NewDocument(lowDoc) assert.NotNil(t, d) @@ -439,7 +527,7 @@ func TestDigitalOceanAsDocFromSHA(t *testing.T) { func TestPetstoreAsDoc(t *testing.T) { data, _ := os.ReadFile("../../../test_specs/petstorev3.json") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, datamodel.NewOpenDocumentConfiguration()) if err != nil { panic("broken something") @@ -452,10 +540,10 @@ func TestPetstoreAsDoc(t *testing.T) { func TestCircularReferencesDoc(t *testing.T) { data, _ := os.ReadFile("../../../test_specs/circular-tests.yaml") info, _ := datamodel.ExtractSpecInfo(data) - var err []error - lowDoc, err = lowv3.CreateDocumentFromConfig(info, datamodel.NewOpenDocumentConfiguration()) - assert.Len(t, err, 3) - d := NewDocument(lowDoc) + + lDoc, err := lowv3.CreateDocumentFromConfig(info, datamodel.NewOpenDocumentConfiguration()) + assert.Len(t, utils.UnwrapErrors(err), 3) + d := NewDocument(lDoc) assert.Len(t, d.Components.Schemas, 9) assert.Len(t, d.Index.GetCircularReferences(), 3) } @@ -604,7 +692,7 @@ components: numPatties: 1` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: true, AllowRemoteReferences: true, @@ -657,7 +745,7 @@ components: required: true` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: true, AllowRemoteReferences: true, @@ -685,7 +773,7 @@ components: ` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) - var err []error + var err error lowDoc, err = lowv3.CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: true, AllowRemoteReferences: true, diff --git a/datamodel/high/v3/media_type_test.go b/datamodel/high/v3/media_type_test.go index 905ed78..fb6b618 100644 --- a/datamodel/high/v3/media_type_test.go +++ b/datamodel/high/v3/media_type_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "os" "strings" "testing" @@ -20,7 +21,7 @@ func TestMediaType_MarshalYAMLInline(t *testing.T) { // load the petstore spec data, _ := os.ReadFile("../../../test_specs/petstorev3.json") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error lowDoc, err = v3.CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{}) if err != nil { panic("broken something") @@ -110,7 +111,7 @@ func TestMediaType_MarshalYAML(t *testing.T) { // load the petstore spec data, _ := os.ReadFile("../../../test_specs/petstorev3.json") info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error lowDoc, err = v3.CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{}) if err != nil { panic("broken something") @@ -161,7 +162,7 @@ func TestMediaType_Examples(t *testing.T) { var n v3.MediaType _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewMediaType(&n) diff --git a/datamodel/high/v3/oauth_flows_test.go b/datamodel/high/v3/oauth_flows_test.go index 42f5229..c2a2641 100644 --- a/datamodel/high/v3/oauth_flows_test.go +++ b/datamodel/high/v3/oauth_flows_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "github.com/pb33f/libopenapi/datamodel/low" v3 "github.com/pb33f/libopenapi/datamodel/low/v3" "github.com/pb33f/libopenapi/index" @@ -43,7 +44,7 @@ clientCredentials: var n v3.OAuthFlows _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewOAuthFlows(&n) diff --git a/datamodel/high/v3/operation_test.go b/datamodel/high/v3/operation_test.go index 55a68d4..ef12f75 100644 --- a/datamodel/high/v3/operation_test.go +++ b/datamodel/high/v3/operation_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "strings" "testing" @@ -43,7 +44,7 @@ callbacks: var n v3.Operation _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewOperation(&n) @@ -140,7 +141,7 @@ security: []` var n v3.Operation _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewOperation(&n) @@ -158,7 +159,7 @@ func TestOperation_NoSecurity(t *testing.T) { var n v3.Operation _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewOperation(&n) diff --git a/datamodel/high/v3/package_test.go b/datamodel/high/v3/package_test.go index 9c9890c..a61ca33 100644 --- a/datamodel/high/v3/package_test.go +++ b/datamodel/high/v3/package_test.go @@ -5,6 +5,7 @@ package v3 import ( "fmt" + "github.com/pb33f/libopenapi/utils" "os" "github.com/pb33f/libopenapi/datamodel" @@ -19,17 +20,14 @@ func Example_createHighLevelOpenAPIDocument() { // Create a new *datamodel.SpecInfo from bytes. info, _ := datamodel.ExtractSpecInfo(data) - var err []error + var err error // Create a new low-level Document, capture any errors thrown during creation. - lowDoc, err = lowv3.CreateDocument(info) + lowDoc, err = lowv3.CreateDocumentFromConfig(info, datamodel.NewOpenDocumentConfiguration()) // Get upset if any errors were thrown. - if len(err) > 0 { - for i := range err { - fmt.Printf("error: %e", err[i]) - } - panic("something went wrong") + for i := range utils.UnwrapErrors(err) { + fmt.Printf("error: %v", i) } // Create a high-level Document from the low-level one. diff --git a/datamodel/high/v3/path_item_test.go b/datamodel/high/v3/path_item_test.go index db03254..d70a3ba 100644 --- a/datamodel/high/v3/path_item_test.go +++ b/datamodel/high/v3/path_item_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "strings" "testing" @@ -28,7 +29,7 @@ func TestPathItem(t *testing.T) { var n v3.PathItem _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewPathItem(&n) @@ -62,7 +63,7 @@ trace: var n v3.PathItem _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewPathItem(&n) diff --git a/datamodel/high/v3/paths_test.go b/datamodel/high/v3/paths_test.go index a2a847f..91c23d7 100644 --- a/datamodel/high/v3/paths_test.go +++ b/datamodel/high/v3/paths_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "strings" "testing" @@ -37,7 +38,7 @@ func TestPaths_MarshalYAML(t *testing.T) { err := low.BuildModel(&idxNode, &n) assert.NoError(t, err) - err = n.Build(nil, idxNode.Content[0], idx) + err = n.Build(context.Background(), nil, idxNode.Content[0], idx) assert.NoError(t, err) high := NewPaths(&n) @@ -89,7 +90,7 @@ func TestPaths_MarshalYAMLInline(t *testing.T) { err := low.BuildModel(&idxNode, &n) assert.NoError(t, err) - err = n.Build(nil, idxNode.Content[0], idx) + err = n.Build(context.Background(), nil, idxNode.Content[0], idx) assert.NoError(t, err) high := NewPaths(&n) diff --git a/datamodel/high/v3/response_test.go b/datamodel/high/v3/response_test.go index 749c6aa..fdf747e 100644 --- a/datamodel/high/v3/response_test.go +++ b/datamodel/high/v3/response_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "strings" "testing" @@ -38,7 +39,7 @@ links: var n v3.Response _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewResponse(&n) @@ -69,7 +70,7 @@ links: var n v3.Response _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewResponse(&n) @@ -97,7 +98,7 @@ links: var n v3.Response _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewResponse(&n) diff --git a/datamodel/high/v3/responses_test.go b/datamodel/high/v3/responses_test.go index 8bee83c..09042a5 100644 --- a/datamodel/high/v3/responses_test.go +++ b/datamodel/high/v3/responses_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "strings" "testing" @@ -30,7 +31,7 @@ func TestNewResponses(t *testing.T) { var n v3.Responses _ = low.BuildModel(&idxNode, &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewResponses(&n) @@ -60,7 +61,7 @@ func TestResponses_MarshalYAML(t *testing.T) { var n v3.Responses _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewResponses(&n) @@ -90,7 +91,7 @@ func TestResponses_MarshalYAMLInline(t *testing.T) { var n v3.Responses _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewResponses(&n) diff --git a/datamodel/high/v3/security_scheme_test.go b/datamodel/high/v3/security_scheme_test.go index 18b35d4..c26addf 100644 --- a/datamodel/high/v3/security_scheme_test.go +++ b/datamodel/high/v3/security_scheme_test.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "github.com/pb33f/libopenapi/datamodel/low" v3 "github.com/pb33f/libopenapi/datamodel/low/v3" "github.com/pb33f/libopenapi/index" @@ -31,7 +32,7 @@ func TestSecurityScheme_MarshalYAML(t *testing.T) { var n v3.SecurityScheme _ = low.BuildModel(idxNode.Content[0], &n) - _ = n.Build(nil, idxNode.Content[0], idx) + _ = n.Build(context.Background(), nil, idxNode.Content[0], idx) r := NewSecurityScheme(&n) diff --git a/datamodel/low/base/contact.go b/datamodel/low/base/contact.go index d612305..c895820 100644 --- a/datamodel/low/base/contact.go +++ b/datamodel/low/base/contact.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/index" @@ -23,7 +24,7 @@ type Contact struct { } // Build is not implemented for Contact (there is nothing to build). -func (c *Contact) Build(_, _ *yaml.Node, _ *index.SpecIndex) error { +func (c *Contact) Build(_ context.Context, _, _ *yaml.Node, _ *index.SpecIndex) error { c.Reference = new(low.Reference) // not implemented. return nil diff --git a/datamodel/low/base/example.go b/datamodel/low/base/example.go index ca8c94a..53ba20c 100644 --- a/datamodel/low/base/example.go +++ b/datamodel/low/base/example.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -60,7 +61,7 @@ func (ex *Example) Hash() [32]byte { } // Build extracts extensions and example value -func (ex *Example) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (ex *Example) Build(_ context.Context, _, root *yaml.Node, _ *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) ex.Reference = new(low.Reference) diff --git a/datamodel/low/base/external_doc.go b/datamodel/low/base/external_doc.go index 652145b..a4056a7 100644 --- a/datamodel/low/base/external_doc.go +++ b/datamodel/low/base/external_doc.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -33,7 +34,7 @@ func (ex *ExternalDoc) FindExtension(ext string) *low.ValueReference[any] { } // Build will extract extensions from the ExternalDoc instance. -func (ex *ExternalDoc) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (ex *ExternalDoc) Build(_ context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) ex.Reference = new(low.Reference) diff --git a/datamodel/low/base/info.go b/datamodel/low/base/info.go index caf1c75..f7440d6 100644 --- a/datamodel/low/base/info.go +++ b/datamodel/low/base/info.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/utils" @@ -45,18 +46,18 @@ func (i *Info) GetExtensions() map[low.KeyReference[string]]low.ValueReference[a } // Build will extract out the Contact and Info objects from the supplied root node. -func (i *Info) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (i *Info) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) i.Reference = new(low.Reference) i.Extensions = low.ExtractExtensions(root) // extract contact - contact, _ := low.ExtractObject[*Contact](ContactLabel, root, idx) + contact, _ := low.ExtractObject[*Contact](ctx, ContactLabel, root, idx) i.Contact = contact // extract license - lic, _ := low.ExtractObject[*License](LicenseLabel, root, idx) + lic, _ := low.ExtractObject[*License](ctx, LicenseLabel, root, idx) i.License = lic return nil } diff --git a/datamodel/low/base/license.go b/datamodel/low/base/license.go index c543875..aa5903b 100644 --- a/datamodel/low/base/license.go +++ b/datamodel/low/base/license.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -25,7 +26,7 @@ type License struct { } // Build out a license, complain if both a URL and identifier are present as they are mutually exclusive -func (l *License) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (l *License) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) l.Reference = new(low.Reference) diff --git a/datamodel/low/base/schema.go b/datamodel/low/base/schema.go index ca49d3a..3bacfe1 100644 --- a/datamodel/low/base/schema.go +++ b/datamodel/low/base/schema.go @@ -1,6 +1,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "sort" @@ -490,13 +491,13 @@ func (s *Schema) GetExtensions() map[low.KeyReference[string]]low.ValueReference // - UnevaluatedItems // - UnevaluatedProperties // - Anchor -func (s *Schema) Build(root *yaml.Node, idx *index.SpecIndex) error { +func (s *Schema) Build(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) s.Reference = new(low.Reference) s.Index = idx if h, _, _ := utils.IsNodeRefValue(root); h { - ref, err := low.LocateRefNode(root, idx) + ref, _, err := low.LocateRefNode(root, idx) if ref != nil { root = ref if err != nil { @@ -704,7 +705,7 @@ func (s *Schema) Build(root *yaml.Node, idx *index.SpecIndex) error { if extDocNode != nil { var exDoc ExternalDoc _ = low.BuildModel(extDocNode, &exDoc) - _ = exDoc.Build(extDocLabel, extDocNode, idx) // throws no errors, can't check for one. + _ = exDoc.Build(ctx, extDocLabel, extDocNode, idx) // throws no errors, can't check for one. s.ExternalDocs = low.NodeReference[*ExternalDoc]{Value: &exDoc, KeyNode: extDocLabel, ValueNode: extDocNode} } @@ -1069,7 +1070,7 @@ func buildPropertyMap(root *yaml.Node, idx *index.SpecIndex, label string) (*low isRef := false refString := "" if h, _, l := utils.IsNodeRefValue(prop); h { - ref, _ := low.LocateRefNode(prop, idx) + ref, _, _ := low.LocateRefNode(prop, idx) if ref != nil { isRef = true prop = ref @@ -1165,7 +1166,7 @@ func buildSchema(schemas chan schemaProxyBuildResult, labelNode, valueNode *yaml h := false if h, _, refLocation = utils.IsNodeRefValue(valueNode); h { isRef = true - ref, _ := low.LocateRefNode(valueNode, idx) + ref, _, _ := low.LocateRefNode(valueNode, idx) if ref != nil { valueNode = ref } else { @@ -1196,7 +1197,7 @@ func buildSchema(schemas chan schemaProxyBuildResult, labelNode, valueNode *yaml h := false if h, _, refLocation = utils.IsNodeRefValue(vn); h { isRef = true - ref, _ := low.LocateRefNode(vn, idx) + ref, _, _ := low.LocateRefNode(vn, idx) if ref != nil { vn = ref } else { @@ -1237,16 +1238,17 @@ func buildSchema(schemas chan schemaProxyBuildResult, labelNode, valueNode *yaml // ExtractSchema will return a pointer to a NodeReference that contains a *SchemaProxy if successful. The function // will specifically look for a key node named 'schema' and extract the value mapped to that key. If the operation // fails then no NodeReference is returned and an error is returned instead. -func ExtractSchema(root *yaml.Node, idx *index.SpecIndex) (*low.NodeReference[*SchemaProxy], error) { +func ExtractSchema(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) (*low.NodeReference[*SchemaProxy], error) { var schLabel, schNode *yaml.Node errStr := "schema build failed: reference '%s' cannot be found at line %d, col %d" isRef := false refLocation := "" + if rf, rl, _ := utils.IsNodeRefValue(root); rf { // locate reference in index. isRef = true - ref, _ := low.LocateRefNode(root, idx) + ref, _, _ := low.LocateRefNode(root, idx) if ref != nil { schNode = ref schLabel = rl @@ -1260,9 +1262,13 @@ func ExtractSchema(root *yaml.Node, idx *index.SpecIndex) (*low.NodeReference[*S h := false if h, _, refLocation = utils.IsNodeRefValue(schNode); h { isRef = true - ref, _ := low.LocateRefNode(schNode, idx) + ref, foundIdx, _, nCtx := low.LocateRefNodeWithContext(ctx, schNode, idx) if ref != nil { schNode = ref + if foundIdx != nil { + //idx = foundIdx + } + ctx = nCtx } else { return nil, fmt.Errorf(errStr, schNode.Content[1].Value, schNode.Content[1].Line, schNode.Content[1].Column) @@ -1273,7 +1279,7 @@ func ExtractSchema(root *yaml.Node, idx *index.SpecIndex) (*low.NodeReference[*S if schNode != nil { // check if schema has already been built. - schema := &SchemaProxy{kn: schLabel, vn: schNode, idx: idx, isReference: isRef, referenceLookup: refLocation} + schema := &SchemaProxy{kn: schLabel, vn: schNode, idx: idx, ctx: ctx, isReference: isRef, referenceLookup: refLocation} return &low.NodeReference[*SchemaProxy]{ Value: schema, KeyNode: schLabel, ValueNode: schNode, ReferenceNode: isRef, Reference: refLocation, diff --git a/datamodel/low/base/schema_proxy.go b/datamodel/low/base/schema_proxy.go index 3e4c727..36d77fd 100644 --- a/datamodel/low/base/schema_proxy.go +++ b/datamodel/low/base/schema_proxy.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "github.com/pb33f/libopenapi/index" @@ -51,14 +52,16 @@ type SchemaProxy struct { buildError error isReference bool // Is the schema underneath originally a $ref? referenceLookup string // If the schema is a $ref, what's its name? + ctx context.Context } // Build will prepare the SchemaProxy for rendering, it does not build the Schema, only sets up internal state. // Key maybe nil if absent. -func (sp *SchemaProxy) Build(key, value *yaml.Node, idx *index.SpecIndex) error { +func (sp *SchemaProxy) Build(ctx context.Context, key, value *yaml.Node, idx *index.SpecIndex) error { sp.kn = key sp.vn = value sp.idx = idx + sp.ctx = ctx if rf, _, r := utils.IsNodeRefValue(value); rf { sp.isReference = true sp.referenceLookup = r @@ -83,7 +86,7 @@ func (sp *SchemaProxy) Schema() *Schema { } schema := new(Schema) utils.CheckForMergeNodes(sp.vn) - err := schema.Build(sp.vn, sp.idx) + err := schema.Build(sp.ctx, sp.vn, sp.idx) if err != nil { sp.buildError = err return nil diff --git a/datamodel/low/base/security_requirement.go b/datamodel/low/base/security_requirement.go index 559b091..910211e 100644 --- a/datamodel/low/base/security_requirement.go +++ b/datamodel/low/base/security_requirement.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -28,7 +29,7 @@ type SecurityRequirement struct { } // Build will extract security requirements from the node (the structure is odd, to be honest) -func (s *SecurityRequirement) Build(_, root *yaml.Node, _ *index.SpecIndex) error { +func (s *SecurityRequirement) Build(_ context.Context, _, root *yaml.Node, _ *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) s.Reference = new(low.Reference) diff --git a/datamodel/low/base/tag.go b/datamodel/low/base/tag.go index bb702ae..a6ead1a 100644 --- a/datamodel/low/base/tag.go +++ b/datamodel/low/base/tag.go @@ -4,6 +4,7 @@ package base import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -34,14 +35,14 @@ func (t *Tag) FindExtension(ext string) *low.ValueReference[any] { } // Build will extract extensions and external docs for the Tag. -func (t *Tag) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (t *Tag) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) t.Reference = new(low.Reference) t.Extensions = low.ExtractExtensions(root) // extract externalDocs - extDocs, err := low.ExtractObject[*ExternalDoc](ExternalDocsLabel, root, idx) + extDocs, err := low.ExtractObject[*ExternalDoc](ctx, ExternalDocsLabel, root, idx) t.ExternalDocs = extDocs return err } diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index 75794b7..99fd00e 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -4,357 +4,470 @@ package low import ( - "crypto/sha256" - "fmt" - "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/utils" - "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" - "gopkg.in/yaml.v3" - "reflect" - "strconv" - "strings" + "context" + "crypto/sha256" + "fmt" + "github.com/pb33f/libopenapi/index" + "github.com/pb33f/libopenapi/utils" + "github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath" + "gopkg.in/yaml.v3" + "net/url" + "path/filepath" + "reflect" + "strconv" + "strings" ) // FindItemInMap accepts a string key and a collection of KeyReference[string] and ValueReference[T]. Every // KeyReference will have its value checked against the string key and if there is a match, it will be returned. func FindItemInMap[T any](item string, collection map[KeyReference[string]]ValueReference[T]) *ValueReference[T] { - for n, o := range collection { - if n.Value == item { - return &o - } - if strings.EqualFold(item, n.Value) { - return &o - } - } - return nil + for n, o := range collection { + if n.Value == item { + return &o + } + if strings.EqualFold(item, n.Value) { + return &o + } + } + return nil } // helper function to generate a list of all the things an index should be searched for. func generateIndexCollection(idx *index.SpecIndex) []func() map[string]*index.Reference { - return []func() map[string]*index.Reference{ - idx.GetAllComponentSchemas, - idx.GetMappedReferences, - idx.GetAllExternalDocuments, - idx.GetAllParameters, - idx.GetAllHeaders, - idx.GetAllCallbacks, - idx.GetAllLinks, - idx.GetAllExamples, - idx.GetAllRequestBodies, - idx.GetAllResponses, - idx.GetAllSecuritySchemes, - } + return []func() map[string]*index.Reference{ + idx.GetAllComponentSchemas, + idx.GetMappedReferences, + idx.GetAllExternalDocuments, + idx.GetAllParameters, + idx.GetAllHeaders, + idx.GetAllCallbacks, + idx.GetAllLinks, + idx.GetAllExamples, + idx.GetAllRequestBodies, + idx.GetAllResponses, + idx.GetAllSecuritySchemes, + } +} + +func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, *index.SpecIndex, error, context.Context) { + + if rf, _, rv := utils.IsNodeRefValue(root); rf { + + if rv == "" { + return nil, nil, fmt.Errorf("reference at line %d, column %d is empty, it cannot be resolved", + root.Line, root.Column), ctx + } + + // run through everything and return as soon as we find a match. + // this operates as fast as possible as ever + collections := generateIndexCollection(idx) + + // if there are any external indexes being used by remote + // documents, then we need to search through them also. + //externalIndexes := idx.GetAllExternalIndexes() + //if len(externalIndexes) > 0 { + // var extCollection []func() map[string]*index.Reference + // for _, extIndex := range externalIndexes { + // extCollection = generateIndexCollection(extIndex) + // collections = append(collections, extCollection...) + // } + //} + + var found map[string]*index.Reference + for _, collection := range collections { + found = collection() + if found != nil && found[rv] != nil { + + // if this is a ref node, we need to keep diving + // until we hit something that isn't a ref. + if jh, _, _ := utils.IsNodeRefValue(found[rv].Node); jh { + // if this node is circular, stop drop and roll. + if !IsCircular(found[rv].Node, idx) { + return LocateRefNodeWithContext(ctx, found[rv].Node, idx) + } else { + return found[rv].Node, idx, fmt.Errorf("circular reference '%s' found during lookup at line "+ + "%d, column %d, It cannot be resolved", + GetCircularReferenceResult(found[rv].Node, idx).GenerateJourneyPath(), + found[rv].Node.Line, + found[rv].Node.Column), ctx + } + } + return utils.NodeAlias(found[rv].Node), idx, nil, ctx + } + } + + // perform a search for the reference in the index + // extract the correct root + + specPath := idx.GetSpecAbsolutePath() + if ctx.Value("currentPath") != nil { + specPath = ctx.Value("currentPath").(string) + } + + explodedRefValue := strings.Split(rv, "#") + if len(explodedRefValue) == 2 { + if !strings.HasPrefix(explodedRefValue[0], "http") { + + if !filepath.IsAbs(explodedRefValue[0]) { + + if strings.HasPrefix(specPath, "http") { + u, _ := url.Parse(specPath) + p := filepath.Dir(u.Path) + abs, _ := filepath.Abs(filepath.Join(p, explodedRefValue[0])) + u.Path = abs + rv = fmt.Sprintf("%s#%s", u.String(), explodedRefValue[1]) + + } else { + if specPath != "" { + + abs, _ := filepath.Abs(filepath.Join(filepath.Dir(specPath), explodedRefValue[0])) + rv = fmt.Sprintf("%s#%s", abs, explodedRefValue[1]) + + } else { + + // check for a config baseURL and use that if it exists. + if idx.GetConfig().BaseURL != nil { + + u := *idx.GetConfig().BaseURL + + abs, _ := filepath.Abs(filepath.Join(u.Path, rv)) + u.Path = abs + rv = fmt.Sprintf("%s#%s", u.String(), explodedRefValue[1]) + } + + } + } + + } + } + } else { + + if !strings.HasPrefix(explodedRefValue[0], "http") { + + if !filepath.IsAbs(explodedRefValue[0]) { + + if strings.HasPrefix(specPath, "http") { + u, _ := url.Parse(specPath) + p := filepath.Dir(u.Path) + abs, _ := filepath.Abs(filepath.Join(p, rv)) + u.Path = abs + rv = u.String() + + } else { + if specPath != "" { + + abs, _ := filepath.Abs(filepath.Join(filepath.Dir(specPath), rv)) + rv = abs + + } else { + + // check for a config baseURL and use that if it exists. + if idx.GetConfig().BaseURL != nil { + u := *idx.GetConfig().BaseURL + + abs, _ := filepath.Abs(filepath.Join(u.Path, rv)) + u.Path = abs + rv = u.String() + } + + } + } + + } + } + + } + + foundRef, fIdx, newCtx := idx.SearchIndexForReferenceWithContext(ctx, rv) + if foundRef != nil { + return utils.NodeAlias(foundRef.Node), fIdx, nil, newCtx + } + + // let's try something else to find our references. + + // cant be found? last resort is to try a path lookup + _, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv) + if friendly != "" { + path, err := yamlpath.NewPath(friendly) + if err == nil { + nodes, fErr := path.Find(idx.GetRootNode()) + if fErr == nil { + if len(nodes) > 0 { + return utils.NodeAlias(nodes[0]), idx, nil, ctx + } + } + } + } + return nil, idx, fmt.Errorf("reference '%s' at line %d, column %d was not found", + rv, root.Line, root.Column), ctx + } + return nil, idx, nil, ctx + } // LocateRefNode will perform a complete lookup for a $ref node. This function searches the entire index for // the reference being supplied. If there is a match found, the reference *yaml.Node is returned. -func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) { - if rf, _, rv := utils.IsNodeRefValue(root); rf { - - // run through everything and return as soon as we find a match. - // this operates as fast as possible as ever - collections := generateIndexCollection(idx) - - // if there are any external indexes being used by remote - // documents, then we need to search through them also. - //externalIndexes := idx.GetAllExternalIndexes() - //if len(externalIndexes) > 0 { - // var extCollection []func() map[string]*index.Reference - // for _, extIndex := range externalIndexes { - // extCollection = generateIndexCollection(extIndex) - // collections = append(collections, extCollection...) - // } - //} - - var found map[string]*index.Reference - for _, collection := range collections { - found = collection() - if found != nil && found[rv] != nil { - - // if this is a ref node, we need to keep diving - // until we hit something that isn't a ref. - if jh, _, _ := utils.IsNodeRefValue(found[rv].Node); jh { - // if this node is circular, stop drop and roll. - if !IsCircular(found[rv].Node, idx) { - return LocateRefNode(found[rv].Node, idx) - } else { - return found[rv].Node, fmt.Errorf("circular reference '%s' found during lookup at line "+ - "%d, column %d, It cannot be resolved", - GetCircularReferenceResult(found[rv].Node, idx).GenerateJourneyPath(), - found[rv].Node.Line, - found[rv].Node.Column) - } - } - return utils.NodeAlias(found[rv].Node), nil - } - } - - // perform a search for the reference in the index - foundRef := idx.SearchIndexForReference(rv) - if foundRef != nil { - return utils.NodeAlias(foundRef.Node), nil - } - - // let's try something else to find our references. - - // cant be found? last resort is to try a path lookup - _, friendly := utils.ConvertComponentIdIntoFriendlyPathSearch(rv) - if friendly != "" { - path, err := yamlpath.NewPath(friendly) - if err == nil { - nodes, fErr := path.Find(idx.GetRootNode()) - if fErr == nil { - if len(nodes) > 0 { - return utils.NodeAlias(nodes[0]), nil - } - } - } - } - return nil, fmt.Errorf("reference '%s' at line %d, column %d was not found", - rv, root.Line, root.Column) - } - return nil, nil +func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, *index.SpecIndex, error) { + r, i, e, _ := LocateRefNodeWithContext(context.Background(), root, idx) + return r, i, e } // ExtractObjectRaw will extract a typed Buildable[N] object from a root yaml.Node. The 'raw' aspect is // that there is no NodeReference wrapper around the result returned, just the raw object. -func ExtractObjectRaw[T Buildable[N], N any](key, root *yaml.Node, idx *index.SpecIndex) (T, error, bool, string) { - var circError error - var isReference bool - var referenceValue string - root = utils.NodeAlias(root) - if h, _, rv := utils.IsNodeRefValue(root); h { - ref, err := LocateRefNode(root, idx) - if ref != nil { - root = ref - isReference = true - referenceValue = rv - if err != nil { - circError = err - } - } else { - if err != nil { - return nil, fmt.Errorf("object extraction failed: %s", err.Error()), isReference, referenceValue - } - } - } - var n T = new(N) - err := BuildModel(root, n) - if err != nil { - return n, err, isReference, referenceValue - } - err = n.Build(key, root, idx) - if err != nil { - return n, err, isReference, referenceValue - } +func ExtractObjectRaw[T Buildable[N], N any](ctx context.Context, key, root *yaml.Node, idx *index.SpecIndex) (T, error, bool, string) { + var circError error + var isReference bool + var referenceValue string + root = utils.NodeAlias(root) + if h, _, rv := utils.IsNodeRefValue(root); h { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, root, idx) + if ref != nil { + root = ref + isReference = true + referenceValue = rv + idx = fIdx + ctx = nCtx + if err != nil { + circError = err + } + } else { + if err != nil { + return nil, fmt.Errorf("object extraction failed: %s", err.Error()), isReference, referenceValue + } + } + } + var n T = new(N) + err := BuildModel(root, n) + if err != nil { + return n, err, isReference, referenceValue + } + err = n.Build(ctx, key, root, idx) + if err != nil { + return n, err, isReference, referenceValue + } - // if this is a reference, keep track of the reference in the value - if isReference { - SetReference(n, referenceValue) - } + // if this is a reference, keep track of the reference in the value + if isReference { + SetReference(n, referenceValue) + } - // do we want to throw an error as well if circular error reporting is on? - if circError != nil && !idx.AllowCircularReferenceResolving() { - return n, circError, isReference, referenceValue - } - return n, nil, isReference, referenceValue + // do we want to throw an error as well if circular error reporting is on? + if circError != nil && !idx.AllowCircularReferenceResolving() { + return n, circError, isReference, referenceValue + } + return n, nil, isReference, referenceValue } // ExtractObject will extract a typed Buildable[N] object from a root yaml.Node. The result is wrapped in a // NodeReference[T] that contains the key node found and value node found when looking up the reference. -func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (NodeReference[T], error) { - var ln, vn *yaml.Node - var circError error - var isReference bool - var referenceValue string - root = utils.NodeAlias(root) - if rf, rl, refVal := utils.IsNodeRefValue(root); rf { - ref, err := LocateRefNode(root, idx) - if ref != nil { - vn = ref - ln = rl - isReference = true - referenceValue = refVal - if err != nil { - circError = err - } - } else { - if err != nil { - return NodeReference[T]{}, fmt.Errorf("object extraction failed: %s", err.Error()) - } - } - } else { - _, ln, vn = utils.FindKeyNodeFull(label, root.Content) - if vn != nil { - if h, _, rVal := utils.IsNodeRefValue(vn); h { - ref, lerr := LocateRefNode(vn, idx) - if ref != nil { - vn = ref - isReference = true - referenceValue = rVal - if lerr != nil { - circError = lerr - } - } else { - if lerr != nil { - return NodeReference[T]{}, fmt.Errorf("object extraction failed: %s", lerr.Error()) - } - } - } - } - } - var n T = new(N) - err := BuildModel(vn, n) - if err != nil { - return NodeReference[T]{}, err - } - if ln == nil { - return NodeReference[T]{}, nil - } - err = n.Build(ln, vn, idx) - if err != nil { - return NodeReference[T]{}, err - } +func ExtractObject[T Buildable[N], N any](ctx context.Context, label string, root *yaml.Node, idx *index.SpecIndex) (NodeReference[T], error) { + var ln, vn *yaml.Node + var circError error + var isReference bool + var referenceValue string + root = utils.NodeAlias(root) + if rf, rl, refVal := utils.IsNodeRefValue(root); rf { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, root, idx) + if ref != nil { + vn = ref + ln = rl + isReference = true + referenceValue = refVal + idx = fIdx + ctx = nCtx + if err != nil { + circError = err + } + } else { + if err != nil { + return NodeReference[T]{}, fmt.Errorf("object extraction failed: %s", err.Error()) + } + } + } else { + _, ln, vn = utils.FindKeyNodeFull(label, root.Content) + if vn != nil { + if h, _, rVal := utils.IsNodeRefValue(vn); h { + ref, fIdx, lerr, nCtx := LocateRefNodeWithContext(ctx, vn, idx) + if ref != nil { + vn = ref + if fIdx != nil { + idx = fIdx + } + ctx = nCtx + isReference = true + referenceValue = rVal + if lerr != nil { + circError = lerr + } + } else { + if lerr != nil { + return NodeReference[T]{}, fmt.Errorf("object extraction failed: %s", lerr.Error()) + } + } + } + } + } + var n T = new(N) + err := BuildModel(vn, n) + if err != nil { + return NodeReference[T]{}, err + } + if ln == nil { + return NodeReference[T]{}, nil + } + err = n.Build(ctx, ln, vn, idx) + if err != nil { + return NodeReference[T]{}, err + } - // if this is a reference, keep track of the reference in the value - if isReference { - SetReference(n, referenceValue) - } + // if this is a reference, keep track of the reference in the value + if isReference { + SetReference(n, referenceValue) + } - res := NodeReference[T]{ - Value: n, - KeyNode: ln, - ValueNode: vn, - ReferenceNode: isReference, - Reference: referenceValue, - } + res := NodeReference[T]{ + Value: n, + KeyNode: ln, + ValueNode: vn, + ReferenceNode: isReference, + Reference: referenceValue, + } - // do we want to throw an error as well if circular error reporting is on? - if circError != nil && !idx.AllowCircularReferenceResolving() { - return res, circError - } - return res, nil + // do we want to throw an error as well if circular error reporting is on? + if circError != nil && !idx.AllowCircularReferenceResolving() { + return res, circError + } + return res, nil } func SetReference(obj any, ref string) { - if obj == nil { - return - } - if r, ok := obj.(IsReferenced); ok { - r.SetReference(ref) - } + if obj == nil { + return + } + if r, ok := obj.(IsReferenced); ok { + r.SetReference(ref) + } } // ExtractArray will extract a slice of []ValueReference[T] from a root yaml.Node that is defined as a sequence. // Used when the value being extracted is an array. -func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) ([]ValueReference[T], - *yaml.Node, *yaml.Node, error, +func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root *yaml.Node, idx *index.SpecIndex) ([]ValueReference[T], + *yaml.Node, *yaml.Node, error, ) { - var ln, vn *yaml.Node - var circError error - root = utils.NodeAlias(root) - if rf, rl, _ := utils.IsNodeRefValue(root); rf { - ref, err := LocateRefNode(root, idx) - if ref != nil { - vn = ref - ln = rl - if err != nil { - circError = err - } - } else { - return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", - root.Content[1].Value) - } - } else { - _, ln, vn = utils.FindKeyNodeFullTop(label, root.Content) - if vn != nil { - if h, _, _ := utils.IsNodeRefValue(vn); h { - ref, err := LocateRefNode(vn, idx) - if ref != nil { - vn = ref - //referenceValue = rVal - if err != nil { - circError = err - } - } else { - if err != nil { - return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", - err.Error()) - } - } - } - } - } + var ln, vn *yaml.Node + var circError error + root = utils.NodeAlias(root) + if rf, rl, _ := utils.IsNodeRefValue(root); rf { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, root, idx) + if ref != nil { + vn = ref + ln = rl + fIdx = fIdx + ctx = nCtx + if err != nil { + circError = err + } + } else { + return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", + root.Content[1].Value) + } + } else { + _, ln, vn = utils.FindKeyNodeFullTop(label, root.Content) + if vn != nil { + if h, _, _ := utils.IsNodeRefValue(vn); h { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, vn, idx) + if ref != nil { + vn = ref + idx = fIdx + ctx = nCtx + //referenceValue = rVal + if err != nil { + circError = err + } + } else { + if err != nil { + return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", + err.Error()) + } + } + } + } + } - var items []ValueReference[T] - if vn != nil && ln != nil { - if !utils.IsNodeArray(vn) { - return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed, input is not an array, line %d, column %d", vn.Line, vn.Column) - } - for _, node := range vn.Content { - localReferenceValue := "" - //localIsReference := false + var items []ValueReference[T] + if vn != nil && ln != nil { + if !utils.IsNodeArray(vn) { + return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed, input is not an array, line %d, column %d", vn.Line, vn.Column) + } + for _, node := range vn.Content { + localReferenceValue := "" + //localIsReference := false - if rf, _, rv := utils.IsNodeRefValue(node); rf { - refg, err := LocateRefNode(node, idx) - if refg != nil { - node = refg - //localIsReference = true - localReferenceValue = rv - if err != nil { - circError = err - } - } else { - if err != nil { - return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", - err.Error()) - } - } - } - var n T = new(N) - err := BuildModel(node, n) - if err != nil { - return []ValueReference[T]{}, ln, vn, err - } - berr := n.Build(ln, node, idx) - if berr != nil { - return nil, ln, vn, berr - } + foundCtx := ctx + foundIndex := idx - if localReferenceValue != "" { - SetReference(n, localReferenceValue) - } + if rf, _, rv := utils.IsNodeRefValue(node); rf { + refg, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, node, idx) + if refg != nil { + node = refg + //localIsReference = true + localReferenceValue = rv + foundIndex = fIdx + foundCtx = nCtx + if err != nil { + circError = err + } + } else { + if err != nil { + return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", + err.Error()) + } + } + } + var n T = new(N) + err := BuildModel(node, n) + if err != nil { + return []ValueReference[T]{}, ln, vn, err + } + berr := n.Build(foundCtx, ln, node, foundIndex) + if berr != nil { + return nil, ln, vn, berr + } - items = append(items, ValueReference[T]{ - Value: n, - ValueNode: node, - ReferenceNode: localReferenceValue != "", - Reference: localReferenceValue, - }) - } - } - // include circular errors? - if circError != nil && !idx.AllowCircularReferenceResolving() { - return items, ln, vn, circError - } - return items, ln, vn, nil + if localReferenceValue != "" { + SetReference(n, localReferenceValue) + } + + items = append(items, ValueReference[T]{ + Value: n, + ValueNode: node, + ReferenceNode: localReferenceValue != "", + Reference: localReferenceValue, + }) + } + } + // include circular errors? + if circError != nil && !idx.AllowCircularReferenceResolving() { + return items, ln, vn, circError + } + return items, ln, vn, nil } // ExtractExample will extract a value supplied as an example into a NodeReference. Value can be anything. // the node value is untyped, so casting will be required when trying to use it. func ExtractExample(expNode, expLabel *yaml.Node) NodeReference[any] { - ref := NodeReference[any]{Value: expNode.Value, KeyNode: expLabel, ValueNode: expNode} - if utils.IsNodeMap(expNode) { - var decoded map[string]interface{} - _ = expNode.Decode(&decoded) - ref.Value = decoded - } - if utils.IsNodeArray(expNode) { - var decoded []interface{} - _ = expNode.Decode(&decoded) - ref.Value = decoded - } - return ref + ref := NodeReference[any]{Value: expNode.Value, KeyNode: expLabel, ValueNode: expNode} + if utils.IsNodeMap(expNode) { + var decoded map[string]interface{} + _ = expNode.Decode(&decoded) + ref.Value = decoded + } + if utils.IsNodeArray(expNode) { + var decoded []interface{} + _ = expNode.Decode(&decoded) + ref.Value = decoded + } + return ref } // ExtractMapNoLookupExtensions will extract a map of KeyReference and ValueReference from a root yaml.Node. The 'NoLookup' part @@ -363,90 +476,101 @@ func ExtractExample(expNode, expLabel *yaml.Node) NodeReference[any] { // // This is useful when the node to be extracted, is already known and does not require a search. func ExtractMapNoLookupExtensions[PT Buildable[N], N any]( - root *yaml.Node, - idx *index.SpecIndex, - includeExtensions bool, + ctx context.Context, + root *yaml.Node, + idx *index.SpecIndex, + includeExtensions bool, ) (map[KeyReference[string]]ValueReference[PT], error) { - valueMap := make(map[KeyReference[string]]ValueReference[PT]) - var circError error - if utils.IsNodeMap(root) { - var currentKey *yaml.Node - skip := false - rlen := len(root.Content) + valueMap := make(map[KeyReference[string]]ValueReference[PT]) + var circError error + if utils.IsNodeMap(root) { + var currentKey *yaml.Node + skip := false + rlen := len(root.Content) - for i := 0; i < rlen; i++ { - node := root.Content[i] - if !includeExtensions { - if strings.HasPrefix(strings.ToLower(node.Value), "x-") { - skip = true - continue - } - } - if skip { - skip = false - continue - } - if i%2 == 0 { - currentKey = node - continue - } + for i := 0; i < rlen; i++ { + node := root.Content[i] + if !includeExtensions { + if strings.HasPrefix(strings.ToLower(node.Value), "x-") { + skip = true + continue + } + } + if skip { + skip = false + continue + } + if i%2 == 0 { + currentKey = node + continue + } - if currentKey.Tag == "!!merge" && currentKey.Value == "<<" { - root.Content = append(root.Content, utils.NodeAlias(node).Content...) - rlen = len(root.Content) - currentKey = nil - continue - } - node = utils.NodeAlias(node) + if currentKey.Tag == "!!merge" && currentKey.Value == "<<" { + root.Content = append(root.Content, utils.NodeAlias(node).Content...) + rlen = len(root.Content) + currentKey = nil + continue + } + node = utils.NodeAlias(node) - var isReference bool - var referenceValue string - // if value is a reference, we have to look it up in the index! - if h, _, rv := utils.IsNodeRefValue(node); h { - ref, err := LocateRefNode(node, idx) - if ref != nil { - node = ref - isReference = true - referenceValue = rv - if err != nil { - circError = err - } - } else { - if err != nil { - return nil, fmt.Errorf("map build failed: reference cannot be found: %s", err.Error()) - } - } - } + foundIndex := idx + foundContext := ctx - var n PT = new(N) - err := BuildModel(node, n) - if err != nil { - return nil, err - } - berr := n.Build(currentKey, node, idx) - if berr != nil { - return nil, berr - } - if isReference { - SetReference(n, referenceValue) - } - if currentKey != nil { - valueMap[KeyReference[string]{ - Value: currentKey.Value, - KeyNode: currentKey, - }] = ValueReference[PT]{ - Value: n, - ValueNode: node, - //IsReference: isReference, - Reference: referenceValue, - } - } - } - } - if circError != nil && !idx.AllowCircularReferenceResolving() { - return valueMap, circError - } - return valueMap, nil + var isReference bool + var referenceValue string + // if value is a reference, we have to look it up in the index! + if h, _, rv := utils.IsNodeRefValue(node); h { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, node, idx) + if ref != nil { + node = ref + isReference = true + referenceValue = rv + if fIdx != nil { + foundIndex = fIdx + } + foundContext = nCtx + if err != nil { + circError = err + } + } else { + if err != nil { + return nil, fmt.Errorf("map build failed: reference cannot be found: %s", err.Error()) + } + } + } + if foundIndex == nil { + foundIndex = idx + } + + var n PT = new(N) + err := BuildModel(node, n) + if err != nil { + return nil, err + } + berr := n.Build(foundContext, currentKey, node, foundIndex) + if berr != nil { + return nil, berr + } + if isReference { + SetReference(n, referenceValue) + } + if currentKey != nil { + valueMap[KeyReference[string]{ + Value: currentKey.Value, + KeyNode: currentKey, + }] = ValueReference[PT]{ + Value: n, + ValueNode: node, + //IsReference: isReference, + Reference: referenceValue, + } + } + } + } + if circError != nil && !idx.AllowCircularReferenceResolving() { + return valueMap, circError + } + return valueMap, nil } @@ -456,15 +580,16 @@ func ExtractMapNoLookupExtensions[PT Buildable[N], N any]( // // This is useful when the node to be extracted, is already known and does not require a search. func ExtractMapNoLookup[PT Buildable[N], N any]( - root *yaml.Node, - idx *index.SpecIndex, + ctx context.Context, + root *yaml.Node, + idx *index.SpecIndex, ) (map[KeyReference[string]]ValueReference[PT], error) { - return ExtractMapNoLookupExtensions[PT, N](root, idx, false) + return ExtractMapNoLookupExtensions[PT, N](ctx, root, idx, false) } type mappingResult[T any] struct { - k KeyReference[string] - v ValueReference[T] + k KeyReference[string] + v ValueReference[T] } // ExtractMapExtensions will extract a map of KeyReference and ValueReference from a root yaml.Node. The 'label' is @@ -474,140 +599,151 @@ type mappingResult[T any] struct { // The second return value is the yaml.Node found for the 'label' and the third return value is the yaml.Node // found for the value extracted from the label node. func ExtractMapExtensions[PT Buildable[N], N any]( - label string, - root *yaml.Node, - idx *index.SpecIndex, - extensions bool, + ctx context.Context, + label string, + root *yaml.Node, + idx *index.SpecIndex, + extensions bool, ) (map[KeyReference[string]]ValueReference[PT], *yaml.Node, *yaml.Node, error) { - //var isReference bool - var referenceValue string - var labelNode, valueNode *yaml.Node - var circError error - root = utils.NodeAlias(root) - if rf, rl, rv := utils.IsNodeRefValue(root); rf { - // locate reference in index. - ref, err := LocateRefNode(root, idx) - if ref != nil { - valueNode = ref - labelNode = rl - //isReference = true - referenceValue = rv - if err != nil { - circError = err - } - } else { - return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s", - root.Content[1].Value) - } - } else { - _, labelNode, valueNode = utils.FindKeyNodeFull(label, root.Content) - valueNode = utils.NodeAlias(valueNode) - if valueNode != nil { - if h, _, rvt := utils.IsNodeRefValue(valueNode); h { - ref, err := LocateRefNode(valueNode, idx) - if ref != nil { - valueNode = ref - //isReference = true - referenceValue = rvt - if err != nil { - circError = err - } - } else { - if err != nil { - return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s", - err.Error()) - } - } - } - } - } - if valueNode != nil { - var currentLabelNode *yaml.Node - valueMap := make(map[KeyReference[string]]ValueReference[PT]) + //var isReference bool + var referenceValue string + var labelNode, valueNode *yaml.Node + var circError error + root = utils.NodeAlias(root) + if rf, rl, rv := utils.IsNodeRefValue(root); rf { + // locate reference in index. + ref, _, err := LocateRefNode(root, idx) + if ref != nil { + valueNode = ref + labelNode = rl + //isReference = true + referenceValue = rv + if err != nil { + circError = err + } + } else { + return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s", + root.Content[1].Value) + } + } else { + _, labelNode, valueNode = utils.FindKeyNodeFull(label, root.Content) + valueNode = utils.NodeAlias(valueNode) + if valueNode != nil { + if h, _, rvt := utils.IsNodeRefValue(valueNode); h { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, valueNode, idx) + if ref != nil { + valueNode = ref + //isReference = true + referenceValue = rvt + idx = fIdx + ctx = nCtx + if err != nil { + circError = err + } + } else { + if err != nil { + return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s", + err.Error()) + } + } + } + } + } + if valueNode != nil { + var currentLabelNode *yaml.Node + valueMap := make(map[KeyReference[string]]ValueReference[PT]) - bChan := make(chan mappingResult[PT]) - eChan := make(chan error) + bChan := make(chan mappingResult[PT]) + eChan := make(chan error) - buildMap := func(label *yaml.Node, value *yaml.Node, c chan mappingResult[PT], ec chan<- error, ref string) { - var n PT = new(N) - value = utils.NodeAlias(value) - _ = BuildModel(value, n) - err := n.Build(label, value, idx) - if err != nil { - ec <- err - return - } + buildMap := func(nctx context.Context, label *yaml.Node, value *yaml.Node, c chan mappingResult[PT], ec chan<- error, ref string, fIdx *index.SpecIndex) { + var n PT = new(N) + value = utils.NodeAlias(value) + _ = BuildModel(value, n) + err := n.Build(nctx, label, value, fIdx) + if err != nil { + ec <- err + return + } - //isRef := false - if ref != "" { - //isRef = true - SetReference(n, ref) - } + //isRef := false + if ref != "" { + //isRef = true + SetReference(n, ref) + } - c <- mappingResult[PT]{ - k: KeyReference[string]{ - KeyNode: label, - Value: label.Value, - }, - v: ValueReference[PT]{ - Value: n, - ValueNode: value, - //IsReference: isRef, - Reference: ref, - }, - } - } + c <- mappingResult[PT]{ + k: KeyReference[string]{ + KeyNode: label, + Value: label.Value, + }, + v: ValueReference[PT]{ + Value: n, + ValueNode: value, + //IsReference: isRef, + Reference: ref, + }, + } + } - totalKeys := 0 - for i, en := range valueNode.Content { - en = utils.NodeAlias(en) - referenceValue = "" - if i%2 == 0 { - currentLabelNode = en - continue - } - // check our valueNode isn't a reference still. - if h, _, refVal := utils.IsNodeRefValue(en); h { - ref, err := LocateRefNode(en, idx) - if ref != nil { - en = ref - referenceValue = refVal - if err != nil { - circError = err - } - } else { - if err != nil { - return nil, labelNode, valueNode, fmt.Errorf("flat map build failed: reference cannot be found: %s", - err.Error()) - } - } - } + totalKeys := 0 + for i, en := range valueNode.Content { + en = utils.NodeAlias(en) + referenceValue = "" + if i%2 == 0 { + currentLabelNode = en + continue + } - if !extensions { - if strings.HasPrefix(currentLabelNode.Value, "x-") { - continue // yo, don't pay any attention to extensions, not here anyway. - } - } - totalKeys++ - go buildMap(currentLabelNode, en, bChan, eChan, referenceValue) - } + foundIndex := idx + foundContext := ctx - completedKeys := 0 - for completedKeys < totalKeys { - select { - case err := <-eChan: - return valueMap, labelNode, valueNode, err - case res := <-bChan: - completedKeys++ - valueMap[res.k] = res.v - } - } - if circError != nil && !idx.AllowCircularReferenceResolving() { - return valueMap, labelNode, valueNode, circError - } - return valueMap, labelNode, valueNode, nil - } - return nil, labelNode, valueNode, nil + // check our valueNode isn't a reference still. + if h, _, refVal := utils.IsNodeRefValue(en); h { + ref, fIdx, err, nCtx := LocateRefNodeWithContext(ctx, en, idx) + if ref != nil { + en = ref + referenceValue = refVal + if fIdx != nil { + foundIndex = fIdx + } + foundContext = nCtx + if err != nil { + circError = err + } + } else { + if err != nil { + return nil, labelNode, valueNode, fmt.Errorf("flat map build failed: reference cannot be found: %s", + err.Error()) + } + } + } + + if !extensions { + if strings.HasPrefix(currentLabelNode.Value, "x-") { + continue // yo, don't pay any attention to extensions, not here anyway. + } + } + totalKeys++ + go buildMap(foundContext, currentLabelNode, en, bChan, eChan, referenceValue, foundIndex) + } + + completedKeys := 0 + for completedKeys < totalKeys { + select { + case err := <-eChan: + return valueMap, labelNode, valueNode, err + case res := <-bChan: + completedKeys++ + valueMap[res.k] = res.v + } + } + if circError != nil && !idx.AllowCircularReferenceResolving() { + return valueMap, labelNode, valueNode, circError + } + return valueMap, labelNode, valueNode, nil + } + return nil, labelNode, valueNode, nil } // ExtractMap will extract a map of KeyReference and ValueReference from a root yaml.Node. The 'label' is @@ -616,11 +752,12 @@ func ExtractMapExtensions[PT Buildable[N], N any]( // The second return value is the yaml.Node found for the 'label' and the third return value is the yaml.Node // found for the value extracted from the label node. func ExtractMap[PT Buildable[N], N any]( - label string, - root *yaml.Node, - idx *index.SpecIndex, + ctx context.Context, + label string, + root *yaml.Node, + idx *index.SpecIndex, ) (map[KeyReference[string]]ValueReference[PT], *yaml.Node, *yaml.Node, error) { - return ExtractMapExtensions[PT, N](label, root, idx, false) + return ExtractMapExtensions[PT, N](ctx, label, root, idx, false) } // ExtractExtensions will extract any 'x-' prefixed key nodes from a root node into a map. Requirements have been pre-cast: @@ -637,79 +774,79 @@ func ExtractMap[PT Buildable[N], N any]( // // int64, float64, bool, string func ExtractExtensions(root *yaml.Node) map[KeyReference[string]]ValueReference[any] { - root = utils.NodeAlias(root) - extensions := utils.FindExtensionNodes(root.Content) - extensionMap := make(map[KeyReference[string]]ValueReference[any]) - for _, ext := range extensions { - if utils.IsNodeMap(ext.Value) { - var v interface{} - _ = ext.Value.Decode(&v) - extensionMap[KeyReference[string]{ - Value: ext.Key.Value, - KeyNode: ext.Key, - }] = ValueReference[any]{Value: v, ValueNode: ext.Value} - } - if utils.IsNodeStringValue(ext.Value) { - extensionMap[KeyReference[string]{ - Value: ext.Key.Value, - KeyNode: ext.Key, - }] = ValueReference[any]{Value: ext.Value.Value, ValueNode: ext.Value} - } - if utils.IsNodeFloatValue(ext.Value) { - fv, _ := strconv.ParseFloat(ext.Value.Value, 64) - extensionMap[KeyReference[string]{ - Value: ext.Key.Value, - KeyNode: ext.Key, - }] = ValueReference[any]{Value: fv, ValueNode: ext.Value} - } - if utils.IsNodeIntValue(ext.Value) { - iv, _ := strconv.ParseInt(ext.Value.Value, 10, 64) - extensionMap[KeyReference[string]{ - Value: ext.Key.Value, - KeyNode: ext.Key, - }] = ValueReference[any]{Value: iv, ValueNode: ext.Value} - } - if utils.IsNodeBoolValue(ext.Value) { - bv, _ := strconv.ParseBool(ext.Value.Value) - extensionMap[KeyReference[string]{ - Value: ext.Key.Value, - KeyNode: ext.Key, - }] = ValueReference[any]{Value: bv, ValueNode: ext.Value} - } - if utils.IsNodeArray(ext.Value) { - var v []interface{} - _ = ext.Value.Decode(&v) - extensionMap[KeyReference[string]{ - Value: ext.Key.Value, - KeyNode: ext.Key, - }] = ValueReference[any]{Value: v, ValueNode: ext.Value} - } - } - return extensionMap + root = utils.NodeAlias(root) + extensions := utils.FindExtensionNodes(root.Content) + extensionMap := make(map[KeyReference[string]]ValueReference[any]) + for _, ext := range extensions { + if utils.IsNodeMap(ext.Value) { + var v interface{} + _ = ext.Value.Decode(&v) + extensionMap[KeyReference[string]{ + Value: ext.Key.Value, + KeyNode: ext.Key, + }] = ValueReference[any]{Value: v, ValueNode: ext.Value} + } + if utils.IsNodeStringValue(ext.Value) { + extensionMap[KeyReference[string]{ + Value: ext.Key.Value, + KeyNode: ext.Key, + }] = ValueReference[any]{Value: ext.Value.Value, ValueNode: ext.Value} + } + if utils.IsNodeFloatValue(ext.Value) { + fv, _ := strconv.ParseFloat(ext.Value.Value, 64) + extensionMap[KeyReference[string]{ + Value: ext.Key.Value, + KeyNode: ext.Key, + }] = ValueReference[any]{Value: fv, ValueNode: ext.Value} + } + if utils.IsNodeIntValue(ext.Value) { + iv, _ := strconv.ParseInt(ext.Value.Value, 10, 64) + extensionMap[KeyReference[string]{ + Value: ext.Key.Value, + KeyNode: ext.Key, + }] = ValueReference[any]{Value: iv, ValueNode: ext.Value} + } + if utils.IsNodeBoolValue(ext.Value) { + bv, _ := strconv.ParseBool(ext.Value.Value) + extensionMap[KeyReference[string]{ + Value: ext.Key.Value, + KeyNode: ext.Key, + }] = ValueReference[any]{Value: bv, ValueNode: ext.Value} + } + if utils.IsNodeArray(ext.Value) { + var v []interface{} + _ = ext.Value.Decode(&v) + extensionMap[KeyReference[string]{ + Value: ext.Key.Value, + KeyNode: ext.Key, + }] = ValueReference[any]{Value: v, ValueNode: ext.Value} + } + } + return extensionMap } // AreEqual returns true if two Hashable objects are equal or not. func AreEqual(l, r Hashable) bool { - if l == nil || r == nil { - return false - } - return l.Hash() == r.Hash() + if l == nil || r == nil { + return false + } + return l.Hash() == r.Hash() } // GenerateHashString will generate a SHA36 hash of any object passed in. If the object is Hashable // then the underlying Hash() method will be called. func GenerateHashString(v any) string { - if v == nil { - return "" - } - if h, ok := v.(Hashable); ok { - if h != nil { - return fmt.Sprintf(HASH, h.Hash()) - } - } - // if we get here, we're a primitive, check if we're a pointer and de-point - if reflect.TypeOf(v).Kind() == reflect.Ptr { - v = reflect.ValueOf(v).Elem().Interface() - } - return fmt.Sprintf(HASH, sha256.Sum256([]byte(fmt.Sprint(v)))) + if v == nil { + return "" + } + if h, ok := v.(Hashable); ok { + if h != nil { + return fmt.Sprintf(HASH, h.Hash()) + } + } + // if we get here, we're a primitive, check if we're a pointer and de-point + if reflect.TypeOf(v).Kind() == reflect.Ptr { + v = reflect.ValueOf(v).Elem().Interface() + } + return fmt.Sprintf(HASH, sha256.Sum256([]byte(fmt.Sprint(v)))) } diff --git a/datamodel/low/extraction_functions_test.go b/datamodel/low/extraction_functions_test.go index 2b6670b..f87c6cf 100644 --- a/datamodel/low/extraction_functions_test.go +++ b/datamodel/low/extraction_functions_test.go @@ -4,15 +4,14 @@ package low import ( - "crypto/sha256" - "fmt" - "os" - "strings" - "testing" + "crypto/sha256" + "fmt" + "strings" + "testing" - "github.com/pb33f/libopenapi/index" - "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" + "github.com/pb33f/libopenapi/index" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestFindItemInMap(t *testing.T) { @@ -1570,38 +1569,6 @@ func TestExtractMapFlat_Ref_Bad(t *testing.T) { assert.Len(t, things, 0) } -func TestLocateRefNode_RemoteFile(t *testing.T) { - - ymlFile := fmt.Sprintf(`components: - schemas: - hey: - $ref: '%s#/components/schemas/hey'`, "remote.yaml") - - ymlRemote := `components: - schemas: - hey: - AlmostWork: 999` - - _ = os.WriteFile("remote.yaml", []byte(ymlRemote), 0665) - defer os.Remove("remote.yaml") - - ymlLocal := `$ref: '#/components/schemas/hey'` - - var idxNode yaml.Node - mErr := yaml.Unmarshal([]byte(ymlFile), &idxNode) // an empty index. - assert.NoError(t, mErr) - idx := index.NewSpecIndexWithConfig(&idxNode, index.CreateOpenAPIIndexConfig()) - - var cNode yaml.Node - e := yaml.Unmarshal([]byte(ymlLocal), &cNode) - assert.NoError(t, e) - - things, _, _, err := ExtractMap[*test_Good]("one", cNode.Content[0], idx) - assert.NoError(t, err) - assert.Len(t, things, 1) - -} - func TestExtractExtensions(t *testing.T) { yml := `x-bing: ding diff --git a/datamodel/low/reference.go b/datamodel/low/reference.go index da45df2..34b6091 100644 --- a/datamodel/low/reference.go +++ b/datamodel/low/reference.go @@ -1,6 +1,7 @@ package low import ( + "context" "fmt" "github.com/pb33f/libopenapi/index" "github.com/pb33f/libopenapi/utils" @@ -38,7 +39,7 @@ type IsReferenced interface { // // Used by generic functions when automatically building out structs based on yaml.Node inputs. type Buildable[T any] interface { - Build(key, value *yaml.Node, idx *index.SpecIndex) error + Build(ctx context.Context, key, value *yaml.Node, idx *index.SpecIndex) error *T } @@ -112,6 +113,8 @@ type NodeReference[T any] struct { // If HasReference is true, then Reference contains the original $ref value. Reference string + + Context context.Context } // KeyReference is a low-level container for key nodes holding a Value of type T. A KeyNode is a pointer to the diff --git a/datamodel/low/v2/definitions.go b/datamodel/low/v2/definitions.go index d659936..316adea 100644 --- a/datamodel/low/v2/definitions.go +++ b/datamodel/low/v2/definitions.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/datamodel/low/base" @@ -71,7 +72,7 @@ func (s *SecurityDefinitions) FindSecurityDefinition(securityDef string) *low.Va } // Build will extract all definitions into SchemaProxy instances. -func (d *Definitions) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (d *Definitions) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) errorChan := make(chan error) @@ -81,7 +82,7 @@ func (d *Definitions) Build(_, root *yaml.Node, idx *index.SpecIndex) error { var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex, r chan definitionResult[*base.SchemaProxy], e chan error) { - obj, err, _, rv := low.ExtractObjectRaw[*base.SchemaProxy](label, value, idx) + obj, err, _, rv := low.ExtractObjectRaw[*base.SchemaProxy](ctx, label, value, idx) if err != nil { e <- err } @@ -133,7 +134,7 @@ func (d *Definitions) Hash() [32]byte { } // Build will extract all ParameterDefinitions into Parameter instances. -func (pd *ParameterDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (pd *ParameterDefinitions) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { errorChan := make(chan error) resultChan := make(chan definitionResult[*Parameter]) var defLabel *yaml.Node @@ -141,7 +142,7 @@ func (pd *ParameterDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex, r chan definitionResult[*Parameter], e chan error) { - obj, err, _, rv := low.ExtractObjectRaw[*Parameter](label, value, idx) + obj, err, _, rv := low.ExtractObjectRaw[*Parameter](ctx, label, value, idx) if err != nil { e <- err } @@ -182,7 +183,7 @@ type definitionResult[T any] struct { } // Build will extract all ResponsesDefinitions into Response instances. -func (r *ResponsesDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (r *ResponsesDefinitions) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { errorChan := make(chan error) resultChan := make(chan definitionResult[*Response]) var defLabel *yaml.Node @@ -190,7 +191,7 @@ func (r *ResponsesDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) e var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex, r chan definitionResult[*Response], e chan error) { - obj, err, _, rv := low.ExtractObjectRaw[*Response](label, value, idx) + obj, err, _, rv := low.ExtractObjectRaw[*Response](ctx, label, value, idx) if err != nil { e <- err } @@ -225,7 +226,7 @@ func (r *ResponsesDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) e } // Build will extract all SecurityDefinitions into SecurityScheme instances. -func (s *SecurityDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (s *SecurityDefinitions) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { errorChan := make(chan error) resultChan := make(chan definitionResult[*SecurityScheme]) var defLabel *yaml.Node @@ -234,7 +235,7 @@ func (s *SecurityDefinitions) Build(_, root *yaml.Node, idx *index.SpecIndex) er var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex, r chan definitionResult[*SecurityScheme], e chan error) { - obj, err, _, rv := low.ExtractObjectRaw[*SecurityScheme](label, value, idx) + obj, err, _, rv := low.ExtractObjectRaw[*SecurityScheme](ctx, label, value, idx) if err != nil { e <- err } diff --git a/datamodel/low/v2/examples.go b/datamodel/low/v2/examples.go index 8bf77cc..ed9a9b0 100644 --- a/datamodel/low/v2/examples.go +++ b/datamodel/low/v2/examples.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -27,7 +28,7 @@ func (e *Examples) FindExample(name string) *low.ValueReference[any] { } // Build will extract all examples and will attempt to unmarshal content into a map or slice based on type. -func (e *Examples) Build(_, root *yaml.Node, _ *index.SpecIndex) error { +func (e *Examples) Build(_ context.Context, _, root *yaml.Node, _ *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) var keyNode, currNode *yaml.Node diff --git a/datamodel/low/v2/header.go b/datamodel/low/v2/header.go index 5bb96ce..dc493fc 100644 --- a/datamodel/low/v2/header.go +++ b/datamodel/low/v2/header.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -51,11 +52,11 @@ func (h *Header) GetExtensions() map[low.KeyReference[string]]low.ValueReference } // Build will build out items, extensions and default value from the supplied node. -func (h *Header) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (h *Header) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) h.Extensions = low.ExtractExtensions(root) - items, err := low.ExtractObject[*Items](ItemsLabel, root, idx) + items, err := low.ExtractObject[*Items](ctx, ItemsLabel, root, idx) if err != nil { return err } diff --git a/datamodel/low/v2/items.go b/datamodel/low/v2/items.go index 416e0d9..36036bc 100644 --- a/datamodel/low/v2/items.go +++ b/datamodel/low/v2/items.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -102,11 +103,11 @@ func (i *Items) Hash() [32]byte { } // Build will build out items and default value. -func (i *Items) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (i *Items) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) i.Extensions = low.ExtractExtensions(root) - items, iErr := low.ExtractObject[*Items](ItemsLabel, root, idx) + items, iErr := low.ExtractObject[*Items](ctx, ItemsLabel, root, idx) if iErr != nil { return iErr } diff --git a/datamodel/low/v2/operation.go b/datamodel/low/v2/operation.go index 17e8f4b..bf4a4c6 100644 --- a/datamodel/low/v2/operation.go +++ b/datamodel/low/v2/operation.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -36,20 +37,20 @@ type Operation struct { } // Build will extract external docs, extensions, parameters, responses and security requirements. -func (o *Operation) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (o *Operation) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) o.Extensions = low.ExtractExtensions(root) // extract externalDocs - extDocs, dErr := low.ExtractObject[*base.ExternalDoc](base.ExternalDocsLabel, root, idx) + extDocs, dErr := low.ExtractObject[*base.ExternalDoc](ctx, base.ExternalDocsLabel, root, idx) if dErr != nil { return dErr } o.ExternalDocs = extDocs // extract parameters - params, ln, vn, pErr := low.ExtractArray[*Parameter](ParametersLabel, root, idx) + params, ln, vn, pErr := low.ExtractArray[*Parameter](ctx, ParametersLabel, root, idx) if pErr != nil { return pErr } @@ -62,14 +63,14 @@ func (o *Operation) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // extract responses - respBody, respErr := low.ExtractObject[*Responses](ResponsesLabel, root, idx) + respBody, respErr := low.ExtractObject[*Responses](ctx, ResponsesLabel, root, idx) if respErr != nil { return respErr } o.Responses = respBody // extract security - sec, sln, svn, sErr := low.ExtractArray[*base.SecurityRequirement](SecurityLabel, root, idx) + sec, sln, svn, sErr := low.ExtractArray[*base.SecurityRequirement](ctx, SecurityLabel, root, idx) if sErr != nil { return sErr } diff --git a/datamodel/low/v2/parameter.go b/datamodel/low/v2/parameter.go index 2e9490f..96514ab 100644 --- a/datamodel/low/v2/parameter.go +++ b/datamodel/low/v2/parameter.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -94,18 +95,18 @@ func (p *Parameter) GetExtensions() map[low.KeyReference[string]]low.ValueRefere } // Build will extract out extensions, schema, items and default value -func (p *Parameter) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (p *Parameter) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) p.Extensions = low.ExtractExtensions(root) - sch, sErr := base.ExtractSchema(root, idx) + sch, sErr := base.ExtractSchema(ctx, root, idx) if sErr != nil { return sErr } if sch != nil { p.Schema = *sch } - items, iErr := low.ExtractObject[*Items](ItemsLabel, root, idx) + items, iErr := low.ExtractObject[*Items](ctx, ItemsLabel, root, idx) if iErr != nil { return iErr } diff --git a/datamodel/low/v2/path_item.go b/datamodel/low/v2/path_item.go index 5046534..944d665 100644 --- a/datamodel/low/v2/path_item.go +++ b/datamodel/low/v2/path_item.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "sort" @@ -48,7 +49,7 @@ func (p *PathItem) GetExtensions() map[low.KeyReference[string]]low.ValueReferen // Build will extract extensions, parameters and operations for all methods. Every method is handled // asynchronously, in order to keep things moving quickly for complex operations. -func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (p *PathItem) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) p.Extensions = low.ExtractExtensions(root) @@ -61,7 +62,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { var ops []low.NodeReference[*Operation] // extract parameters - params, ln, vn, pErr := low.ExtractArray[*Parameter](ParametersLabel, root, idx) + params, ln, vn, pErr := low.ExtractArray[*Parameter](ctx, ParametersLabel, root, idx) if pErr != nil { return pErr } @@ -158,7 +159,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { opErrorChan := make(chan error) var buildOpFunc = func(op low.NodeReference[*Operation], ch chan<- bool, errCh chan<- error) { - er := op.Value.Build(op.KeyNode, op.ValueNode, idx) + er := op.Value.Build(ctx, op.KeyNode, op.ValueNode, idx) if er != nil { errCh <- er } diff --git a/datamodel/low/v2/paths.go b/datamodel/low/v2/paths.go index 97647e6..cc8cbda 100644 --- a/datamodel/low/v2/paths.go +++ b/datamodel/low/v2/paths.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "sort" @@ -54,7 +55,7 @@ func (p *Paths) FindExtension(ext string) *low.ValueReference[any] { } // Build will extract extensions and paths from node. -func (p *Paths) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (p *Paths) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) p.Extensions = low.ExtractExtensions(root) @@ -126,7 +127,7 @@ func (p *Paths) Build(_, root *yaml.Node, idx *index.SpecIndex) error { cNode := value.currentNode path := new(PathItem) _ = low.BuildModel(pNode, path) - err := path.Build(cNode, pNode, idx) + err := path.Build(ctx, cNode, pNode, idx) if err != nil { return pathBuildResult{}, err } diff --git a/datamodel/low/v2/response.go b/datamodel/low/v2/response.go index 266adc1..4bbc174 100644 --- a/datamodel/low/v2/response.go +++ b/datamodel/low/v2/response.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -43,11 +44,11 @@ func (r *Response) FindHeader(hType string) *low.ValueReference[*Header] { } // Build will extract schema, extensions, examples and headers from node -func (r *Response) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (r *Response) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) r.Extensions = low.ExtractExtensions(root) - s, err := base.ExtractSchema(root, idx) + s, err := base.ExtractSchema(ctx, root, idx) if err != nil { return err } @@ -56,14 +57,14 @@ func (r *Response) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // extract examples - examples, expErr := low.ExtractObject[*Examples](ExamplesLabel, root, idx) + examples, expErr := low.ExtractObject[*Examples](ctx, ExamplesLabel, root, idx) if expErr != nil { return expErr } r.Examples = examples //extract headers - headers, lN, kN, err := low.ExtractMap[*Header](HeadersLabel, root, idx) + headers, lN, kN, err := low.ExtractMap[*Header](ctx, HeadersLabel, root, idx) if err != nil { return err } diff --git a/datamodel/low/v2/responses.go b/datamodel/low/v2/responses.go index 43a6942..9514edb 100644 --- a/datamodel/low/v2/responses.go +++ b/datamodel/low/v2/responses.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -27,13 +28,13 @@ func (r *Responses) GetExtensions() map[low.KeyReference[string]]low.ValueRefere } // Build will extract default value and extensions from node. -func (r *Responses) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (r *Responses) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) r.Extensions = low.ExtractExtensions(root) if utils.IsNodeMap(root) { - codes, err := low.ExtractMapNoLookup[*Response](root, idx) + codes, err := low.ExtractMapNoLookup[*Response](ctx, root, idx) if err != nil { return err } diff --git a/datamodel/low/v2/scopes.go b/datamodel/low/v2/scopes.go index 87cc0a2..c1fcad8 100644 --- a/datamodel/low/v2/scopes.go +++ b/datamodel/low/v2/scopes.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -34,7 +35,7 @@ func (s *Scopes) FindScope(scope string) *low.ValueReference[string] { } // Build will extract scope values and extensions from node. -func (s *Scopes) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (s *Scopes) Build(_ context.Context, _, root *yaml.Node, _ *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) s.Extensions = low.ExtractExtensions(root) diff --git a/datamodel/low/v2/security_scheme.go b/datamodel/low/v2/security_scheme.go index 0a025c4..bdf235d 100644 --- a/datamodel/low/v2/security_scheme.go +++ b/datamodel/low/v2/security_scheme.go @@ -4,6 +4,7 @@ package v2 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -38,12 +39,12 @@ func (ss *SecurityScheme) GetExtensions() map[low.KeyReference[string]]low.Value } // Build will extract extensions and scopes from the node. -func (ss *SecurityScheme) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (ss *SecurityScheme) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) ss.Extensions = low.ExtractExtensions(root) - scopes, sErr := low.ExtractObject[*Scopes](ScopesLabel, root, idx) + scopes, sErr := low.ExtractObject[*Scopes](ctx, ScopesLabel, root, idx) if sErr != nil { return sErr } diff --git a/datamodel/low/v2/swagger.go b/datamodel/low/v2/swagger.go index 58fd015..15b5678 100644 --- a/datamodel/low/v2/swagger.go +++ b/datamodel/low/v2/swagger.go @@ -12,6 +12,7 @@ package v2 import ( + "context" "github.com/pb33f/libopenapi/datamodel" "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/datamodel/low/base" @@ -20,7 +21,7 @@ import ( ) // processes a property of a Swagger document asynchronously using bool and error channels for signals. -type documentFunction func(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) +type documentFunction func(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) // Swagger represents a high-level Swagger / OpenAPI 2 document. An instance of Swagger is the root of the specification. type Swagger struct { @@ -129,6 +130,9 @@ func CreateDocumentFromConfig(info *datamodel.SpecInfo, // CreateDocument will create a new Swagger document from the provided SpecInfo. // // Deprecated: Use CreateDocumentFromConfig instead. + +// TODO; DELETE ME + func CreateDocument(info *datamodel.SpecInfo) (*Swagger, []error) { return createDocument(info, &datamodel.DocumentConfiguration{ AllowRemoteReferences: true, @@ -155,8 +159,10 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur // build out swagger scalar variables. _ = low.BuildModel(info.RootNode.Content[0], &doc) + ctx := context.Background() + // extract externalDocs - extDocs, err := low.ExtractObject[*base.ExternalDoc](base.ExternalDocsLabel, info.RootNode, idx) + extDocs, err := low.ExtractObject[*base.ExternalDoc](ctx, base.ExternalDocsLabel, info.RootNode, idx) if err != nil { errors = append(errors, err) } @@ -186,7 +192,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur doneChan := make(chan bool) errChan := make(chan error) for i := range extractionFuncs { - go extractionFuncs[i](info.RootNode.Content[0], &doc, idx, doneChan, errChan) + go extractionFuncs[i](ctx, info.RootNode.Content[0], &doc, idx, doneChan, errChan) } completedExtractions := 0 for completedExtractions < len(extractionFuncs) { @@ -210,8 +216,8 @@ func (s *Swagger) GetExternalDocs() *low.NodeReference[any] { } } -func extractInfo(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - info, err := low.ExtractObject[*base.Info](base.InfoLabel, root, idx) +func extractInfo(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + info, err := low.ExtractObject[*base.Info](ctx, base.InfoLabel, root, idx) if err != nil { e <- err return @@ -220,8 +226,8 @@ func extractInfo(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- b c <- true } -func extractPaths(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - paths, err := low.ExtractObject[*Paths](PathsLabel, root, idx) +func extractPaths(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + paths, err := low.ExtractObject[*Paths](ctx, PathsLabel, root, idx) if err != nil { e <- err return @@ -229,8 +235,8 @@ func extractPaths(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- doc.Paths = paths c <- true } -func extractDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - def, err := low.ExtractObject[*Definitions](DefinitionsLabel, root, idx) +func extractDefinitions(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + def, err := low.ExtractObject[*Definitions](ctx, DefinitionsLabel, root, idx) if err != nil { e <- err return @@ -238,8 +244,8 @@ func extractDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c c doc.Definitions = def c <- true } -func extractParamDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - param, err := low.ExtractObject[*ParameterDefinitions](ParametersLabel, root, idx) +func extractParamDefinitions(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + param, err := low.ExtractObject[*ParameterDefinitions](ctx, ParametersLabel, root, idx) if err != nil { e <- err return @@ -248,8 +254,8 @@ func extractParamDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIndex c <- true } -func extractResponsesDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - resp, err := low.ExtractObject[*ResponsesDefinitions](ResponsesLabel, root, idx) +func extractResponsesDefinitions(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + resp, err := low.ExtractObject[*ResponsesDefinitions](ctx, ResponsesLabel, root, idx) if err != nil { e <- err return @@ -258,8 +264,8 @@ func extractResponsesDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecI c <- true } -func extractSecurityDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - sec, err := low.ExtractObject[*SecurityDefinitions](SecurityDefinitionsLabel, root, idx) +func extractSecurityDefinitions(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + sec, err := low.ExtractObject[*SecurityDefinitions](ctx, SecurityDefinitionsLabel, root, idx) if err != nil { e <- err return @@ -268,8 +274,8 @@ func extractSecurityDefinitions(root *yaml.Node, doc *Swagger, idx *index.SpecIn c <- true } -func extractTags(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - tags, ln, vn, err := low.ExtractArray[*base.Tag](base.TagsLabel, root, idx) +func extractTags(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + tags, ln, vn, err := low.ExtractArray[*base.Tag](ctx, base.TagsLabel, root, idx) if err != nil { e <- err return @@ -282,8 +288,8 @@ func extractTags(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- b c <- true } -func extractSecurity(root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { - sec, ln, vn, err := low.ExtractArray[*base.SecurityRequirement](SecurityLabel, root, idx) +func extractSecurity(ctx context.Context, root *yaml.Node, doc *Swagger, idx *index.SpecIndex, c chan<- bool, e chan<- error) { + sec, ln, vn, err := low.ExtractArray[*base.SecurityRequirement](ctx, SecurityLabel, root, idx) if err != nil { e <- err return diff --git a/datamodel/low/v3/callback.go b/datamodel/low/v3/callback.go index bd66c27..814a7f1 100644 --- a/datamodel/low/v3/callback.go +++ b/datamodel/low/v3/callback.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/utils" @@ -39,7 +40,7 @@ func (cb *Callback) FindExpression(exp string) *low.ValueReference[*PathItem] { } // Build will extract extensions, expressions and PathItem objects for Callback -func (cb *Callback) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (cb *Callback) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) cb.Reference = new(low.Reference) @@ -57,7 +58,7 @@ func (cb *Callback) Build(_, root *yaml.Node, idx *index.SpecIndex) error { if strings.HasPrefix(currentCB.Value, "x-") { continue // ignore extension. } - callback, eErr, _, rv := low.ExtractObjectRaw[*PathItem](currentCB, callbackNode, idx) + callback, eErr, _, rv := low.ExtractObjectRaw[*PathItem](ctx, currentCB, callbackNode, idx) if eErr != nil { return eErr } diff --git a/datamodel/low/v3/components.go b/datamodel/low/v3/components.go index d8b6f8d..61c5183 100644 --- a/datamodel/low/v3/components.go +++ b/datamodel/low/v3/components.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "sort" @@ -141,7 +142,7 @@ func (co *Components) FindCallback(callback string) *low.ValueReference[*Callbac // Build converts root YAML node containing components to low level model. // Process each component in parallel. -func (co *Components) Build(root *yaml.Node, idx *index.SpecIndex) error { +func (co *Components) Build(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) co.Reference = new(low.Reference) @@ -161,55 +162,55 @@ func (co *Components) Build(root *yaml.Node, idx *index.SpecIndex) error { } go func() { - schemas, err := extractComponentValues[*base.SchemaProxy](SchemasLabel, root, idx) + schemas, err := extractComponentValues[*base.SchemaProxy](ctx, SchemasLabel, root, idx) captureError(err) co.Schemas = schemas wg.Done() }() go func() { - parameters, err := extractComponentValues[*Parameter](ParametersLabel, root, idx) + parameters, err := extractComponentValues[*Parameter](ctx, ParametersLabel, root, idx) captureError(err) co.Parameters = parameters wg.Done() }() go func() { - responses, err := extractComponentValues[*Response](ResponsesLabel, root, idx) + responses, err := extractComponentValues[*Response](ctx, ResponsesLabel, root, idx) captureError(err) co.Responses = responses wg.Done() }() go func() { - examples, err := extractComponentValues[*base.Example](base.ExamplesLabel, root, idx) + examples, err := extractComponentValues[*base.Example](ctx, base.ExamplesLabel, root, idx) captureError(err) co.Examples = examples wg.Done() }() go func() { - requestBodies, err := extractComponentValues[*RequestBody](RequestBodiesLabel, root, idx) + requestBodies, err := extractComponentValues[*RequestBody](ctx, RequestBodiesLabel, root, idx) captureError(err) co.RequestBodies = requestBodies wg.Done() }() go func() { - headers, err := extractComponentValues[*Header](HeadersLabel, root, idx) + headers, err := extractComponentValues[*Header](ctx, HeadersLabel, root, idx) captureError(err) co.Headers = headers wg.Done() }() go func() { - securitySchemes, err := extractComponentValues[*SecurityScheme](SecuritySchemesLabel, root, idx) + securitySchemes, err := extractComponentValues[*SecurityScheme](ctx, SecuritySchemesLabel, root, idx) captureError(err) co.SecuritySchemes = securitySchemes wg.Done() }() go func() { - links, err := extractComponentValues[*Link](LinksLabel, root, idx) + links, err := extractComponentValues[*Link](ctx, LinksLabel, root, idx) captureError(err) co.Links = links wg.Done() }() go func() { - callbacks, err := extractComponentValues[*Callback](CallbacksLabel, root, idx) + callbacks, err := extractComponentValues[*Callback](ctx, CallbacksLabel, root, idx) captureError(err) co.Callbacks = callbacks wg.Done() @@ -222,7 +223,7 @@ func (co *Components) Build(root *yaml.Node, idx *index.SpecIndex) error { // extractComponentValues converts all the YAML nodes of a component type to // low level model. // Process each node in parallel. -func extractComponentValues[T low.Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (low.NodeReference[map[low.KeyReference[string]]low.ValueReference[T]], error) { +func extractComponentValues[T low.Buildable[N], N any](ctx context.Context, label string, root *yaml.Node, idx *index.SpecIndex) (low.NodeReference[map[low.KeyReference[string]]low.ValueReference[T]], error) { var emptyResult low.NodeReference[map[low.KeyReference[string]]low.ValueReference[T]] _, nodeLabel, nodeValue := utils.FindKeyNodeFullTop(label, root.Content) if nodeValue == nil { @@ -288,7 +289,7 @@ func extractComponentValues[T low.Buildable[N], N any](label string, root *yaml. // TODO: check circular crazy on this. It may explode var err error if h, _, _ := utils.IsNodeRefValue(node); h && label != SchemasLabel { - node, err = low.LocateRefNode(node, idx) + node, _, err = low.LocateRefNode(node, idx) } if err != nil { return componentBuildResult[T]{}, err @@ -296,7 +297,7 @@ func extractComponentValues[T low.Buildable[N], N any](label string, root *yaml. // build. _ = low.BuildModel(node, n) - err = n.Build(currentLabel, node, idx) + err = n.Build(ctx, currentLabel, node, idx) if err != nil { return componentBuildResult[T]{}, err } diff --git a/datamodel/low/v3/create_document.go b/datamodel/low/v3/create_document.go index 14cfff6..d3901db 100644 --- a/datamodel/low/v3/create_document.go +++ b/datamodel/low/v3/create_document.go @@ -1,6 +1,7 @@ package v3 import ( + "context" "errors" "os" "path/filepath" @@ -39,28 +40,24 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur version = low.NodeReference[string]{Value: versionNode.Value, KeyNode: labelNode, ValueNode: versionNode} doc := Document{Version: version} - // get current working directory as a basePath - cwd, _ := os.Getwd() - if config.BasePath != "" { - cwd = config.BasePath - } - // TODO: configure allowFileReferences and allowRemoteReferences stuff // create an index config and shadow the document configuration. idxConfig := index.CreateOpenAPIIndexConfig() idxConfig.SpecInfo = info - idxConfig.BasePath = cwd idxConfig.IgnoreArrayCircularReferences = config.IgnoreArrayCircularReferences idxConfig.IgnorePolymorphicCircularReferences = config.IgnorePolymorphicCircularReferences - idxConfig.AvoidCircularReferenceCheck = config.SkipCircularReferenceCheck - + idxConfig.AvoidCircularReferenceCheck = true + idxConfig.BaseURL = config.BaseURL + idxConfig.BasePath = config.BasePath rolodex := index.NewRolodex(idxConfig) + rolodex.SetRootNode(info.RootNode) doc.Rolodex = rolodex - // If basePath is provided override it - if config.BasePath != "" { + // If basePath is provided, add a local filesystem to the rolodex. + if idxConfig.BasePath != "" { var absError error + var cwd string cwd, absError = filepath.Abs(config.BasePath) if absError != nil { return nil, absError @@ -77,13 +74,34 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur } - // TODO: Remote filesystem + // if base url is provided, add a remote filesystem to the rolodex. + if idxConfig.BaseURL != nil { + + // create a remote filesystem + remoteFS, fsErr := index.NewRemoteFSWithConfig(idxConfig) + if fsErr != nil { + return nil, fsErr + } + if config.RemoteURLHandler != nil { + remoteFS.RemoteHandlerFunc = config.RemoteURLHandler + } + // add to the rolodex + rolodex.AddRemoteFS(config.BaseURL.String(), remoteFS) + } // index the rolodex - err := rolodex.IndexTheRolodex() var errs []error - if err != nil { - errs = append(errs, rolodex.GetCaughtErrors()...) + + _ = rolodex.IndexTheRolodex() + + if !config.SkipCircularReferenceCheck { + rolodex.CheckForCircularReferences() + } + + roloErrs := rolodex.GetCaughtErrors() + + if roloErrs != nil { + errs = append(errs, roloErrs...) } doc.Index = rolodex.GetRootIndex() @@ -125,17 +143,17 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur } } - runExtraction := func(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex, - runFunc func(i *datamodel.SpecInfo, d *Document, idx *index.SpecIndex) error, + runExtraction := func(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex, + runFunc func(ctx context.Context, i *datamodel.SpecInfo, d *Document, idx *index.SpecIndex) error, ers *[]error, wg *sync.WaitGroup, ) { - if er := runFunc(info, doc, idx); er != nil { + if er := runFunc(ctx, info, doc, idx); er != nil { *ers = append(*ers, er) } wg.Done() } - extractionFuncs := []func(i *datamodel.SpecInfo, d *Document, idx *index.SpecIndex) error{ + extractionFuncs := []func(ctx context.Context, i *datamodel.SpecInfo, d *Document, idx *index.SpecIndex) error{ extractInfo, extractServers, extractTags, @@ -146,28 +164,30 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur extractWebhooks, } + ctx := context.Background() + wg.Add(len(extractionFuncs)) for _, f := range extractionFuncs { - go runExtraction(info, &doc, rolodex.GetRootIndex(), f, &errs, &wg) + go runExtraction(ctx, info, &doc, rolodex.GetRootIndex(), f, &errs, &wg) } wg.Wait() return &doc, errors.Join(errs...) } -func extractInfo(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { +func extractInfo(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { _, ln, vn := utils.FindKeyNodeFullTop(base.InfoLabel, info.RootNode.Content[0].Content) if vn != nil { ir := base.Info{} _ = low.BuildModel(vn, &ir) - _ = ir.Build(ln, vn, idx) + _ = ir.Build(ctx, ln, vn, idx) nr := low.NodeReference[*base.Info]{Value: &ir, ValueNode: vn, KeyNode: ln} doc.Info = nr } return nil } -func extractSecurity(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { - sec, ln, vn, err := low.ExtractArray[*base.SecurityRequirement](SecurityLabel, info.RootNode.Content[0], idx) +func extractSecurity(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { + sec, ln, vn, err := low.ExtractArray[*base.SecurityRequirement](ctx, SecurityLabel, info.RootNode.Content[0], idx) if err != nil { return err } @@ -181,8 +201,8 @@ func extractSecurity(info *datamodel.SpecInfo, doc *Document, idx *index.SpecInd return nil } -func extractExternalDocs(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { - extDocs, dErr := low.ExtractObject[*base.ExternalDoc](base.ExternalDocsLabel, info.RootNode.Content[0], idx) +func extractExternalDocs(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { + extDocs, dErr := low.ExtractObject[*base.ExternalDoc](ctx, base.ExternalDocsLabel, info.RootNode.Content[0], idx) if dErr != nil { return dErr } @@ -190,12 +210,12 @@ func extractExternalDocs(info *datamodel.SpecInfo, doc *Document, idx *index.Spe return nil } -func extractComponents(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { +func extractComponents(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { _, ln, vn := utils.FindKeyNodeFullTop(ComponentsLabel, info.RootNode.Content[0].Content) if vn != nil { ir := Components{} _ = low.BuildModel(vn, &ir) - err := ir.Build(vn, idx) + err := ir.Build(ctx, vn, idx) if err != nil { return err } @@ -205,7 +225,7 @@ func extractComponents(info *datamodel.SpecInfo, doc *Document, idx *index.SpecI return nil } -func extractServers(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { +func extractServers(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { _, ln, vn := utils.FindKeyNodeFull(ServersLabel, info.RootNode.Content[0].Content) if vn != nil { if utils.IsNodeArray(vn) { @@ -214,7 +234,7 @@ func extractServers(info *datamodel.SpecInfo, doc *Document, idx *index.SpecInde if utils.IsNodeMap(srvN) { srvr := Server{} _ = low.BuildModel(srvN, &srvr) - _ = srvr.Build(ln, srvN, idx) + _ = srvr.Build(ctx, ln, srvN, idx) servers = append(servers, low.ValueReference[*Server]{ Value: &srvr, ValueNode: srvN, @@ -231,7 +251,7 @@ func extractServers(info *datamodel.SpecInfo, doc *Document, idx *index.SpecInde return nil } -func extractTags(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { +func extractTags(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { _, ln, vn := utils.FindKeyNodeFull(base.TagsLabel, info.RootNode.Content[0].Content) if vn != nil { if utils.IsNodeArray(vn) { @@ -240,7 +260,7 @@ func extractTags(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) if utils.IsNodeMap(tagN) { tag := base.Tag{} _ = low.BuildModel(tagN, &tag) - if err := tag.Build(ln, tagN, idx); err != nil { + if err := tag.Build(ctx, ln, tagN, idx); err != nil { return err } tags = append(tags, low.ValueReference[*base.Tag]{ @@ -259,11 +279,11 @@ func extractTags(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) return nil } -func extractPaths(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { +func extractPaths(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { _, ln, vn := utils.FindKeyNodeFull(PathsLabel, info.RootNode.Content[0].Content) if vn != nil { ir := Paths{} - err := ir.Build(ln, vn, idx) + err := ir.Build(ctx, ln, vn, idx) if err != nil { return err } @@ -273,8 +293,8 @@ func extractPaths(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) return nil } -func extractWebhooks(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { - hooks, hooksL, hooksN, eErr := low.ExtractMap[*PathItem](WebhooksLabel, info.RootNode, idx) +func extractWebhooks(ctx context.Context, info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error { + hooks, hooksL, hooksN, eErr := low.ExtractMap[*PathItem](ctx, WebhooksLabel, info.RootNode, idx) if eErr != nil { return eErr } diff --git a/datamodel/low/v3/create_document_test.go b/datamodel/low/v3/create_document_test.go index 0f58a1c..a13c7b6 100644 --- a/datamodel/low/v3/create_document_test.go +++ b/datamodel/low/v3/create_document_test.go @@ -147,9 +147,8 @@ func TestCreateDocumentStripe(t *testing.T) { d, err := CreateDocumentFromConfig(info, &datamodel.DocumentConfiguration{ AllowFileReferences: false, AllowRemoteReferences: false, - BasePath: "/here", }) - assert.Len(t, err, 3) + assert.Len(t, utils.UnwrapErrors(err), 3) assert.Equal(t, "3.0.0", d.Version.Value) assert.Equal(t, "Stripe API", d.Info.Value.Title.Value) @@ -206,7 +205,8 @@ func TestCreateDocument_WebHooks(t *testing.T) { } func TestCreateDocument_WebHooks_Error(t *testing.T) { - yml := `webhooks: + yml := `openapi: 3.0 +webhooks: $ref: #bork` info, _ := datamodel.ExtractSpecInfo([]byte(yml)) @@ -215,7 +215,7 @@ func TestCreateDocument_WebHooks_Error(t *testing.T) { AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Len(t, err, 1) + assert.Len(t, utils.UnwrapErrors(err), 1) } func TestCreateDocument_Servers(t *testing.T) { @@ -613,7 +613,7 @@ webhooks: AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Equal(t, "flat map build failed: reference cannot be found: reference '' at line 4, column 5 was not found", + assert.Equal(t, "flat map build failed: reference cannot be found: reference at line 4, column 5 is empty, it cannot be resolved", err.Error()) } @@ -630,7 +630,7 @@ components: AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Equal(t, "reference '' at line 5, column 7 was not found", err.Error()) + assert.Equal(t, "reference at line 5, column 7 is empty, it cannot be resolved", err.Error()) } func TestCreateDocument_Paths_Errors(t *testing.T) { @@ -661,7 +661,7 @@ tags: AllowRemoteReferences: false, }) assert.Equal(t, - "object extraction failed: reference '' at line 3, column 5 was not found", err.Error()) + "object extraction failed: reference at line 3, column 5 is empty, it cannot be resolved", err.Error()) } func TestCreateDocument_Security_Error(t *testing.T) { @@ -676,7 +676,7 @@ security: AllowRemoteReferences: false, }) assert.Equal(t, - "array build failed: reference cannot be found: reference '' at line 3, column 3 was not found", + "array build failed: reference cannot be found: reference at line 3, column 3 is empty, it cannot be resolved", err.Error()) } @@ -691,7 +691,7 @@ externalDocs: AllowFileReferences: false, AllowRemoteReferences: false, }) - assert.Equal(t, "object extraction failed: reference '' at line 3, column 3 was not found", err.Error()) + assert.Equal(t, "object extraction failed: reference at line 3, column 3 is empty, it cannot be resolved", err.Error()) } func TestCreateDocument_YamlAnchor(t *testing.T) { diff --git a/datamodel/low/v3/encoding.go b/datamodel/low/v3/encoding.go index 6e3079f..0dd469b 100644 --- a/datamodel/low/v3/encoding.go +++ b/datamodel/low/v3/encoding.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -58,11 +59,11 @@ func (en *Encoding) Hash() [32]byte { } // Build will extract all Header objects from supplied node. -func (en *Encoding) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (en *Encoding) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) en.Reference = new(low.Reference) - headers, hL, hN, err := low.ExtractMap[*Header](HeadersLabel, root, idx) + headers, hL, hN, err := low.ExtractMap[*Header](ctx, HeadersLabel, root, idx) if err != nil { return err } diff --git a/datamodel/low/v3/header.go b/datamodel/low/v3/header.go index 095640f..ac7e15c 100644 --- a/datamodel/low/v3/header.go +++ b/datamodel/low/v3/header.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -95,7 +96,7 @@ func (h *Header) Hash() [32]byte { } // Build will extract extensions, examples, schema and content/media types from node. -func (h *Header) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (h *Header) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) h.Reference = new(low.Reference) @@ -108,7 +109,7 @@ func (h *Header) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle examples if set. - exps, expsL, expsN, eErr := low.ExtractMap[*base.Example](base.ExamplesLabel, root, idx) + exps, expsL, expsN, eErr := low.ExtractMap[*base.Example](ctx, base.ExamplesLabel, root, idx) if eErr != nil { return eErr } @@ -121,7 +122,7 @@ func (h *Header) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle schema - sch, sErr := base.ExtractSchema(root, idx) + sch, sErr := base.ExtractSchema(ctx, root, idx) if sErr != nil { return sErr } @@ -130,7 +131,7 @@ func (h *Header) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle content, if set. - con, cL, cN, cErr := low.ExtractMap[*MediaType](ContentLabel, root, idx) + con, cL, cN, cErr := low.ExtractMap[*MediaType](ctx, ContentLabel, root, idx) if cErr != nil { return cErr } diff --git a/datamodel/low/v3/link.go b/datamodel/low/v3/link.go index d5272b9..deadeea 100644 --- a/datamodel/low/v3/link.go +++ b/datamodel/low/v3/link.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -53,13 +54,13 @@ func (l *Link) FindExtension(ext string) *low.ValueReference[any] { } // Build will extract extensions and servers from the node. -func (l *Link) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (l *Link) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) l.Reference = new(low.Reference) l.Extensions = low.ExtractExtensions(root) // extract server. - ser, sErr := low.ExtractObject[*Server](ServerLabel, root, idx) + ser, sErr := low.ExtractObject[*Server](ctx, ServerLabel, root, idx) if sErr != nil { return sErr } diff --git a/datamodel/low/v3/media_type.go b/datamodel/low/v3/media_type.go index 59c7b8e..4979b1c 100644 --- a/datamodel/low/v3/media_type.go +++ b/datamodel/low/v3/media_type.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -54,7 +55,7 @@ func (mt *MediaType) GetAllExamples() map[low.KeyReference[string]]low.ValueRefe } // Build will extract examples, extensions, schema and encoding from node. -func (mt *MediaType) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (mt *MediaType) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) mt.Reference = new(low.Reference) @@ -83,7 +84,7 @@ func (mt *MediaType) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } //handle schema - sch, sErr := base.ExtractSchema(root, idx) + sch, sErr := base.ExtractSchema(ctx, root, idx) if sErr != nil { return sErr } @@ -92,7 +93,7 @@ func (mt *MediaType) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle examples if set. - exps, expsL, expsN, eErr := low.ExtractMap[*base.Example](base.ExamplesLabel, root, idx) + exps, expsL, expsN, eErr := low.ExtractMap[*base.Example](ctx, base.ExamplesLabel, root, idx) if eErr != nil { return eErr } @@ -105,7 +106,7 @@ func (mt *MediaType) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle encoding - encs, encsL, encsN, encErr := low.ExtractMap[*Encoding](EncodingLabel, root, idx) + encs, encsL, encsN, encErr := low.ExtractMap[*Encoding](ctx, EncodingLabel, root, idx) if encErr != nil { return encErr } diff --git a/datamodel/low/v3/oauth_flows.go b/datamodel/low/v3/oauth_flows.go index cd16da9..f268d2b 100644 --- a/datamodel/low/v3/oauth_flows.go +++ b/datamodel/low/v3/oauth_flows.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -36,31 +37,31 @@ func (o *OAuthFlows) FindExtension(ext string) *low.ValueReference[any] { } // Build will extract extensions and all OAuthFlow types from the supplied node. -func (o *OAuthFlows) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (o *OAuthFlows) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) o.Reference = new(low.Reference) o.Extensions = low.ExtractExtensions(root) - v, vErr := low.ExtractObject[*OAuthFlow](ImplicitLabel, root, idx) + v, vErr := low.ExtractObject[*OAuthFlow](ctx, ImplicitLabel, root, idx) if vErr != nil { return vErr } o.Implicit = v - v, vErr = low.ExtractObject[*OAuthFlow](PasswordLabel, root, idx) + v, vErr = low.ExtractObject[*OAuthFlow](ctx, PasswordLabel, root, idx) if vErr != nil { return vErr } o.Password = v - v, vErr = low.ExtractObject[*OAuthFlow](ClientCredentialsLabel, root, idx) + v, vErr = low.ExtractObject[*OAuthFlow](ctx, ClientCredentialsLabel, root, idx) if vErr != nil { return vErr } o.ClientCredentials = v - v, vErr = low.ExtractObject[*OAuthFlow](AuthorizationCodeLabel, root, idx) + v, vErr = low.ExtractObject[*OAuthFlow](ctx, AuthorizationCodeLabel, root, idx) if vErr != nil { return vErr } @@ -116,7 +117,7 @@ func (o *OAuthFlow) FindExtension(ext string) *low.ValueReference[any] { } // Build will extract extensions from the node. -func (o *OAuthFlow) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (o *OAuthFlow) Build(_ context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { o.Reference = new(low.Reference) o.Extensions = low.ExtractExtensions(root) return nil diff --git a/datamodel/low/v3/operation.go b/datamodel/low/v3/operation.go index 909ef8a..f139df4 100644 --- a/datamodel/low/v3/operation.go +++ b/datamodel/low/v3/operation.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -55,21 +56,21 @@ func (o *Operation) FindSecurityRequirement(name string) []low.ValueReference[st } // Build will extract external docs, parameters, request body, responses, callbacks, security and servers. -func (o *Operation) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (o *Operation) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) o.Reference = new(low.Reference) o.Extensions = low.ExtractExtensions(root) // extract externalDocs - extDocs, dErr := low.ExtractObject[*base.ExternalDoc](base.ExternalDocsLabel, root, idx) + extDocs, dErr := low.ExtractObject[*base.ExternalDoc](ctx, base.ExternalDocsLabel, root, idx) if dErr != nil { return dErr } o.ExternalDocs = extDocs // extract parameters - params, ln, vn, pErr := low.ExtractArray[*Parameter](ParametersLabel, root, idx) + params, ln, vn, pErr := low.ExtractArray[*Parameter](ctx, ParametersLabel, root, idx) if pErr != nil { return pErr } @@ -82,21 +83,21 @@ func (o *Operation) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // extract request body - rBody, rErr := low.ExtractObject[*RequestBody](RequestBodyLabel, root, idx) + rBody, rErr := low.ExtractObject[*RequestBody](ctx, RequestBodyLabel, root, idx) if rErr != nil { return rErr } o.RequestBody = rBody // extract responses - respBody, respErr := low.ExtractObject[*Responses](ResponsesLabel, root, idx) + respBody, respErr := low.ExtractObject[*Responses](ctx, ResponsesLabel, root, idx) if respErr != nil { return respErr } o.Responses = respBody // extract callbacks - callbacks, cbL, cbN, cbErr := low.ExtractMap[*Callback](CallbacksLabel, root, idx) + callbacks, cbL, cbN, cbErr := low.ExtractMap[*Callback](ctx, CallbacksLabel, root, idx) if cbErr != nil { return cbErr } @@ -109,7 +110,7 @@ func (o *Operation) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // extract security - sec, sln, svn, sErr := low.ExtractArray[*base.SecurityRequirement](SecurityLabel, root, idx) + sec, sln, svn, sErr := low.ExtractArray[*base.SecurityRequirement](ctx, SecurityLabel, root, idx) if sErr != nil { return sErr } @@ -134,7 +135,7 @@ func (o *Operation) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // extract servers - servers, sl, sn, serErr := low.ExtractArray[*Server](ServersLabel, root, idx) + servers, sl, sn, serErr := low.ExtractArray[*Server](ctx, ServersLabel, root, idx) if serErr != nil { return serErr } diff --git a/datamodel/low/v3/parameter.go b/datamodel/low/v3/parameter.go index 0543f0d..2903a3c 100644 --- a/datamodel/low/v3/parameter.go +++ b/datamodel/low/v3/parameter.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -58,7 +59,7 @@ func (p *Parameter) GetExtensions() map[low.KeyReference[string]]low.ValueRefere } // Build will extract examples, extensions and content/media types. -func (p *Parameter) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (p *Parameter) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) p.Reference = new(low.Reference) @@ -71,7 +72,7 @@ func (p *Parameter) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle schema - sch, sErr := base.ExtractSchema(root, idx) + sch, sErr := base.ExtractSchema(ctx, root, idx) if sErr != nil { return sErr } @@ -80,7 +81,7 @@ func (p *Parameter) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle examples if set. - exps, expsL, expsN, eErr := low.ExtractMap[*base.Example](base.ExamplesLabel, root, idx) + exps, expsL, expsN, eErr := low.ExtractMap[*base.Example](ctx, base.ExamplesLabel, root, idx) if eErr != nil { return eErr } @@ -93,7 +94,7 @@ func (p *Parameter) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle content, if set. - con, cL, cN, cErr := low.ExtractMap[*MediaType](ContentLabel, root, idx) + con, cL, cN, cErr := low.ExtractMap[*MediaType](ctx, ContentLabel, root, idx) if cErr != nil { return cErr } diff --git a/datamodel/low/v3/path_item.go b/datamodel/low/v3/path_item.go index 9a0a157..9a38966 100644 --- a/datamodel/low/v3/path_item.go +++ b/datamodel/low/v3/path_item.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "sort" @@ -109,7 +110,7 @@ func (p *PathItem) GetExtensions() map[low.KeyReference[string]]low.ValueReferen // Build extracts extensions, parameters, servers and each http method defined. // everything is extracted asynchronously for speed. -func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (p *PathItem) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) p.Reference = new(low.Reference) @@ -123,7 +124,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { var ops []low.NodeReference[*Operation] // extract parameters - params, ln, vn, pErr := low.ExtractArray[*Parameter](ParametersLabel, root, idx) + params, ln, vn, pErr := low.ExtractArray[*Parameter](ctx, ParametersLabel, root, idx) if pErr != nil { return pErr } @@ -143,7 +144,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { if utils.IsNodeMap(srvN) { srvr := new(Server) _ = low.BuildModel(srvN, srvr) - srvr.Build(ln, srvN, idx) + srvr.Build(ctx, ln, srvN, idx) servers = append(servers, low.ValueReference[*Server]{ Value: srvr, ValueNode: srvN, @@ -198,6 +199,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { continue // ignore everything else. } + foundContext := ctx var op Operation opIsRef := false var opRefVal string @@ -213,12 +215,15 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { opIsRef = true opRefVal = ref - r, err := low.LocateRefNode(pathNode, idx) + r, newIdx, err, nCtx := low.LocateRefNodeWithContext(ctx, pathNode, idx) if r != nil { if r.Kind == yaml.DocumentNode { r = r.Content[0] } pathNode = r + foundContext = nCtx + foundContext = context.WithValue(foundContext, "foundIndex", newIdx) + if r.Tag == "" { // If it's a node from file, tag is empty pathNode = r.Content[0] @@ -233,6 +238,8 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { return fmt.Errorf("path item build failed: cannot find reference: %s at line %d, col %d", pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column) } + } else { + foundContext = context.WithValue(foundContext, "foundIndex", idx) } wg.Add(1) low.BuildModelAsync(pathNode, &op, &wg, &errors) @@ -241,6 +248,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { Value: &op, KeyNode: currentNode, ValueNode: pathNode, + Context: foundContext, } if opIsRef { opRef.Reference = opRefVal @@ -277,7 +285,7 @@ func (p *PathItem) Build(_, root *yaml.Node, idx *index.SpecIndex) error { ref = op.Reference } - err := op.Value.Build(op.KeyNode, op.ValueNode, idx) + err := op.Value.Build(op.Context, op.KeyNode, op.ValueNode, op.Context.Value("foundIndex").(*index.SpecIndex)) if ref != "" { op.Value.Reference.Reference = ref } diff --git a/datamodel/low/v3/paths.go b/datamodel/low/v3/paths.go index 1e5501e..b389c09 100644 --- a/datamodel/low/v3/paths.go +++ b/datamodel/low/v3/paths.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "sort" @@ -60,7 +61,7 @@ func (p *Paths) GetExtensions() map[low.KeyReference[string]]low.ValueReference[ } // Build will extract extensions and all PathItems. This happens asynchronously for speed. -func (p *Paths) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (p *Paths) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) p.Reference = new(low.Reference) @@ -134,7 +135,7 @@ func (p *Paths) Build(_, root *yaml.Node, idx *index.SpecIndex) error { cNode := value.currentNode if ok, _, _ := utils.IsNodeRefValue(pNode); ok { - r, err := low.LocateRefNode(pNode, idx) + r, _, err := low.LocateRefNode(pNode, idx) if r != nil { pNode = r if r.Tag == "" { @@ -156,9 +157,12 @@ func (p *Paths) Build(_, root *yaml.Node, idx *index.SpecIndex) error { path := new(PathItem) _ = low.BuildModel(pNode, path) - err := path.Build(cNode, pNode, idx) + err := path.Build(ctx, cNode, pNode, idx) + + // don't fail the pipeline if there is an error, log it instead. if err != nil { - return buildResult{}, err + //return buildResult{}, err + idx.GetLogger().Error(fmt.Sprintf("error building path item '%s'", err.Error())) } return buildResult{ diff --git a/datamodel/low/v3/request_body.go b/datamodel/low/v3/request_body.go index 37952e1..e76164f 100644 --- a/datamodel/low/v3/request_body.go +++ b/datamodel/low/v3/request_body.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -40,14 +41,14 @@ func (rb *RequestBody) FindContent(cType string) *low.ValueReference[*MediaType] } // Build will extract extensions and MediaType objects from the node. -func (rb *RequestBody) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (rb *RequestBody) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) rb.Reference = new(low.Reference) rb.Extensions = low.ExtractExtensions(root) // handle content, if set. - con, cL, cN, cErr := low.ExtractMap[*MediaType](ContentLabel, root, idx) + con, cL, cN, cErr := low.ExtractMap[*MediaType](ctx, ContentLabel, root, idx) if cErr != nil { return cErr } diff --git a/datamodel/low/v3/response.go b/datamodel/low/v3/response.go index 7c7fdb0..7e3b577 100644 --- a/datamodel/low/v3/response.go +++ b/datamodel/low/v3/response.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -54,14 +55,14 @@ func (r *Response) FindLink(hType string) *low.ValueReference[*Link] { } // Build will extract headers, extensions, content and links from node. -func (r *Response) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (r *Response) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) r.Reference = new(low.Reference) r.Extensions = low.ExtractExtensions(root) //extract headers - headers, lN, kN, err := low.ExtractMapExtensions[*Header](HeadersLabel, root, idx, true) + headers, lN, kN, err := low.ExtractMapExtensions[*Header](ctx, HeadersLabel, root, idx, true) if err != nil { return err } @@ -73,7 +74,7 @@ func (r *Response) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } } - con, clN, cN, cErr := low.ExtractMap[*MediaType](ContentLabel, root, idx) + con, clN, cN, cErr := low.ExtractMap[*MediaType](ctx, ContentLabel, root, idx) if cErr != nil { return cErr } @@ -86,7 +87,7 @@ func (r *Response) Build(_, root *yaml.Node, idx *index.SpecIndex) error { } // handle links if set - links, linkLabel, linkValue, lErr := low.ExtractMap[*Link](LinksLabel, root, idx) + links, linkLabel, linkValue, lErr := low.ExtractMap[*Link](ctx, LinksLabel, root, idx) if lErr != nil { return lErr } diff --git a/datamodel/low/v3/responses.go b/datamodel/low/v3/responses.go index 0089acc..092f9f0 100644 --- a/datamodel/low/v3/responses.go +++ b/datamodel/low/v3/responses.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -45,13 +46,13 @@ func (r *Responses) GetExtensions() map[low.KeyReference[string]]low.ValueRefere } // Build will extract default response and all Response objects for each code -func (r *Responses) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (r *Responses) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) r.Reference = new(low.Reference) r.Extensions = low.ExtractExtensions(root) utils.CheckForMergeNodes(root) if utils.IsNodeMap(root) { - codes, err := low.ExtractMapNoLookup[*Response](root, idx) + codes, err := low.ExtractMapNoLookup[*Response](ctx, root, idx) if err != nil { return err diff --git a/datamodel/low/v3/security_scheme.go b/datamodel/low/v3/security_scheme.go index 5ee59fb..ef5e365 100644 --- a/datamodel/low/v3/security_scheme.go +++ b/datamodel/low/v3/security_scheme.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "fmt" "github.com/pb33f/libopenapi/datamodel/low" @@ -48,13 +49,13 @@ func (ss *SecurityScheme) GetExtensions() map[low.KeyReference[string]]low.Value } // Build will extract OAuthFlows and extensions from the node. -func (ss *SecurityScheme) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (ss *SecurityScheme) Build(ctx context.Context, _, root *yaml.Node, idx *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) ss.Reference = new(low.Reference) ss.Extensions = low.ExtractExtensions(root) - oa, oaErr := low.ExtractObject[*OAuthFlows](OAuthFlowsLabel, root, idx) + oa, oaErr := low.ExtractObject[*OAuthFlows](ctx, OAuthFlowsLabel, root, idx) if oaErr != nil { return oaErr } diff --git a/datamodel/low/v3/server.go b/datamodel/low/v3/server.go index f2a0015..586bb7e 100644 --- a/datamodel/low/v3/server.go +++ b/datamodel/low/v3/server.go @@ -4,6 +4,7 @@ package v3 import ( + "context" "crypto/sha256" "github.com/pb33f/libopenapi/datamodel/low" "github.com/pb33f/libopenapi/index" @@ -34,7 +35,7 @@ func (s *Server) FindVariable(serverVar string) *low.ValueReference[*ServerVaria } // Build will extract server variables from the supplied node. -func (s *Server) Build(_, root *yaml.Node, idx *index.SpecIndex) error { +func (s *Server) Build(_ context.Context, _, root *yaml.Node, _ *index.SpecIndex) error { root = utils.NodeAlias(root) utils.CheckForMergeNodes(root) s.Reference = new(low.Reference) diff --git a/index/extract_refs.go b/index/extract_refs.go index 6ef4ba3..1f92e93 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -64,6 +64,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, Definition: definitionPath, Node: node.Content[i+1], Path: jsonPath, + Index: index, } isRef, _, _ := utils.IsNodeRefValue(node.Content[i+1]) @@ -120,6 +121,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, Definition: definitionPath, Node: prop, Path: jsonPath, + Index: index, } isRef, _, _ := utils.IsNodeRefValue(prop) @@ -165,6 +167,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, Definition: definitionPath, Node: element, Path: jsonPath, + Index: index, } isRef, _, _ := utils.IsNodeRefValue(element) @@ -341,6 +344,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, Name: name, Node: node, Path: p, + Index: index, } // add to raw sequenced refs @@ -367,6 +371,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, Name: ref.Name, Node: &copiedNode, Path: p, + Index: index, } // protect this data using a copy, prevent the resolver from destroying things. index.refsWithSiblings[value] = copied @@ -592,6 +597,11 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc locate := func(ref *Reference, refIndex int, sequence []*ReferenceMapped) { located := index.FindComponent(ref.FullDefinition, ref.Node) if located != nil { + + if located.Index == nil { + index.logger.Warn("located component has no index", "component", located.FullDefinition) + } + index.refLock.Lock() // have we already mapped this? if index.allMappedRefs[ref.FullDefinition] == nil { diff --git a/index/find_component.go b/index/find_component.go index d294421..7c3ea34 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -50,7 +50,7 @@ func (index *SpecIndex) FindComponent(componentId string, parent *yaml.Node) *Re } } -func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Reference { +func FindComponent(root *yaml.Node, componentId, absoluteFilePath string, index *SpecIndex) *Reference { // check component for url encoding. if strings.Contains(componentId, "%") { // decode the url. @@ -72,6 +72,12 @@ func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Refer fullDef := fmt.Sprintf("%s%s", absoluteFilePath, componentId) + // TODO: clean this shit up + + newIndexWithUpdatedPath := *index + newIndexWithUpdatedPath.specAbsolutePath = absoluteFilePath + newIndexWithUpdatedPath.AbsoluteFile = absoluteFilePath + // extract properties ref := &Reference{ FullDefinition: fullDef, @@ -79,6 +85,8 @@ func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Refer Name: name, Node: resNode, Path: friendlySearch, + RemoteLocation: absoluteFilePath, + Index: &newIndexWithUpdatedPath, RequiredRefProperties: extractDefinitionRequiredRefProperties(resNode, map[string][]string{}, fullDef), } @@ -89,7 +97,7 @@ func FindComponent(root *yaml.Node, componentId, absoluteFilePath string) *Refer func (index *SpecIndex) FindComponentInRoot(componentId string) *Reference { if index.root != nil { - return FindComponent(index.root, componentId, index.specAbsolutePath) + return FindComponent(index.root, componentId, index.specAbsolutePath, index) } return nil } @@ -139,6 +147,9 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { var parsedDocument *yaml.Node var err error + + idx := index + if ext != "" { // extract the document from the rolodex. @@ -153,6 +164,9 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { index.logger.Error("rolodex file is empty!", "file", absoluteFileLocation) return nil } + if rFile.GetIndex() != nil { + idx = rFile.GetIndex() + } parsedDocument, err = rFile.GetContentAsYAMLNode() if err != nil { @@ -184,6 +198,7 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { FullDefinition: absoluteFileLocation, Definition: fileName, Name: fileName, + Index: idx, Node: parsedDocument, IsRemote: true, RemoteLocation: absoluteFileLocation, @@ -192,7 +207,7 @@ func (index *SpecIndex) lookupRolodex(uri []string) *Reference { } return foundRef } else { - foundRef = FindComponent(parsedDocument, query, absoluteFileLocation) + foundRef = FindComponent(parsedDocument, query, absoluteFileLocation, index) if foundRef != nil { foundRef.IsRemote = true foundRef.RemoteLocation = absoluteFileLocation diff --git a/index/find_component_test.go b/index/find_component_test.go index f9f2c81..62d14e8 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -61,9 +61,14 @@ func TestSpecIndex_CheckCircularIndex(t *testing.T) { index := rolo.GetRootIndex() assert.Nil(t, index.uri) - assert.NotNil(t, index.SearchIndexForReference("second.yaml#/properties/property2")) - assert.NotNil(t, index.SearchIndexForReference("second.yaml")) - assert.Nil(t, index.SearchIndexForReference("fourth.yaml")) + + a, _ := index.SearchIndexForReference("second.yaml#/properties/property2") + b, _ := index.SearchIndexForReference("second.yaml") + c, _ := index.SearchIndexForReference("fourth.yaml") + + assert.NotNil(t, a) + assert.NotNil(t, b) + assert.Nil(t, c) } func TestSpecIndex_performExternalLookup_invalidURL(t *testing.T) { diff --git a/index/index_model.go b/index/index_model.go index cebb0aa..2221485 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -35,6 +35,7 @@ type Reference struct { Circular bool Seen bool IsRemote bool + Index *SpecIndex // index that contains this reference. RemoteLocation string Path string // this won't always be available. RequiredRefProperties map[string][]string // definition names (eg, #/definitions/One) to a list of required properties on this definition which reference that definition @@ -166,6 +167,8 @@ func CreateClosedAPIIndexConfig() *SpecIndexConfig { // quick direct access to paths, operations, tags are all available. No need to walk the entire node tree in rules, // everything is pre-walked if you need it. type SpecIndex struct { + specAbsolutePath string + AbsoluteFile string rolodex *Rolodex // the rolodex is used to fetch remote and file based documents. allRefs map[string]*Reference // all (deduplicated) refs rawSequencedRefs []*Reference // all raw references in sequence as they are scanned, not deduped. @@ -261,7 +264,6 @@ type SpecIndex struct { httpClient *http.Client componentIndexChan chan bool polyComponentIndexChan chan bool - specAbsolutePath string resolver *Resolver cache syncmap.Map built bool diff --git a/index/resolver.go b/index/resolver.go index 1ff229c..90f591d 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -267,7 +267,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j if j.FullDefinition == r.FullDefinition { var foundDup *Reference - foundRef := resolver.specIndex.SearchIndexForReferenceByReference(r) + foundRef, _ := resolver.specIndex.SearchIndexForReferenceByReference(r) if foundRef != nil { foundDup = foundRef } @@ -307,7 +307,7 @@ func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, j if !skip { var original *Reference - foundRef := resolver.specIndex.SearchIndexForReferenceByReference(r) + foundRef, _ := resolver.specIndex.SearchIndexForReferenceByReference(r) if foundRef != nil { original = foundRef } @@ -335,7 +335,7 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDe } for refDefinition := range ref.RequiredRefProperties { - r := resolver.specIndex.SearchIndexForReference(refDefinition) + r, _ := resolver.specIndex.SearchIndexForReference(refDefinition) if initialRef != nil && initialRef.Definition == r.Definition { return true, visitedDefinitions } @@ -497,7 +497,7 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No IsRemote: true, } - locatedRef = resolver.specIndex.SearchIndexForReferenceByReference(searchRef) + locatedRef, _ = resolver.specIndex.SearchIndexForReferenceByReference(searchRef) if locatedRef == nil { _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(value) diff --git a/index/resolver_test.go b/index/resolver_test.go index 0eac6f9..d5f31f5 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -596,7 +596,7 @@ func TestResolver_ResolveComponents_MixedRef(t *testing.T) { resolver := index().GetResolver() assert.Len(t, resolver.GetCircularErrors(), 0) - assert.Equal(t, 3, resolver.GetIndexesVisited()) + assert.Equal(t, 2, resolver.GetIndexesVisited()) // in v0.8.2 a new check was added when indexing, to prevent re-indexing the same file multiple times. assert.Equal(t, 6, resolver.GetRelativesSeen()) diff --git a/index/rolodex_file_loader.go b/index/rolodex_file_loader.go index c5f8334..13dab1e 100644 --- a/index/rolodex_file_loader.go +++ b/index/rolodex_file_loader.go @@ -87,6 +87,7 @@ func (l *LocalFile) Index(config *SpecIndexConfig) (*SpecIndex, error) { index := NewSpecIndexWithConfig(info.RootNode, config) index.specAbsolutePath = l.fullPath + l.index = index return index, nil diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index 473d6ef..1f52245 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -176,13 +176,10 @@ const ( func NewRemoteFSWithConfig(specIndexConfig *SpecIndexConfig) (*RemoteFS, error) { remoteRootURL := specIndexConfig.BaseURL - - // TODO: handle logging - log := specIndexConfig.Logger if log == nil { log = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ - Level: slog.LevelDebug, + Level: slog.LevelError, })) } @@ -324,7 +321,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { // remove from processing i.ProcessingFiles.Delete(remoteParsedURL.Path) - i.logger.Error("Unable to fetch remote document", + i.logger.Error("unable to fetch remote document", "file", remoteParsedURL.Path, "status", response.StatusCode, "resp", string(responseBytes)) return nil, fmt.Errorf("unable to fetch remote document: %s", string(responseBytes)) } @@ -371,8 +368,6 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { copiedCfg.SpecAbsolutePath = remoteParsedURL.String() idx, idxError := remoteFile.Index(&copiedCfg) - i.Files.Store(absolutePath, remoteFile) - if len(remoteFile.data) > 0 { i.logger.Debug("successfully loaded file", "file", absolutePath) } @@ -390,6 +385,7 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { // remove from processing i.ProcessingFiles.Delete(remoteParsedURL.Path) + i.Files.Store(absolutePath, remoteFile) //if !i.remoteRunning { return remoteFile, errors.Join(i.remoteErrors...) diff --git a/index/search_index.go b/index/search_index.go index 1739ecf..b5e76b6 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -4,130 +4,151 @@ package index import ( - "fmt" - "path/filepath" - "strings" + "context" + "fmt" + "path/filepath" + "strings" ) -func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) *Reference { +type ContextKey string - if v, ok := index.cache.Load(fullRef); ok { - return v.(*Reference) - } +const CurrentPathKey ContextKey = "currentPath" - ref := fullRef.FullDefinition - refAlt := ref - absPath := index.specAbsolutePath - if absPath == "" { - absPath = index.config.BasePath - } - var roloLookup string - uri := strings.Split(ref, "#/") - if len(uri) == 2 { - if uri[0] != "" { - if strings.HasPrefix(uri[0], "http") { - roloLookup = fullRef.FullDefinition - } else { - if filepath.IsAbs(uri[0]) { - roloLookup = uri[0] - } else { - if filepath.Ext(absPath) != "" { - absPath = filepath.Dir(absPath) - } - roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) - } - } - } else { - - if filepath.Ext(uri[1]) != "" { - roloLookup = absPath - } else { - roloLookup = "" - } - - ref = fmt.Sprintf("#/%s", uri[1]) - refAlt = fmt.Sprintf("%s#/%s", absPath, uri[1]) - - } - - } else { - if filepath.IsAbs(uri[0]) { - roloLookup = uri[0] - } else { - - if strings.HasPrefix(uri[0], "http") { - roloLookup = ref - } else { - if filepath.Ext(absPath) != "" { - absPath = filepath.Dir(absPath) - } - roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) - } - } - ref = uri[0] - } - - if r, ok := index.allMappedRefs[ref]; ok { - index.cache.Store(ref, r) - return r - } - - if r, ok := index.allMappedRefs[refAlt]; ok { - index.cache.Store(refAlt, r) - return r - } - - // check the rolodex for the reference. - if roloLookup != "" { - rFile, err := index.rolodex.Open(roloLookup) - if err != nil { - return nil - } - - // extract the index from the rolodex file. - idx := rFile.GetIndex() - if index.resolver != nil { - index.resolver.indexesVisited++ - } - if idx != nil { - - // check mapped refs. - if r, ok := idx.allMappedRefs[ref]; ok { - return r - } - - // build a collection of all the inline schemas and search them - // for the reference. - var d []*Reference - d = append(d, idx.allInlineSchemaDefinitions...) - d = append(d, idx.allRefSchemaDefinitions...) - d = append(d, idx.allInlineSchemaObjectDefinitions...) - for _, s := range d { - if s.Definition == ref { - index.cache.Store(ref, s) - return s - } - } - - // does component exist in the root? - node, _ := rFile.GetContentAsYAMLNode() - if node != nil { - found := idx.FindComponent(ref, node) - if found != nil { - index.cache.Store(ref, found) - return found - } - } - } - } - - fmt.Printf("unable to locate reference: %s, within index: %s\n", ref, index.specAbsolutePath) - return nil +func (index *SpecIndex) SearchIndexForReferenceByReference(fullRef *Reference) (*Reference, *SpecIndex) { + r, idx, _ := index.SearchIndexForReferenceByReferenceWithContext(context.Background(), fullRef) + return r, idx } // SearchIndexForReference searches the index for a reference, first looking through the mapped references // and then externalSpecIndex for a match. If no match is found, it will recursively search the child indexes // extracted when parsing the OpenAPI Spec. -func (index *SpecIndex) SearchIndexForReference(ref string) *Reference { - return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref}) +func (index *SpecIndex) SearchIndexForReference(ref string) (*Reference, *SpecIndex) { + return index.SearchIndexForReferenceByReference(&Reference{FullDefinition: ref}) +} + +func (index *SpecIndex) SearchIndexForReferenceWithContext(ctx context.Context, ref string) (*Reference, *SpecIndex, context.Context) { + return index.SearchIndexForReferenceByReferenceWithContext(ctx, &Reference{FullDefinition: ref}) +} + +func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx context.Context, searchRef *Reference) (*Reference, *SpecIndex, context.Context) { + + if v, ok := index.cache.Load(searchRef.FullDefinition); ok { + return v.(*Reference), index, context.WithValue(ctx, CurrentPathKey, v.(*Reference).RemoteLocation) + } + + ref := searchRef.FullDefinition + refAlt := ref + absPath := index.specAbsolutePath + if absPath == "" { + absPath = index.config.BasePath + } + var roloLookup string + uri := strings.Split(ref, "#/") + if len(uri) == 2 { + if uri[0] != "" { + if strings.HasPrefix(uri[0], "http") { + roloLookup = searchRef.FullDefinition + } else { + if filepath.IsAbs(uri[0]) { + roloLookup = uri[0] + } else { + if filepath.Ext(absPath) != "" { + absPath = filepath.Dir(absPath) + } + roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + } + } + } else { + + if filepath.Ext(uri[1]) != "" { + roloLookup = absPath + } else { + roloLookup = "" + } + + ref = fmt.Sprintf("#/%s", uri[1]) + refAlt = fmt.Sprintf("%s#/%s", absPath, uri[1]) + + } + + } else { + if filepath.IsAbs(uri[0]) { + roloLookup = uri[0] + } else { + + if strings.HasPrefix(uri[0], "http") { + roloLookup = ref + } else { + if filepath.Ext(absPath) != "" { + absPath = filepath.Dir(absPath) + } + roloLookup, _ = filepath.Abs(filepath.Join(absPath, uri[0])) + } + } + ref = uri[0] + } + + if r, ok := index.allMappedRefs[ref]; ok { + index.cache.Store(ref, r) + return r, r.Index, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation) + } + + if r, ok := index.allMappedRefs[refAlt]; ok { + index.cache.Store(refAlt, r) + return r, r.Index, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation) + } + + // check the rolodex for the reference. + if roloLookup != "" { + rFile, err := index.rolodex.Open(roloLookup) + if err != nil { + return nil, index, ctx + } + + // extract the index from the rolodex file. + if rFile != nil { + idx := rFile.GetIndex() + if index.resolver != nil { + index.resolver.indexesVisited++ + } + if idx != nil { + + // check mapped refs. + if r, ok := idx.allMappedRefs[ref]; ok { + index.cache.Store(ref, r) + idx.cache.Store(ref, r) + return r, r.Index, context.WithValue(ctx, CurrentPathKey, r.RemoteLocation) + } + + // build a collection of all the inline schemas and search them + // for the reference. + var d []*Reference + d = append(d, idx.allInlineSchemaDefinitions...) + d = append(d, idx.allRefSchemaDefinitions...) + d = append(d, idx.allInlineSchemaObjectDefinitions...) + for _, s := range d { + if s.FullDefinition == ref { + idx.cache.Store(ref, s) + index.cache.Store(ref, s) + return s, s.Index, context.WithValue(ctx, CurrentPathKey, s.RemoteLocation) + } + } + + // does component exist in the root? + node, _ := rFile.GetContentAsYAMLNode() + if node != nil { + found := idx.FindComponent(ref, node) + if found != nil { + idx.cache.Store(ref, found) + index.cache.Store(ref, found) + return found, found.Index, context.WithValue(ctx, CurrentPathKey, found.RemoteLocation) + } + } + } + } + } + + index.logger.Error("unable to locate reference anywhere in the rolodex", "reference", ref) + return nil, index, ctx + } diff --git a/index/search_index_test.go b/index/search_index_test.go index 62dd2a8..4fc6c5e 100644 --- a/index/search_index_test.go +++ b/index/search_index_test.go @@ -18,6 +18,6 @@ func TestSpecIndex_SearchIndexForReference(t *testing.T) { c := CreateOpenAPIIndexConfig() idx := NewSpecIndexWithConfig(&rootNode, c) - ref := idx.SearchIndexForReference("#/components/schemas/Pet") + ref, _ := idx.SearchIndexForReference("#/components/schemas/Pet") assert.NotNil(t, ref) } diff --git a/index/spec_index.go b/index/spec_index.go index 603559d..5245475 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -149,6 +149,14 @@ func (index *SpecIndex) BuildIndex() { index.built = true } +func (index *SpecIndex) GetSpecAbsolutePath() string { + return index.specAbsolutePath +} + +func (index *SpecIndex) GetLogger() *slog.Logger { + return index.logger +} + // GetRootNode returns document root node. func (index *SpecIndex) GetRootNode() *yaml.Node { return index.root diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 915c931..7f43051 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -949,7 +949,7 @@ func TestSpecIndex_lookupFileReference_MultiRes(t *testing.T) { index := rolo.GetRootIndex() //index.seenRemoteSources = make(map[string]*yaml.Node) absoluteRef, _ := filepath.Abs("embie.yaml#/naughty") - fRef := index.SearchIndexForReference(absoluteRef) + fRef, _ := index.SearchIndexForReference(absoluteRef) assert.NotNil(t, fRef) } diff --git a/index/utility_methods.go b/index/utility_methods.go index 2baf8a4..5393b7e 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -347,7 +347,7 @@ func (index *SpecIndex) scanOperationParams(params []*yaml.Node, pathItemNode *y paramRef := index.allMappedRefs[paramRefName] if paramRef == nil { // could be in the rolodex - ref := index.SearchIndexForReference(paramRefName) + ref, _ := index.SearchIndexForReference(paramRefName) if ref != nil { paramRef = ref }