From c320b2b9f1f93ee1363b034ace201298c0a623c0 Mon Sep 17 00:00:00 2001 From: Dave Shanley Date: Fri, 12 Aug 2022 09:41:03 -0400 Subject: [PATCH] Operating at speed now. Stripe API is slowest to build at about 1.2s, the depth of the schemas on that spec are insane. The only way to speed things up here is to NOT wait for threads to finish before returning the model, and this creates a broken guarantee as the model won't be finished building by the time its returned to the consumer. When running tests against this - they fail randomly. depending on which thread reaches the finish first, before trying to be read. This isn't an option, so for now, this is as fast as she will get, which is pretty good. 5ms for petstore and about 250ms for k8s. --- datamodel/low/3.0/info.go | 2 +- datamodel/low/3.0/path.go | 2 +- datamodel/low/3.0/response.go | 127 ++++++++++++++++---------------- index/spec_index.go | 8 +- openapi/create_document.go | 4 - openapi/create_document_test.go | 11 +++ 6 files changed, 80 insertions(+), 74 deletions(-) diff --git a/datamodel/low/3.0/info.go b/datamodel/low/3.0/info.go index 605f729..56c06b8 100644 --- a/datamodel/low/3.0/info.go +++ b/datamodel/low/3.0/info.go @@ -32,7 +32,7 @@ func (i *Info) Build(root *yaml.Node) error { license := License{} _, kln, ln := utils.FindKeyNodeFull("license", root.Content) go BuildModelAsync(ln, &license, &wg, &errs) - wg.Wait() + //wg.Wait() i.Contact = low.NodeReference[*Contact]{Value: &contact, ValueNode: cn, KeyNode: kln} i.License = low.NodeReference[*License]{Value: &license, ValueNode: ln, KeyNode: kln} return nil diff --git a/datamodel/low/3.0/path.go b/datamodel/low/3.0/path.go index f71dee7..9a7e364 100644 --- a/datamodel/low/3.0/path.go +++ b/datamodel/low/3.0/path.go @@ -132,7 +132,7 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error { var op Operation wg.Add(1) - BuildModelAsync(pathNode, &op, &wg, &errors) + go BuildModelAsync(pathNode, &op, &wg, &errors) opRef := low.NodeReference[*Operation]{ Value: &op, diff --git a/datamodel/low/3.0/response.go b/datamodel/low/3.0/response.go index 725059f..9e43471 100644 --- a/datamodel/low/3.0/response.go +++ b/datamodel/low/3.0/response.go @@ -1,99 +1,98 @@ package v3 import ( - "github.com/pb33f/libopenapi/datamodel/low" - "github.com/pb33f/libopenapi/index" - "github.com/pb33f/libopenapi/utils" - "gopkg.in/yaml.v3" + "github.com/pb33f/libopenapi/datamodel/low" + "github.com/pb33f/libopenapi/index" + "github.com/pb33f/libopenapi/utils" + "gopkg.in/yaml.v3" ) const ( - LinksLabel = "links" + LinksLabel = "links" ) type Responses struct { - Codes map[low.KeyReference[string]]low.ValueReference[*Response] - Default low.NodeReference[*Response] + Codes map[low.KeyReference[string]]low.ValueReference[*Response] + Default low.NodeReference[*Response] } func (r *Responses) Build(root *yaml.Node, idx *index.SpecIndex) error { - if utils.IsNodeMap(root) { - codes, err := ExtractMapFlatNoLookup[*Response](root, idx) - if err != nil { - return err - } - if codes != nil { - r.Codes = codes - } - } - return nil + if utils.IsNodeMap(root) { + codes, err := ExtractMapFlatNoLookup[*Response](root, idx) + if err != nil { + return err + } + if codes != nil { + r.Codes = codes + } + } + return nil } func (r *Responses) FindResponseByCode(code string) *low.ValueReference[*Response] { - return FindItemInMap[*Response](code, r.Codes) + return FindItemInMap[*Response](code, r.Codes) } type Response struct { - Description low.NodeReference[string] - Headers low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Header]] - Content low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*MediaType]] - Extensions map[low.KeyReference[string]]low.ValueReference[any] - Links low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Link]] + Description low.NodeReference[string] + Headers low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Header]] + Content low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*MediaType]] + Extensions map[low.KeyReference[string]]low.ValueReference[any] + Links low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Link]] } func (r *Response) FindContent(cType string) *low.ValueReference[*MediaType] { - return FindItemInMap[*MediaType](cType, r.Content.Value) + return FindItemInMap[*MediaType](cType, r.Content.Value) } func (r *Response) FindHeader(hType string) *low.ValueReference[*Header] { - return FindItemInMap[*Header](hType, r.Headers.Value) + return FindItemInMap[*Header](hType, r.Headers.Value) } func (r *Response) FindLink(hType string) *low.ValueReference[*Link] { - return FindItemInMap[*Link](hType, r.Links.Value) + return FindItemInMap[*Link](hType, r.Links.Value) } func (r *Response) Build(root *yaml.Node, idx *index.SpecIndex) error { - r.Extensions = ExtractExtensions(root) + r.Extensions = ExtractExtensions(root) - // extract headers - headers, lN, kN, err := ExtractMapFlat[*Header](HeadersLabel, root, idx) - if err != nil { - return err - } - if headers != nil { - r.Headers = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Header]]{ - Value: headers, - KeyNode: lN, - ValueNode: kN, - } - } + // extract headers + headers, lN, kN, err := ExtractMapFlat[*Header](HeadersLabel, root, idx) + if err != nil { + return err + } + if headers != nil { + r.Headers = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Header]]{ + Value: headers, + KeyNode: lN, + ValueNode: kN, + } + } - // handle content, if set. - con, clN, cN, cErr := ExtractMapFlat[*MediaType](ContentLabel, root, idx) - if cErr != nil { - return cErr - } - if con != nil { - r.Content = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*MediaType]]{ - Value: con, - KeyNode: clN, - ValueNode: cN, - } - } + con, clN, cN, cErr := ExtractMapFlat[*MediaType](ContentLabel, root, idx) + if cErr != nil { + return cErr + } + if con != nil { + r.Content = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*MediaType]]{ + Value: con, + KeyNode: clN, + ValueNode: cN, + } + } - // handle links if set - links, linkLabel, linkValue, lErr := ExtractMapFlat[*Link](LinksLabel, root, idx) - if lErr != nil { - return lErr - } - if links != nil { - r.Links = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Link]]{ - Value: links, - KeyNode: linkLabel, - ValueNode: linkValue, - } - } + // handle links if set + links, linkLabel, linkValue, lErr := ExtractMapFlat[*Link](LinksLabel, root, idx) + if lErr != nil { + return lErr + } + if links != nil { + r.Links = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Link]]{ + Value: links, + KeyNode: linkLabel, + ValueNode: linkValue, + } + } - return nil + return nil } diff --git a/index/spec_index.go b/index/spec_index.go index dfe1776..23295fa 100644 --- a/index/spec_index.go +++ b/index/spec_index.go @@ -886,7 +886,7 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { return index.globalCallbacksCount } - index.pathRefsLock.Lock() + //index.pathRefsLock.Lock() for path, p := range index.pathRefs { for _, m := range p { @@ -918,7 +918,7 @@ func (index *SpecIndex) GetGlobalCallbacksCount() int { } } } - index.pathRefsLock.Unlock() + //index.pathRefsLock.Unlock() return index.globalCallbacksCount } @@ -932,7 +932,7 @@ func (index *SpecIndex) GetGlobalLinksCount() int { return index.globalLinksCount } - index.pathRefsLock.Lock() + //index.pathRefsLock.Lock() for path, p := range index.pathRefs { for _, m := range p { @@ -963,7 +963,7 @@ func (index *SpecIndex) GetGlobalLinksCount() int { } } } - index.pathRefsLock.Unlock() + //index.pathRefsLock.Unlock() return index.globalLinksCount } diff --git a/openapi/create_document.go b/openapi/create_document.go index 9cf8ee6..3c41b6a 100644 --- a/openapi/create_document.go +++ b/openapi/create_document.go @@ -15,10 +15,6 @@ func CreateDocument(info *datamodel.SpecInfo) (*v3.Document, error) { // build an index idx := index.NewSpecIndex(info.RootNode) - //rsolvr := resolver.NewResolver() - - // todo handle errors - //rsolvr.Resolve() var wg sync.WaitGroup var errors []error diff --git a/openapi/create_document_test.go b/openapi/create_document_test.go index 2b9898c..c3afd54 100644 --- a/openapi/create_document_test.go +++ b/openapi/create_document_test.go @@ -35,6 +35,17 @@ func BenchmarkCreateDocument_Stripe(b *testing.B) { } } +func BenchmarkCreateDocument_Petstore(b *testing.B) { + data, _ := ioutil.ReadFile("../test_specs/petstorev3.json") + info, _ := datamodel.ExtractSpecInfo(data) + for i := 0; i < b.N; i++ { + _, err := CreateDocument(info) + if err != nil { + panic("this should not error") + } + } +} + func TestCreateDocument(t *testing.T) { assert.Equal(t, "3.0.1", doc.Version.Value) assert.Equal(t, "Burger Shop", doc.Info.Value.Title.Value)