LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-06 20:47:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Huge performance increase with building.

Browse Source 
Using some designs unearthed from building the higher level model, I have brough that design down to the lower level to speed things up. It only took 8 years, but finally, I think I have mastered mult-threading. No more deadlocks, and no more need for waitgroups for everything.
This commit is contained in:
Dave Shanley
2022-08-22 09:46:44 -04:00
parent 6c2de6c151
commit 1a71f449ff
13 changed files with 562 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
datamodel/high/3.0/components.go
View File

@@ -24,6 +24,10 @@ const (
var seenSchemas map[string]*Schema
func init() {
clearSchemas()
}
func clearSchemas() {
seenSchemas = make(map[string]*Schema)
}
@@ -108,7 +112,7 @@ func NewComponents(comp *low.Components) *Components {
}
for k, v := range comp.Schemas.Value {
go buildSchema(k, v.Value, schemaChan)
go buildSchema(k, v, schemaChan)
}
totalComponents := len(comp.Callbacks.Value) + len(comp.Links.Value) + len(comp.Responses.Value) +
@@ -169,12 +173,13 @@ func buildComponent[N any, O any](comp int, key string, orig O, c chan component
c <- componentResult[N]{comp: comp, res: f(orig), key: key}
}
func buildSchema(key lowmodel.KeyReference[string], orig *low.Schema, c chan componentResult[*Schema]) {
func buildSchema(key lowmodel.KeyReference[string], orig lowmodel.ValueReference[*low.Schema], c chan componentResult[*Schema]) {
var sch *Schema
if ss := getSeenSchema(key.GenerateMapKey()); ss != nil {
if ss := getSeenSchema(orig.GenerateMapKey()); ss != nil {
sch = ss
} else {
sch = NewSchema(orig)
sch = NewSchema(orig.Value)
addSeenSchema(orig.GenerateMapKey(), sch)
}
c <- componentResult[*Schema]{res: sch, key: key.Value}
}

132
datamodel/low/3.0/components.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"sync"
)
const (
@@ -16,6 +17,31 @@ const (
SchemasLabel = "schemas"
)
var seenSchemas map[string]*Schema
func init() {
clearSchemas()
}
func clearSchemas() {
seenSchemas = make(map[string]*Schema)
}
var seenSchemaLock sync.RWMutex
func addSeenSchema(key string, schema *Schema) {
defer seenSchemaLock.Unlock()
seenSchemaLock.Lock()
if seenSchemas[key] == nil {
seenSchemas[key] = schema
}
}
func getSeenSchema(key string) *Schema {
defer seenSchemaLock.Unlock()
seenSchemaLock.Lock()
return seenSchemas[key]
}
type Components struct {
Schemas low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Schema]]
Responses low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Response]]
@@ -94,72 +120,54 @@ func (co *Components) Build(root *yaml.Node, idx *index.SpecIndex) error {
n := 0
total := 9
var stateCheck = func() bool {
n++
if n == total {
return true
}
return false
}
allDone:
for {
for n < total {
select {
case buildError := <-errorChan:
return buildError
case <-skipChan:
if stateCheck() {
break allDone
}
n++
case params := <-paramChan:
co.Parameters = params
if stateCheck() {
break allDone
}
n++
case schemas := <-schemaChan:
co.Schemas = schemas
if stateCheck() {
break allDone
}
cacheSchemas(co.Schemas.Value)
n++
case responses := <-responsesChan:
co.Responses = responses
if stateCheck() {
break allDone
}
n++
case examples := <-examplesChan:
co.Examples = examples
if stateCheck() {
break allDone
}
n++
case reqBody := <-requestBodiesChan:
co.RequestBodies = reqBody
if stateCheck() {
break allDone
}
n++
case headers := <-headersChan:
co.Headers = headers
if stateCheck() {
break allDone
}
n++
case sScheme := <-securitySchemesChan:
co.SecuritySchemes = sScheme
if stateCheck() {
break allDone
}
n++
case links := <-linkChan:
co.Links = links
if stateCheck() {
break allDone
}
n++
case callbacks := <-callbackChan:
co.Callbacks = callbacks
if stateCheck() {
break allDone
n++
}
}
return nil
}
func cacheSchemas(sch map[low.KeyReference[string]]low.ValueReference[*Schema]) {
for _, v := range sch {
addSeenSchema(v.GenerateMapKey(), v.Value)
}
}
return nil
type componentBuildResult[T any] struct {
k low.KeyReference[string]
v low.ValueReference[T]
}
func extractComponentValues[T low.Buildable[N], N any](label string, root *yaml.Node,
@@ -175,25 +183,47 @@ func extractComponentValues[T low.Buildable[N], N any](label string, root *yaml.
errorChan <- fmt.Errorf("node is array, cannot be used in components: line %d, column %d", nodeValue.Line, nodeValue.Column)
return
}
// for every component, build in a new thread!
bChan := make(chan componentBuildResult[T])
var buildComponent = func(label *yaml.Node, value *yaml.Node, c chan componentBuildResult[T], ec chan<- error) {
var n T = new(N)
_ = low.BuildModel(value, n)
err := n.Build(value, idx)
if err != nil {
ec <- err
return
}
c <- componentBuildResult[T]{
k: low.KeyReference[string]{
KeyNode: label,
Value: label.Value,
},
v: low.ValueReference[T]{
Value: n,
ValueNode: value,
},
}
}
for i, v := range nodeValue.Content {
if i%2 == 0 {
currentLabel = v
continue
}
var n T = new(N)
_ = low.BuildModel(v, n)
err := n.Build(v, idx)
if err != nil {
errorChan <- err
go buildComponent(currentLabel, v, bChan, errorChan)
}
componentValues[low.KeyReference[string]{
KeyNode: currentLabel,
Value: currentLabel.Value,
}] = low.ValueReference[T]{
Value: n,
ValueNode: v,
totalComponents := len(nodeValue.Content) / 2
completedComponents := 0
for completedComponents < totalComponents {
select {
case r := <-bChan:
componentValues[r.k] = r.v
completedComponents++
}
}
results := low.NodeReference[map[low.KeyReference[string]]low.ValueReference[T]]{
KeyNode: nodeLabel,
ValueNode: nodeValue,

14
datamodel/low/3.0/create_document.go
View File

@@ -4,18 +4,25 @@ import (
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/resolver"
"github.com/pb33f/libopenapi/utils"
"sync"
)
func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) {
// clean state
clearSchemas()
doc := Document{Version: low.ValueReference[string]{Value: info.Version, ValueNode: info.RootNode}}
// build an index
idx := index.NewSpecIndex(info.RootNode)
doc.Index = idx
// create resolver and check for circular references.
resolve := resolver.NewResolver(idx)
_ = resolve.CheckForCircularReferences()
var wg sync.WaitGroup
var errors []error
@@ -32,24 +39,23 @@ func CreateDocument(info *datamodel.SpecInfo) (*Document, []error) {
wg.Done()
}
extractionFuncs := []func(i *datamodel.SpecInfo, d *Document, idx *index.SpecIndex) error{
extractInfo,
extractServers,
extractTags,
extractPaths,
extractComponents,
extractSecurity,
extractExternalDocs,
extractPaths,
}
wg.Add(len(extractionFuncs))
for _, f := range extractionFuncs {
go runExtraction(info, &doc, idx, f, &errors, &wg)
}
wg.Wait()
return &doc, errors
}
func extractInfo(info *datamodel.SpecInfo, doc *Document, idx *index.SpecIndex) error {

52
datamodel/low/3.0/create_document_test.go
View File

@@ -9,7 +9,10 @@ import (
var doc *Document
func init() {
func initTest() {
if doc != nil {
return
}
data, _ := ioutil.ReadFile("../../../test_specs/burgershop.openapi.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
var err []error
@@ -27,8 +30,8 @@ func BenchmarkCreateDocument(b *testing.B) {
}
}
func BenchmarkCreateDocument_Stripe(b *testing.B) {
data, _ := ioutil.ReadFile("../../../test_specs/stripe.yaml")
func BenchmarkCreateDocument_Circular(b *testing.B) {
data, _ := ioutil.ReadFile("../../../test_specs/circular-tests.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
for i := 0; i < b.N; i++ {
_, err := CreateDocument(info)
@@ -38,6 +41,32 @@ func BenchmarkCreateDocument_Stripe(b *testing.B) {
}
}
func BenchmarkCreateDocument_k8s(b *testing.B) {
data, _ := ioutil.ReadFile("../../../test_specs/k8s.json")
info, _ := datamodel.ExtractSpecInfo(data)
for i := 0; i < b.N; i++ {
_, err := CreateDocument(info)
if err != nil {
panic("this should not error")
}
}
}
func BenchmarkCreateDocument_Stripe(b *testing.B) {
for i := 0; i < b.N; i++ {
data, _ := ioutil.ReadFile("../../../test_specs/stripe.yaml")
info, _ := datamodel.ExtractSpecInfo(data)
_, err := CreateDocument(info)
if err != nil {
panic("this should not error")
}
}
}
func BenchmarkCreateDocument_Petstore(b *testing.B) {
data, _ := ioutil.ReadFile("../../../test_specs/petstorev3.json")
info, _ := datamodel.ExtractSpecInfo(data)
@@ -50,12 +79,14 @@ func BenchmarkCreateDocument_Petstore(b *testing.B) {
}
func TestCreateDocument(t *testing.T) {
initTest()
assert.Equal(t, "3.0.1", doc.Version.Value)
assert.Equal(t, "Burger Shop", doc.Info.Value.Title.Value)
assert.NotEmpty(t, doc.Info.Value.Title.Value)
}
func TestCreateDocument_Info(t *testing.T) {
initTest()
assert.Equal(t, "https://pb33f.io", doc.Info.Value.TermsOfService.Value)
assert.Equal(t, "pb33f", doc.Info.Value.Contact.Value.Name.Value)
assert.Equal(t, "buckaroo@pb33f.io", doc.Info.Value.Contact.Value.Email.Value)
@@ -65,6 +96,7 @@ func TestCreateDocument_Info(t *testing.T) {
}
func TestCreateDocument_Servers(t *testing.T) {
initTest()
assert.Len(t, doc.Servers.Value, 2)
server1 := doc.Servers.Value[0].Value
server2 := doc.Servers.Value[1].Value
@@ -89,6 +121,7 @@ func TestCreateDocument_Servers(t *testing.T) {
}
func TestCreateDocument_Tags(t *testing.T) {
initTest()
assert.Len(t, doc.Tags.Value, 2)
// tag1
@@ -134,6 +167,7 @@ func TestCreateDocument_Tags(t *testing.T) {
}
func TestCreateDocument_Paths(t *testing.T) {
initTest()
assert.Len(t, doc.Paths.Value.PathItems, 5)
burgerId := doc.Paths.Value.FindPath("/burgers/{burgerId}")
assert.NotNil(t, burgerId)
@@ -262,6 +296,7 @@ func TestCreateDocument_Paths(t *testing.T) {
}
func TestCreateDocument_Components_Schemas(t *testing.T) {
initTest()
components := doc.Components.Value
assert.NotNil(t, components)
@@ -286,6 +321,7 @@ func TestCreateDocument_Components_Schemas(t *testing.T) {
}
func TestCreateDocument_Components_SecuritySchemes(t *testing.T) {
initTest()
components := doc.Components.Value
securitySchemes := components.SecuritySchemes.Value
assert.Len(t, securitySchemes, 3)
@@ -314,6 +350,7 @@ func TestCreateDocument_Components_SecuritySchemes(t *testing.T) {
}
func TestCreateDocument_Components_Responses(t *testing.T) {
initTest()
components := doc.Components.Value
responses := components.Responses.Value
assert.Len(t, responses, 1)
@@ -326,6 +363,7 @@ func TestCreateDocument_Components_Responses(t *testing.T) {
}
func TestCreateDocument_Components_Examples(t *testing.T) {
initTest()
components := doc.Components.Value
examples := components.Examples.Value
assert.Len(t, examples, 1)
@@ -337,6 +375,7 @@ func TestCreateDocument_Components_Examples(t *testing.T) {
}
func TestCreateDocument_Components_RequestBodies(t *testing.T) {
initTest()
components := doc.Components.Value
requestBodies := components.RequestBodies.Value
assert.Len(t, requestBodies, 1)
@@ -348,6 +387,7 @@ func TestCreateDocument_Components_RequestBodies(t *testing.T) {
}
func TestCreateDocument_Components_Headers(t *testing.T) {
initTest()
components := doc.Components.Value
headers := components.Headers.Value
assert.Len(t, headers, 1)
@@ -359,6 +399,7 @@ func TestCreateDocument_Components_Headers(t *testing.T) {
}
func TestCreateDocument_Components_Links(t *testing.T) {
initTest()
components := doc.Components.Value
links := components.Links.Value
assert.Len(t, links, 2)
@@ -373,6 +414,7 @@ func TestCreateDocument_Components_Links(t *testing.T) {
}
func TestCreateDocument_Doc_Security(t *testing.T) {
initTest()
security := doc.Security.Value
assert.NotNil(t, security)
assert.Len(t, security.ValueRequirements, 1)
@@ -382,6 +424,7 @@ func TestCreateDocument_Doc_Security(t *testing.T) {
}
func TestCreateDocument_Callbacks(t *testing.T) {
initTest()
callbacks := doc.Components.Value.Callbacks.Value
assert.Len(t, callbacks, 1)
@@ -396,6 +439,7 @@ func TestCreateDocument_Callbacks(t *testing.T) {
}
func TestCreateDocument_Component_Discriminator(t *testing.T) {
initTest()
components := doc.Components.Value
dsc := components.FindSchema("Drink").Value.Discriminator.Value
@@ -406,6 +450,7 @@ func TestCreateDocument_Component_Discriminator(t *testing.T) {
}
func TestCreateDocument_CheckAdditionalProperties_Schema(t *testing.T) {
initTest()
components := doc.Components.Value
d := components.FindSchema("Dressing")
assert.NotNil(t, d.Value.AdditionalProperties.Value)
@@ -417,6 +462,7 @@ func TestCreateDocument_CheckAdditionalProperties_Schema(t *testing.T) {
}
func TestCreateDocument_CheckAdditionalProperties_Bool(t *testing.T) {
initTest()
components := doc.Components.Value
d := components.FindSchema("Drink")
assert.NotNil(t, d.Value.AdditionalProperties.Value)

72
datamodel/low/3.0/path.go
View File

@@ -49,6 +49,46 @@ func (p *Paths) Build(root *yaml.Node, idx *index.SpecIndex) error {
pathsMap := make(map[low.KeyReference[string]]low.ValueReference[*PathItem])
// build each new path, in a new thread.
type pathBuildResult struct {
k low.KeyReference[string]
v low.ValueReference[*PathItem]
}
bChan := make(chan pathBuildResult)
eChan := make(chan error)
var buildPathItem = func(cNode, pNode *yaml.Node, b chan<- pathBuildResult, e chan<- error) {
if ok, _, _ := utils.IsNodeRefValue(pNode); ok {
r := low.LocateRefNode(pNode, idx)
if r != nil {
pNode = r
} else {
e <- fmt.Errorf("path item build failed: cannot find reference: %s at line %d, col %d",
pNode.Content[1].Value, pNode.Content[1].Line, pNode.Content[1].Column)
return
}
}
path := new(PathItem)
_ = low.BuildModel(pNode, path)
err := path.Build(pNode, idx)
if err != nil {
e <- err
return
}
b <- pathBuildResult{
k: low.KeyReference[string]{
Value: cNode.Value,
KeyNode: cNode,
},
v: low.ValueReference[*PathItem]{
Value: path,
ValueNode: pNode,
},
}
}
pathCount := 0
for i, pathNode := range root.Content {
if strings.HasPrefix(strings.ToLower(pathNode.Value), "x-") {
skip = true
@@ -62,34 +102,20 @@ func (p *Paths) Build(root *yaml.Node, idx *index.SpecIndex) error {
currentNode = pathNode
continue
}
if ok, _, _ := utils.IsNodeRefValue(pathNode); ok {
r := low.LocateRefNode(pathNode, idx)
if r != nil {
pathNode = r
} else {
return fmt.Errorf("path item build failed: cannot find reference: %s at line %d, col %d",
pathNode.Content[1].Value, pathNode.Content[1].Line, pathNode.Content[1].Column)
}
pathCount++
go buildPathItem(currentNode, pathNode, bChan, eChan)
}
var path = PathItem{}
_ = low.BuildModel(pathNode, &path)
err := path.Build(pathNode, idx)
if err != nil {
completedItems := 0
for completedItems < pathCount {
select {
case err := <-eChan:
return err
}
// add bulk here
pathsMap[low.KeyReference[string]{
Value: currentNode.Value,
KeyNode: currentNode,
}] = low.ValueReference[*PathItem]{
Value: &path,
ValueNode: pathNode,
case res := <-bChan:
completedItems++
pathsMap[res.k] = res.v
}
}
p.PathItems = pathsMap
return nil

8
datamodel/low/3.0/path_item.go
View File

@@ -174,17 +174,13 @@ func (p *PathItem) Build(root *yaml.Node, idx *index.SpecIndex) error {
}
n := 0
allDone:
for {
total := len(ops)
for n < total {
select {
case buildError := <-opErrorChan:
return buildError
case <-opBuildChan:
n++
if n == len(ops) {
break allDone
}
}
}

108
datamodel/low/3.0/schema.go
View File

@@ -69,11 +69,15 @@ func (s *Schema) Build(root *yaml.Node, idx *index.SpecIndex) error {
}
func (s *Schema) BuildLevel(root *yaml.Node, idx *index.SpecIndex, level int) error {
level++
if low.IsCircular(root, idx) {
return nil // circular references cannot be built.
}
if level > 30 {
return fmt.Errorf("schema is too nested to continue: %d levels deep, is too deep", level) // we're done, son! too fricken deep.
}
level++
if h, _, _ := utils.IsNodeRefValue(root); h {
ref := low.LocateRefNode(root, idx)
if ref != nil {
@@ -135,11 +139,51 @@ func (s *Schema) BuildLevel(root *yaml.Node, idx *index.SpecIndex, level int) er
s.XML = low.NodeReference[*XML]{Value: &xml, KeyNode: xmlLabel, ValueNode: xmlNode}
}
// for property, build in a new thread!
bChan := make(chan schemaBuildResult)
eChan := make(chan error)
var buildProperty = func(label *yaml.Node, value *yaml.Node, c chan schemaBuildResult, ec chan<- error) {
// have we seen this before?
seen := getSeenSchema(fmt.Sprintf("%d:%d", value.Line, value.Column))
if seen != nil {
c <- schemaBuildResult{
k: low.KeyReference[string]{
KeyNode: label,
Value: label.Value,
},
v: low.ValueReference[*Schema]{
Value: seen,
ValueNode: value,
},
}
return
}
p := new(Schema)
_ = low.BuildModel(value, p)
err := p.BuildLevel(value, idx, level)
if err != nil {
ec <- err
return
}
c <- schemaBuildResult{
k: low.KeyReference[string]{
KeyNode: label,
Value: label.Value,
},
v: low.ValueReference[*Schema]{
Value: p,
ValueNode: value,
},
}
}
// handle properties
_, propLabel, propsNode := utils.FindKeyNodeFull(PropertiesLabel, root.Content)
if propsNode != nil {
propertyMap := make(map[low.KeyReference[string]]low.ValueReference[*Schema])
var currentProp *yaml.Node
totalProps := 0
for i, prop := range propsNode.Content {
if i%2 == 0 {
currentProp = prop
@@ -156,19 +200,17 @@ func (s *Schema) BuildLevel(root *yaml.Node, idx *index.SpecIndex, level int) er
prop.Content[1].Value, prop.Content[1].Column, prop.Content[1].Line)
}
}
var property Schema
_ = low.BuildModel(prop, &property)
err := property.BuildLevel(prop, idx, level)
if err != nil {
return err
totalProps++
go buildProperty(currentProp, prop, bChan, eChan)
}
propertyMap[low.KeyReference[string]{
Value: currentProp.Value,
KeyNode: currentProp,
}] = low.ValueReference[*Schema]{
Value: &property,
ValueNode: prop,
completedProps := 0
for completedProps < totalProps {
select {
case err := <-eChan:
return err
case res := <-bChan:
completedProps++
propertyMap[res.k] = res.v
}
}
s.Properties = low.NodeReference[map[low.KeyReference[string]]low.ValueReference[*Schema]]{
@@ -233,6 +275,11 @@ func (s *Schema) BuildLevel(root *yaml.Node, idx *index.SpecIndex, level int) er
return nil
}
type schemaBuildResult struct {
k low.KeyReference[string]
v low.ValueReference[*Schema]
}
func (s *Schema) extractExtensions(root *yaml.Node) {
s.Extensions = low.ExtractExtensions(root)
}
@@ -247,9 +294,9 @@ func buildSchema(schemas *[]low.NodeReference[*Schema], attribute string, rootNo
}
if valueNode != nil {
var build = func(kn *yaml.Node, vn *yaml.Node) *low.NodeReference[*Schema] {
var schema Schema
build := func(kn *yaml.Node, vn *yaml.Node) *low.NodeReference[*Schema] {
schema := new(Schema)
if h, _, _ := utils.IsNodeRefValue(vn); h {
ref := low.LocateRefNode(vn, idx)
if ref != nil {
@@ -261,14 +308,28 @@ func buildSchema(schemas *[]low.NodeReference[*Schema], attribute string, rootNo
}
}
_ = low.BuildModel(vn, &schema)
seen := getSeenSchema(fmt.Sprintf("%d:%d", vn.Line, vn.Column))
if seen != nil {
return &low.NodeReference[*Schema]{
Value: seen,
KeyNode: kn,
ValueNode: vn,
}
}
_ = low.BuildModel(vn, schema)
// add schema before we build, so it doesn't get stuck in an infinite loop.
addSeenSchema(fmt.Sprintf("%d:%d", vn.Line, vn.Column), schema)
err := schema.BuildLevel(vn, idx, level)
if err != nil {
*errors = append(*errors, err)
return nil
}
return &low.NodeReference[*Schema]{
Value: &schema,
Value: schema,
KeyNode: kn,
ValueNode: vn,
}
@@ -292,8 +353,8 @@ func buildSchema(schemas *[]low.NodeReference[*Schema], attribute string, rootNo
}
}
if utils.IsNodeArray(valueNode) {
//fmt.Println("polymorphic looping sucks dude.")
for _, vn := range valueNode.Content {
if h, _, _ := utils.IsNodeRefValue(vn); h {
ref := low.LocateRefNode(vn, idx)
if ref != nil {
@@ -312,7 +373,6 @@ func buildSchema(schemas *[]low.NodeReference[*Schema], attribute string, rootNo
}
}
//wg.Done()
return labelNode, valueNode
}
@@ -345,12 +405,20 @@ func ExtractSchema(root *yaml.Node, idx *index.SpecIndex) (*low.NodeReference[*S
}
if schNode != nil {
// check if schema has already been built.
seen := getSeenSchema(fmt.Sprintf("%d:%d", schNode.Line, schNode.Column))
if seen != nil {
return &low.NodeReference[*Schema]{Value: seen, KeyNode: schLabel, ValueNode: schNode}, nil
}
var schema Schema
_ = low.BuildModel(schNode, &schema)
err := schema.Build(schNode, idx)
addSeenSchema(fmt.Sprintf("%d:%d", schNode.Line, schNode.Column), &schema)
if err != nil {
return nil, err
}
return &low.NodeReference[*Schema]{Value: &schema, KeyNode: schLabel, ValueNode: schNode}, nil
}
return nil, nil

142
datamodel/low/3.0/schema_test.go
View File

@@ -9,7 +9,7 @@ import (
)
func Test_Schema(t *testing.T) {
clearSchemas()
testSpec := `type: object
description: something object
discriminator:
@@ -221,7 +221,7 @@ additionalProperties: true `
}
func TestSchema_BuildLevel_TooDeep(t *testing.T) {
clearSchemas()
// if you design data models like this, you're doing it fucking wrong. Seriously. why, what is so complex about a model
// that it needs to be 30+ levels deep? I have seen this shit in the wild, it's unreadable, un-parsable garbage.
yml := `type: object
@@ -339,7 +339,7 @@ properties:
}
func TestSchema_Build_ErrorAdditionalProps(t *testing.T) {
clearSchemas()
yml := `additionalProperties:
$ref: #borko`
@@ -357,19 +357,19 @@ func TestSchema_Build_ErrorAdditionalProps(t *testing.T) {
}
func TestSchema_Build_PropsLookup(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
properties:
aValue:
$ref: '#/components/schemas/Something'`
@@ -385,19 +385,19 @@ properties:
}
func TestSchema_Build_PropsLookup_Fail(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
properties:
aValue:
$ref: '#/bork'`
@@ -412,8 +412,8 @@ properties:
}
func Test_Schema_Polymorphism_Array_Ref(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
type: object
@@ -425,11 +425,11 @@ func Test_Schema_Polymorphism_Array_Ref(t *testing.T) {
example: anything`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
- $ref: '#/components/schemas/Something'
oneOf:
@@ -460,8 +460,8 @@ items:
}
func Test_Schema_Polymorphism_Array_Ref_Fail(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
type: object
@@ -473,11 +473,11 @@ func Test_Schema_Polymorphism_Array_Ref_Fail(t *testing.T) {
example: anything`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
- $ref: '#/components/schemas/Missing'
oneOf:
@@ -502,8 +502,8 @@ items:
}
func Test_Schema_Polymorphism_Map_Ref(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
type: object
@@ -515,11 +515,11 @@ func Test_Schema_Polymorphism_Map_Ref(t *testing.T) {
example: anything`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
$ref: '#/components/schemas/Something'
oneOf:
@@ -550,8 +550,8 @@ items:
}
func Test_Schema_Polymorphism_Map_Ref_Fail(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
type: object
@@ -563,11 +563,11 @@ func Test_Schema_Polymorphism_Map_Ref_Fail(t *testing.T) {
example: anything`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
$ref: '#/components/schemas/Missing'
oneOf:
@@ -592,18 +592,19 @@ items:
}
func Test_Schema_Polymorphism_BorkParent(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
$ref: #borko`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
$ref: '#/components/schemas/Something'`
@@ -620,18 +621,19 @@ allOf:
}
func Test_Schema_Polymorphism_BorkChild(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
$ref: #borko`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
$ref: #borko`
@@ -648,8 +650,9 @@ allOf:
}
func Test_Schema_Polymorphism_RefMadness(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
$ref: '#/components/schemas/Else'
@@ -657,11 +660,11 @@ func Test_Schema_Polymorphism_RefMadness(t *testing.T) {
description: madness`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
$ref: '#/components/schemas/Something'`
@@ -681,8 +684,9 @@ allOf:
}
func Test_Schema_Polymorphism_RefMadnessBork(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
$ref: '#/components/schemas/Else'
@@ -690,11 +694,11 @@ func Test_Schema_Polymorphism_RefMadnessBork(t *testing.T) {
$ref: #borko`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `type: object
yml = `type: object
allOf:
$ref: '#/components/schemas/Something'`
@@ -711,10 +715,11 @@ allOf:
}
func Test_Schema_Polymorphism_RefMadnessIllegal(t *testing.T) {
clearSchemas()
// this does not work, but it won't error out.
doc := `components:
yml := `components:
schemas:
Something:
$ref: '#/components/schemas/Else'
@@ -722,11 +727,11 @@ func Test_Schema_Polymorphism_RefMadnessIllegal(t *testing.T) {
description: hey!`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `$ref: '#/components/schemas/Something'`
yml = `$ref: '#/components/schemas/Something'`
var sch Schema
var idxNode yaml.Node
@@ -741,19 +746,20 @@ func Test_Schema_Polymorphism_RefMadnessIllegal(t *testing.T) {
}
func TestExtractSchema(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `schema:
yml = `schema:
type: object
properties:
aValue:
@@ -765,23 +771,25 @@ func TestExtractSchema(t *testing.T) {
res, err := ExtractSchema(idxNode.Content[0], idx)
assert.NoError(t, err)
assert.NotNil(t, res.Value)
assert.Equal(t, "this is something", res.Value.FindProperty("aValue").Value.Description.Value)
aValue := res.Value.FindProperty("aValue")
assert.Equal(t, "this is something", aValue.Value.Description.Value)
}
func TestExtractSchema_Ref(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `schema:
yml = `schema:
$ref: '#/components/schemas/Something'`
var idxNode yaml.Node
@@ -794,19 +802,20 @@ func TestExtractSchema_Ref(t *testing.T) {
}
func TestExtractSchema_Ref_Fail(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `schema:
yml = `schema:
$ref: '#/components/schemas/Missing'`
var idxNode yaml.Node
@@ -817,19 +826,20 @@ func TestExtractSchema_Ref_Fail(t *testing.T) {
}
func TestExtractSchema_RefRoot(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `$ref: '#/components/schemas/Something'`
yml = `$ref: '#/components/schemas/Something'`
var idxNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &idxNode)
@@ -841,19 +851,20 @@ func TestExtractSchema_RefRoot(t *testing.T) {
}
func TestExtractSchema_RefRoot_Fail(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
description: this is something
type: string`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `$ref: '#/components/schemas/Missing'`
yml = `$ref: '#/components/schemas/Missing'`
var idxNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &idxNode)
@@ -864,18 +875,19 @@ func TestExtractSchema_RefRoot_Fail(t *testing.T) {
}
func TestExtractSchema_RefRoot_Child_Fail(t *testing.T) {
clearSchemas()
doc := `components:
yml := `components:
schemas:
Something:
$ref: #bork`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `$ref: '#/components/schemas/Something'`
yml = `$ref: '#/components/schemas/Something'`
var idxNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &idxNode)
@@ -887,17 +899,19 @@ func TestExtractSchema_RefRoot_Child_Fail(t *testing.T) {
func TestExtractSchema_DoNothing(t *testing.T) {
doc := `components:
clearSchemas()
yml := `components:
schemas:
Something:
$ref: #bork`
var iNode yaml.Node
mErr := yaml.Unmarshal([]byte(doc), &iNode)
mErr := yaml.Unmarshal([]byte(yml), &iNode)
assert.NoError(t, mErr)
idx := index.NewSpecIndex(&iNode)
yml := `please: do nothing.`
yml = `please: do nothing.`
var idxNode yaml.Node
_ = yaml.Unmarshal([]byte(yml), &idxNode)

56
datamodel/low/extraction_functions.go
View File

@@ -285,6 +285,11 @@ func ExtractMapFlatNoLookup[PT Buildable[N], N any](root *yaml.Node, idx *index.
return valueMap, nil
}
type mappingResult[T any] struct {
k KeyReference[string]
v ValueReference[T]
}
func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (map[KeyReference[string]]ValueReference[PT], *yaml.Node, *yaml.Node, error) {
var labelNode, valueNode *yaml.Node
if rf, rl, _ := utils.IsNodeRefValue(root); rf {
@@ -314,12 +319,36 @@ func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *
if valueNode != nil {
var currentLabelNode *yaml.Node
valueMap := make(map[KeyReference[string]]ValueReference[PT])
bChan := make(chan mappingResult[PT])
eChan := make(chan error)
var buildMap = func(label *yaml.Node, value *yaml.Node, c chan mappingResult[PT], ec chan<- error) {
var n PT = new(N)
_ = BuildModel(value, n)
err := n.Build(value, idx)
if err != nil {
ec <- err
return
}
c <- mappingResult[PT]{
k: KeyReference[string]{
KeyNode: label,
Value: label.Value,
},
v: ValueReference[PT]{
Value: n,
ValueNode: value,
},
}
}
totalKeys := 0
for i, en := range valueNode.Content {
if i%2 == 0 {
currentLabelNode = en
continue
}
// check our valueNode isn't a reference still.
if h, _, _ := utils.IsNodeRefValue(en); h {
ref := LocateRefNode(en, idx)
@@ -334,21 +363,18 @@ func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *
if strings.HasPrefix(strings.ToLower(currentLabelNode.Value), "x-") {
continue // yo, don't pay any attention to extensions, not here anyway.
}
var n PT = new(N)
err := BuildModel(en, n)
if err != nil {
return nil, labelNode, valueNode, err
totalKeys++
go buildMap(currentLabelNode, en, bChan, eChan)
}
berr := n.Build(en, idx)
if berr != nil {
return nil, labelNode, valueNode, berr
}
valueMap[KeyReference[string]{
Value: currentLabelNode.Value,
KeyNode: currentLabelNode,
}] = ValueReference[PT]{
Value: n,
ValueNode: en,
completedKeys := 0
for completedKeys < totalKeys {
select {
case err := <-eChan:
return valueMap, labelNode, valueNode, err
case res := <-bChan:
completedKeys++
valueMap[res.k] = res.v
}
}
return valueMap, labelNode, valueNode, nil

13
datamodel/low/reference.go
View File

@@ -54,3 +54,16 @@ func (n KeyReference[T]) IsEmpty() bool {
func (n KeyReference[T]) GenerateMapKey() string {
return fmt.Sprintf("%d:%d", n.KeyNode.Line, n.KeyNode.Column)
}
func IsCircular(node *yaml.Node, idx *index.SpecIndex) bool {
if idx == nil {
return false // no index! nothing we can do.
}
refs := idx.GetCircularReferences()
for i := range idx.GetCircularReferences() {
if refs[i].LoopPoint.Node == node {
return true
}
}
return false
}

6
resolver/resolver_test.go
View File

@@ -35,7 +35,7 @@ func TestResolver_ResolveComponents_CircularSpec(t *testing.T) {
assert.NotNil(t, resolver)
circ := resolver.Resolve()
assert.Len(t, circ, 3)
assert.Len(t, circ, 4)
_, err := yaml.Marshal(resolver.resolvedRoot)
assert.NoError(t, err)
@@ -53,7 +53,7 @@ func TestResolver_ResolveComponents_Stripe(t *testing.T) {
assert.NotNil(t, resolver)
circ := resolver.Resolve()
assert.Len(t, circ, 0)
assert.Len(t, circ, 229)
}
@@ -85,5 +85,5 @@ func TestResolver_ResolveComponents_k8s(t *testing.T) {
assert.NotNil(t, resolver)
circ := resolver.Resolve()
assert.Len(t, circ, 0)
assert.Len(t, circ, 1)
}