Building out model some more.

everything is holding up well.
This commit is contained in:
Dave Shanley
2022-07-31 16:15:01 -04:00
parent 2f60694047
commit a4012594de
13 changed files with 2061 additions and 1857 deletions

View File

@@ -8,10 +8,10 @@ type Document struct {
Version low.NodeReference[string]
Info low.NodeReference[*Info]
Servers []low.NodeReference[*Server]
Paths *Paths
Paths low.NodeReference[*Paths]
Components *Components
Security []*SecurityRequirement
Tags []low.NodeReference[*Tag]
ExternalDocs *ExternalDoc
Extensions map[string]low.ObjectReference
Extensions map[low.NodeReference[string]]low.NodeReference[any]
}

View File

@@ -8,6 +8,10 @@ import (
"sync"
)
const (
InfoLabel = "info"
)
type Info struct {
Title low.NodeReference[string]
Description low.NodeReference[string]

View File

@@ -1,23 +1,79 @@
package v3
import (
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
)
const (
ParametersLabel = "parameters"
)
type Operation struct {
Node *yaml.Node
Tags []low.NodeReference[string]
Summary low.NodeReference[string]
Description low.NodeReference[string]
ExternalDocs ExternalDoc
ExternalDocs *low.NodeReference[*ExternalDoc]
OperationId low.NodeReference[string]
Parameters []Parameter
RequestBody RequestBody
Responses Responses
Callbacks map[string]Callback
Deprecated low.NodeReference[bool]
Security []SecurityRequirement
Servers []Server
Extensions map[string]low.ObjectReference
Parameters []low.NodeReference[*Parameter]
RequestBody *low.NodeReference[*RequestBody]
Responses *low.NodeReference[*Responses]
Callbacks map[low.NodeReference[string]]low.NodeReference[*Callback]
Deprecated *low.NodeReference[bool]
Security []low.NodeReference[*SecurityRequirement]
Servers []low.NodeReference[*Server]
Extensions map[low.NodeReference[string]]low.NodeReference[any]
}
func (o *Operation) Build(root *yaml.Node, idx *index.SpecIndex) error {
extensionMap, err := datamodel.ExtractExtensions(root)
if err != nil {
return err
}
o.Extensions = extensionMap
// extract external docs
_, ln, exDocs := utils.FindKeyNodeFull(ExternalDocsLabel, root.Content)
if exDocs != nil {
var externalDoc ExternalDoc
err = datamodel.BuildModel(exDocs, &externalDoc)
if err != nil {
return err
}
o.ExternalDocs = &low.NodeReference[*ExternalDoc]{
Value: &externalDoc,
KeyNode: ln,
ValueNode: exDocs,
}
}
// build parameters
_, paramLabel, paramNode := utils.FindKeyNodeFull(ParametersLabel, root.Content)
if paramNode != nil && paramLabel != nil {
var params []low.NodeReference[*Parameter]
for _, pN := range paramNode.Content {
var param Parameter
err = datamodel.BuildModel(pN, &param)
if err != nil {
return err
}
err = param.Build(pN, idx)
if err != nil {
return err
}
params = append(params, low.NodeReference[*Parameter]{
Value: &param,
ValueNode: paramNode,
KeyNode: paramLabel,
})
}
}
return nil
}

View File

@@ -1,16 +1,38 @@
package v3
import (
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"gopkg.in/yaml.v3"
)
type Parameter struct {
Node *yaml.Node
Name low.NodeReference[string]
In low.NodeReference[string]
Description low.NodeReference[string]
Required low.NodeReference[bool]
Deprecated low.NodeReference[bool]
AllowEmptyValue low.NodeReference[bool]
Style low.NodeReference[string]
Explode low.NodeReference[bool]
AllowReserved low.NodeReference[bool]
Schema low.NodeReference[*Schema]
Example low.NodeReference[any]
Examples map[low.NodeReference[string]]low.NodeReference[*Example]
Extensions map[low.NodeReference[string]]low.NodeReference[any]
}
func (p *Parameter) Build(root *yaml.Node, idx *index.SpecIndex) error {
// extract extensions
extensionMap, err := datamodel.ExtractExtensions(root)
if err != nil {
return err
}
p.Extensions = extensionMap
// deal with schema
return nil
}

View File

@@ -1,30 +1,200 @@
package v3
import (
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/datamodel/low"
"github.com/pb33f/libopenapi/index"
"gopkg.in/yaml.v3"
"strings"
"sync"
)
const (
PathsLabel = "paths"
GetLabel = "get"
PostLabel = "post"
PatchLabel = "patch"
PutLabel = "put"
DeleteLabel = "delete"
OptionsLabel = "options"
HeadLabel = "head"
TraceLabel = "trace"
)
type Paths struct {
Node *yaml.Node
Paths map[string]Path
Extensions map[string]low.ObjectReference
Paths map[low.NodeReference[string]]low.NodeReference[*Path]
Extensions map[low.NodeReference[string]]low.NodeReference[any]
}
func (p *Paths) GetPathMap() map[string]*Path {
pMap := make(map[string]*Path)
for i, pv := range p.Paths {
pMap[i.Value] = pv.Value
}
return pMap
}
func (p *Paths) Build(root *yaml.Node, idx *index.SpecIndex) error {
// extract extensions
extensionMap, err := datamodel.ExtractExtensions(root)
if err != nil {
return err
}
p.Extensions = extensionMap
skip := false
var currentNode *yaml.Node
pathsMap := make(map[low.NodeReference[string]]low.NodeReference[*Path])
for i, pathNode := range root.Content {
if strings.HasPrefix(strings.ToLower(pathNode.Value), "x-") {
skip = true
continue
}
if skip {
skip = false
continue
}
if i%2 == 0 {
currentNode = pathNode
continue
}
var path = Path{}
err = datamodel.BuildModel(pathNode, &path)
if err != nil {
}
err = path.Build(pathNode, idx)
if err != nil {
return err
}
// add bulk here
pathsMap[low.NodeReference[string]{
Value: currentNode.Value,
KeyNode: currentNode,
}] = low.NodeReference[*Path]{
Value: &path,
ValueNode: pathNode,
}
}
p.Paths = pathsMap
return nil
}
type Path struct {
Node *yaml.Node
Value low.NodeReference[string]
Summary low.NodeReference[string]
Description low.NodeReference[string]
Get Operation
Put Operation
Post Operation
Delete Operation
Options Operation
Head Operation
Patch Operation
Trace Operation
Servers []Server
Parameters []Parameter
Extensions map[string]low.ObjectReference
Summary low.NodeReference[string]
Get *low.NodeReference[*Operation]
Put *low.NodeReference[*Operation]
Post *low.NodeReference[*Operation]
Delete *low.NodeReference[*Operation]
Options *low.NodeReference[*Operation]
Head *low.NodeReference[*Operation]
Patch *low.NodeReference[*Operation]
Trace *low.NodeReference[*Operation]
Servers []*low.NodeReference[*Server]
Parameters []*low.NodeReference[*Parameter]
Extensions map[low.NodeReference[string]]low.NodeReference[any]
}
func (p *Path) Build(root *yaml.Node, idx *index.SpecIndex) error {
extensionMap, err := datamodel.ExtractExtensions(root)
if err != nil {
return err
}
p.Extensions = extensionMap
skip := false
var currentNode *yaml.Node
var wg sync.WaitGroup
var errors []error
var ops []low.NodeReference[*Operation]
for i, pathNode := range root.Content {
if strings.HasPrefix(strings.ToLower(pathNode.Value), "x-") {
skip = true
continue
}
if skip {
skip = false
continue
}
if i%2 == 0 {
currentNode = pathNode
continue
}
var op Operation
wg.Add(1)
datamodel.BuildModelAsync(pathNode, &op, &wg, &errors)
opRef := low.NodeReference[*Operation]{
Value: &op,
KeyNode: currentNode,
ValueNode: pathNode,
}
ops = append(ops, opRef)
switch currentNode.Value {
case GetLabel:
p.Get = &opRef
case PostLabel:
p.Post = &opRef
case PutLabel:
p.Put = &opRef
case PatchLabel:
p.Patch = &opRef
case DeleteLabel:
p.Delete = &opRef
case HeadLabel:
p.Head = &opRef
case OptionsLabel:
p.Options = &opRef
case TraceLabel:
p.Trace = &opRef
}
}
wg.Wait()
// all operations have been superficially built,
// now we need to build out the operation, we will do this asynchronously for speed.
opBuildChan := make(chan bool)
opErrorChan := make(chan error)
var buildOpFunc = func(op low.NodeReference[*Operation], ch chan<- bool, errCh chan<- error) {
// build out the operation.
er := op.Value.Build(op.ValueNode, idx)
if err != nil {
errCh <- er
}
ch <- true
}
for _, op := range ops {
go buildOpFunc(op, opBuildChan, opErrorChan)
}
n := 0
allDone:
for {
select {
case buildError := <-opErrorChan:
return buildError
case <-opBuildChan:
if n == len(ops)-1 {
break allDone
}
}
}
return nil
}

View File

@@ -8,7 +8,8 @@ import (
)
const (
Variables = "variables"
VariablesLabel = "variables"
ServersLabel = "servers"
)
type Server struct {
@@ -18,7 +19,7 @@ type Server struct {
}
func (s *Server) Build(root *yaml.Node) error {
kn, vars := utils.FindKeyNode(Variables, root.Content)
kn, vars := utils.FindKeyNode(VariablesLabel, root.Content)
if vars == nil {
return nil
}

View File

@@ -8,8 +8,8 @@ import (
)
const (
Tags = "tags"
ExternalDocs = "externalDocs"
TagsLabel = "tags"
ExternalDocsLabel = "externalDocs"
)
type Tag struct {
@@ -20,8 +20,6 @@ type Tag struct {
}
func (t *Tag) Build(root *yaml.Node) error {
_, ln, exDocs := utils.FindKeyNodeFull(ExternalDocs, root.Content)
// extract extensions
extensionMap, err := datamodel.ExtractExtensions(root)
if err != nil {
@@ -29,6 +27,7 @@ func (t *Tag) Build(root *yaml.Node) error {
}
t.Extensions = extensionMap
_, ln, exDocs := utils.FindKeyNodeFull(ExternalDocsLabel, root.Content)
// extract external docs
var externalDoc ExternalDoc
err = datamodel.BuildModel(exDocs, &externalDoc)

View File

@@ -31,6 +31,8 @@ type Reference struct {
Resolved bool
Circular bool
Seen bool
IsRemote bool
RemoteLocation string
Path string // this won't always be available.
}
@@ -543,18 +545,9 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string,
value := node.Content[i+1].Value
if strings.Contains(value, "~1") {
indexError := &IndexingError{
Error: errors.New("schema reference is contains '~1' win32 truncation and cannot be processed"),
Node: node.Content[i+1],
Path: fmt.Sprintf("$.%s", strings.Join(fp, ".")),
}
index.refErrors = append(index.refErrors, indexError)
continue
}
segs := strings.Split(value, "/")
name := segs[len(segs)-1]
//name := strings.ReplaceAll(segs[len(segs)-1], "~1", "/")
ref := &Reference{
Definition: value,
Name: name,
@@ -1378,18 +1371,6 @@ func (index *SpecIndex) ExtractComponentsFromRefs(refs []*Reference) []*Referenc
var found []*Reference
for _, ref := range refs {
// so, some really strange shit showed up when linting api.guru
if strings.Contains(ref.Definition, "~1") { // this was from azure! jesus guys, win32? wtf.
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
indexError := &IndexingError{
Error: fmt.Errorf("component '%s' contains freaky win32 '~1' file truncation, can't be used.", ref.Definition),
Node: ref.Node,
Path: path,
}
index.refErrors = append(index.refErrors, indexError)
continue
}
// check reference for back slashes (hah yeah seen this too!)
if strings.Contains(ref.Definition, "\\") { // this was from blazemeter.com haha!
_, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition)
@@ -1681,6 +1662,8 @@ func (index *SpecIndex) performExternalLookup(uri []string, componentId string,
Definition: componentId,
Name: nameSegs[len(nameSegs)-1],
Node: foundNode,
IsRemote: true,
RemoteLocation: componentId,
}
return ref
}
@@ -1855,6 +1838,10 @@ func (index *SpecIndex) lookupRemoteReference(ref string) (*yaml.Node, *yaml.Nod
// lookup item from reference by using a path query.
query := fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
// remove any URL encoding
query = strings.Replace(query, "~1", "./", 1)
query = strings.ReplaceAll(query, "~1", "/")
path, err := yamlpath.NewPath(query)
if err != nil {
return nil, nil, err
@@ -1907,6 +1894,10 @@ func (index *SpecIndex) lookupFileReference(ref string) (*yaml.Node, *yaml.Node,
// lookup item from reference by using a path query.
query := fmt.Sprintf("$%s", strings.ReplaceAll(uri[1], "/", "."))
// remove any URL encoding
query = strings.Replace(query, "~1", "./", 1)
query = strings.ReplaceAll(query, "~1", "/")
path, err := yamlpath.NewPath(query)
if err != nil {
return nil, nil, err

View File

@@ -170,17 +170,17 @@ func TestSpecIndex_BurgerShop(t *testing.T) {
index := NewSpecIndex(&rootNode)
assert.Len(t, index.allRefs, 5)
assert.Len(t, index.allMappedRefs, 5)
assert.Equal(t, 5, len(index.GetMappedReferences()))
assert.Equal(t, 5, len(index.GetMappedReferencesSequenced()))
assert.Len(t, index.allRefs, 6)
assert.Len(t, index.allMappedRefs, 6)
assert.Equal(t, 6, len(index.GetMappedReferences()))
assert.Equal(t, 6, len(index.GetMappedReferencesSequenced()))
assert.Equal(t, 5, index.pathCount)
assert.Equal(t, 5, index.GetPathCount())
assert.Equal(t, 7, index.pathCount)
assert.Equal(t, 7, index.GetPathCount())
assert.Equal(t, 5, len(index.GetAllSchemas()))
assert.Equal(t, 18, len(index.GetAllSequencedReferences()))
assert.Equal(t, 19, len(index.GetAllSequencedReferences()))
assert.NotNil(t, index.GetSchemasNode())
assert.Nil(t, index.GetParametersNode())
@@ -224,7 +224,7 @@ func TestSpecIndex_BurgerShop(t *testing.T) {
assert.Equal(t, 0, len(index.GetAllResponses()))
assert.Equal(t, 2, len(index.GetInlineOperationDuplicateParameters()))
assert.Equal(t, 0, len(index.GetReferencesWithSiblings()))
assert.Equal(t, 5, len(index.GetAllReferences()))
assert.Equal(t, 6, len(index.GetAllReferences()))
assert.Equal(t, 0, len(index.GetOperationParametersIndexErrors()))
assert.Equal(t, 5, len(index.GetAllPaths()))
assert.Equal(t, 5, len(index.GetOperationTags()))

View File

@@ -1,57 +1,63 @@
package openapi
import (
"fmt"
"github.com/pb33f/libopenapi/datamodel"
"github.com/pb33f/libopenapi/datamodel/low"
v3 "github.com/pb33f/libopenapi/datamodel/low/3.0"
"github.com/pb33f/libopenapi/index"
"github.com/pb33f/libopenapi/resolver"
"github.com/pb33f/libopenapi/utils"
"gopkg.in/yaml.v3"
"strconv"
"sync"
)
const (
Info = "info"
Servers = "servers"
)
func CreateDocument(info *datamodel.SpecInfo) (*v3.Document, error) {
doc := v3.Document{Version: low.NodeReference[string]{Value: info.Version, ValueNode: info.RootNode}}
// build an index
//idx := index.NewSpecIndex(info.RootNode)
//datamodel.BuildModel(info.RootNode.Content[0], &doc)
idx := index.NewSpecIndex(info.RootNode)
rsolvr := resolver.NewResolver(idx)
// todo handle errors
rsolvr.Resolve()
var wg sync.WaitGroup
var errors []error
var runExtraction = func(info *datamodel.SpecInfo, doc *v3.Document,
runFunc func(i *datamodel.SpecInfo, d *v3.Document) error,
runFunc func(i *datamodel.SpecInfo, d *v3.Document, idx *index.SpecIndex) error,
ers *[]error,
wg *sync.WaitGroup) {
if er := runFunc(info, doc); er != nil {
if er := runFunc(info, doc, idx); er != nil {
*ers = append(*ers, er)
}
wg.Done()
}
wg.Add(3)
go runExtraction(info, &doc, extractInfo, &errors, &wg)
go runExtraction(info, &doc, extractServers, &errors, &wg)
go runExtraction(info, &doc, extractTags, &errors, &wg)
extractionFuncs := []func(i *datamodel.SpecInfo, d *v3.Document, idx *index.SpecIndex) error{
extractInfo,
extractServers,
extractTags,
extractPaths,
}
wg.Add(len(extractionFuncs))
for _, f := range extractionFuncs {
go runExtraction(info, &doc, f, &errors, &wg)
}
wg.Wait()
// todo fix this.
if len(errors) > 0 {
return &doc, errors[0]
}
fmt.Sprint(idx)
return &doc, nil
}
func extractInfo(info *datamodel.SpecInfo, doc *v3.Document) error {
_, ln, vn := utils.FindKeyNodeFull(Info, info.RootNode.Content)
func extractInfo(info *datamodel.SpecInfo, doc *v3.Document, idx *index.SpecIndex) error {
_, ln, vn := utils.FindKeyNodeFull(v3.InfoLabel, info.RootNode.Content)
if vn != nil {
ir := v3.Info{}
err := datamodel.BuildModel(vn, &ir)
@@ -65,8 +71,8 @@ func extractInfo(info *datamodel.SpecInfo, doc *v3.Document) error {
return nil
}
func extractServers(info *datamodel.SpecInfo, doc *v3.Document) error {
_, ln, vn := utils.FindKeyNodeFull(Servers, info.RootNode.Content)
func extractServers(info *datamodel.SpecInfo, doc *v3.Document, idx *index.SpecIndex) error {
_, ln, vn := utils.FindKeyNodeFull(v3.ServersLabel, info.RootNode.Content)
if vn != nil {
if utils.IsNodeArray(vn) {
var servers []low.NodeReference[*v3.Server]
@@ -91,8 +97,8 @@ func extractServers(info *datamodel.SpecInfo, doc *v3.Document) error {
return nil
}
func extractTags(info *datamodel.SpecInfo, doc *v3.Document) error {
_, ln, vn := utils.FindKeyNodeFull(v3.Tags, info.RootNode.Content)
func extractTags(info *datamodel.SpecInfo, doc *v3.Document, idx *index.SpecIndex) error {
_, ln, vn := utils.FindKeyNodeFull(v3.TagsLabel, info.RootNode.Content)
if vn != nil {
if utils.IsNodeArray(vn) {
var tags []low.NodeReference[*v3.Tag]
@@ -117,66 +123,16 @@ func extractTags(info *datamodel.SpecInfo, doc *v3.Document) error {
return nil
}
func ExtractExtensions(root *yaml.Node) (map[low.NodeReference[string]]low.NodeReference[any], error) {
extensions := utils.FindExtensionNodes(root.Content)
extensionMap := make(map[low.NodeReference[string]]low.NodeReference[any])
for _, ext := range extensions {
// this is an object, decode into an unknown map.
if utils.IsNodeMap(ext.Value) {
var v interface{}
err := ext.Value.Decode(&v)
func extractPaths(info *datamodel.SpecInfo, doc *v3.Document, idx *index.SpecIndex) error {
_, ln, vn := utils.FindKeyNodeFull(v3.PathsLabel, info.RootNode.Content)
if vn != nil {
ir := v3.Paths{}
err := ir.Build(vn, idx)
if err != nil {
return nil, err
return err
}
extensionMap[low.NodeReference[string]{
Value: ext.Key.Value,
KeyNode: ext.Key,
ValueNode: ext.Value,
}] = low.NodeReference[any]{Value: v, KeyNode: ext.Key}
nr := low.NodeReference[*v3.Paths]{Value: &ir, ValueNode: vn, KeyNode: ln}
doc.Paths = nr
}
if utils.IsNodeStringValue(ext.Value) {
extensionMap[low.NodeReference[string]{
Value: ext.Key.Value,
KeyNode: ext.Key,
ValueNode: ext.Value,
}] = low.NodeReference[any]{Value: ext.Value.Value, ValueNode: ext.Value}
}
if utils.IsNodeFloatValue(ext.Value) {
fv, _ := strconv.ParseFloat(ext.Value.Value, 64)
extensionMap[low.NodeReference[string]{
Value: ext.Key.Value,
KeyNode: ext.Key,
ValueNode: ext.Value,
}] = low.NodeReference[any]{Value: fv, ValueNode: ext.Value}
}
if utils.IsNodeIntValue(ext.Value) {
iv, _ := strconv.ParseInt(ext.Value.Value, 10, 64)
extensionMap[low.NodeReference[string]{
Value: ext.Key.Value,
KeyNode: ext.Key,
ValueNode: ext.Value,
}] = low.NodeReference[any]{Value: iv, ValueNode: ext.Value}
}
if utils.IsNodeBoolValue(ext.Value) {
bv, _ := strconv.ParseBool(ext.Value.Value)
extensionMap[low.NodeReference[string]{
Value: ext.Key.Value,
KeyNode: ext.Key,
ValueNode: ext.Value,
}] = low.NodeReference[any]{Value: bv, ValueNode: ext.Value}
}
if utils.IsNodeArray(ext.Value) {
var v []interface{}
err := ext.Value.Decode(&v)
if err != nil {
return nil, err
}
extensionMap[low.NodeReference[string]{
Value: ext.Key.Value,
KeyNode: ext.Key,
ValueNode: ext.Value,
}] = low.NodeReference[any]{Value: v, ValueNode: ext.Value}
}
}
return extensionMap, nil
return nil
}

View File

@@ -54,7 +54,11 @@ servers:
default: "pb33f.io"
description: the default host for this API is 'pb33f.com'
paths:
x-milky-milk: milky
/refingtons:
$ref: '../test_specs/petstorev3.json#/paths~1pet~1findByStatus'
/burgers:
x-burger-meta: meaty
post:
operationId: createBurger
tags:
@@ -385,3 +389,4 @@ components:
type: string
description: what size man? S/M/L
example: M

View File

@@ -448,7 +448,7 @@ func IsHttpVerb(verb string) bool {
func ConvertComponentIdIntoFriendlyPathSearch(id string) (string, string) {
segs := strings.Split(id, "/")
name := segs[len(segs)-1]
name := strings.ReplaceAll(segs[len(segs)-1], "~1", "/")
replaced := strings.ReplaceAll(fmt.Sprintf("%s['%s']",
strings.Join(segs[:len(segs)-1], "."), name), "#", "$")