From cbce025d6c87b2b31959e411be8ff308029934e1 Mon Sep 17 00:00:00 2001 From: Dave Shanley Date: Fri, 16 Sep 2022 10:44:30 -0400 Subject: [PATCH] More docs going in, Added README details. still lots to do, a long way to go, but it's starting to take shape. --- README.md | 156 ++++++++++++++++++++++++++++- datamodel/high/v3/callback.go | 4 + datamodel/high/v3/document.go | 69 ++++++++++--- datamodel/high/v3/document_test.go | 54 +++++----- datamodel/high/v3/encoding.go | 5 + datamodel/high/v3/package_test.go | 41 ++++++++ 6 files changed, 288 insertions(+), 41 deletions(-) create mode 100644 datamodel/high/v3/package_test.go diff --git a/README.md b/README.md index a8a3886..5c19f02 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,158 @@ -# libopenapi +# libopenapi - enterprise grade OpenAPI tools for golang. ![Pipeline](https://github.com/pb33f/libopenapi/workflows/Build/badge.svg) +![GoReportCard](https://goreportcard.com/badge/github.com/pb33f/libopenapi) [![codecov](https://codecov.io/gh/pb33f/libopenapi/branch/main/graph/badge.svg?)](https://codecov.io/gh/pb33f/libopenapi) -A place for up to date, modern golang OpenAPI models and utilities. +libopenapi has full support for Swagger (OpenAPI 2), OpenAPI 3, and OpenAPI 3.1. + +## Introduction - Why? + +There is already a great OpenAPI library for golang, it's called [kin-openapi](https://github.com/getkin/kin-openapi). + +### So why does this exist? + +[kin-openapi](https://github.com/getkin/kin-openapi) is great, and you should use it. + +**However, it's missing one critical feature** + +When building tooling that needs to analyze OpenAPI specifications at a *low* level, [kin-openapi](https://github.com/getkin/kin-openapi) +**runs out of power** when you need to know the original line numbers and columns, or comments within all keys and values in the spec. + +All that data is **lost** when the spec is loaded in by [kin-openapi](https://github.com/getkin/kin-openapi). It's mainly +because the library will unmarshal data directly into structs, which works great - if you don't need access to the original +specification low level details. + +## libopenapi retains _everything_. + +libopenapi has been designed to retain all of that really low-level detail about the AST, line numbers, column numbers, comments, +original AST structure - everything you could need. + +libopenapi has a **porcelain** (high-level) and a **plumbing** (low-level) API. Every high level struct, has the +ability to '**GoLow**' and dive from the high-level model, down to the low-level model and look-up any detail about the +underlying raw data backing that model. + +This library exists because this very need existed inside [VMware](https://vmware.com), we built our own internal +version of libopenapi, which isn't something that can be released as it's bespoke. + +libopenapi is the result of years of learning and battle testing OpenAPI in golang. This library represents what we +would have created, if we knew then - what we know now. + +> If you need to know which line, or column a key or value for something is? **libopenapi has you covered** + +## Comes with an Index and a Resolver + +Want a lightning fast way to look up any element in an OpenAPI swagger spec? **libopenapi has you covered** + +Need a way to 'resolve' OpenAPI documents that are exploded out across multiple files, remotely or locally? +**libopenapi has you covered** + +--- + +## Some examples to get you started. + +### Load an OpenAPI 3+ spec into a model + +```go +// load an OpenAPI 3 specification from bytes +petstore, _ := ioutil.ReadFile("test_specs/petstorev3.json") + +// create a new document from specification bytes +document, err := NewDocument(petstore) + +// if anything went wrong, an error is thrown +if err != nil { + panic(fmt.Sprintf("cannot create new document: %e", err)) +} + +// because we know this is a v3 spec, we can build a ready to go model from it. +v3Model, errors := document.BuildV3Model() + +// if anything went wrong when building the v3 model, a slice of errors will be returned +if len(errors) > 0 { + for i := range errors { + fmt.Printf("error: %e\n", errors[i]) + } + panic(fmt.Sprintf("cannot create v3 model from document: %d errors reported", len(errors))) +} + +// get a count of the number of paths and schemas. +paths := len(v3Model.Model.Paths.PathItems) +schemas := len(v3Model.Model.Components.Schemas) + +// print the number of paths and schemas in the document +fmt.Printf("There are %d paths and %d schemas in the document", paths, schemas) +``` + +This will print: + +``` +There are 13 paths and 8 schemas in the document +``` + + +### Load a Swagger (OpenAPI 2) spec into a model +```go +// load a Swagger specification from bytes +petstore, _ := ioutil.ReadFile("test_specs/petstorev2.json") + +// create a new document from specification bytes +document, err := NewDocument(petstore) + +// if anything went wrong, an error is thrown +if err != nil { + panic(fmt.Sprintf("cannot create new document: %e", err)) +} + +// because we know this is a v2 spec, we can build a ready to go model from it. +v2Model, errors := document.BuildV2Model() + +// if anything went wrong when building the v3 model, a slice of errors will be returned +if len(errors) > 0 { + for i := range errors { + fmt.Printf("error: %e\n", errors[i]) + } + panic(fmt.Sprintf("cannot create v3 model from document: %d errors reported", len(errors))) +} + +// get a count of the number of paths and schemas. +paths := len(v2Model.Model.Paths.PathItems) +schemas := len(v2Model.Model.Definitions.Definitions) + +// print the number of paths and schemas in the document +fmt.Printf("There are %d paths and %d schemas in the document", paths, schemas) +``` + +This will print: + +``` +There are 14 paths and 6 schemas in the document +``` + +### Dropping down from the high-level API to the low-level one + +This example shows how after loading an OpenAPI spec into a document, navigating to an Operation is pretty simple. +It then shows how to _drop-down_ to the low-level API and query the line and start column of the RequestBody description. + +```go +// load an OpenAPI 3 specification from bytes +petstore, _ := ioutil.ReadFile("test_specs/petstorev3.json") + +// create a new document from specification bytes (ignore errors for the same of the example) +document, _ := NewDocument(petstore) + +// because we know this is a v3 spec, we can build a ready to go model from it (ignoring errors for the example) +v3Model, _ := document.BuildV3Model() + +// extract the RequestBody from the 'put' operation under the /pet path +reqBody := h.Paths.PathItems["/pet"].Put.RequestBody + +// dropdown to the low-level API for RequestBody +lowReqBody := reqBody.GoLow() + +// print out the value, the line it appears on and the start columns for the key and value of the nodes. +fmt.Printf("value is %s, the value is on line %d, starting column %d, the key is on line %d, column %d", + reqBody.Description, lowReqBody.Description.ValueNode.Line, lowReqBody.Description.ValueNode.Column, + lowReqBody.Description.KeyNode.Line, lowReqBody.KeyNode.Column) + +``` diff --git a/datamodel/high/v3/callback.go b/datamodel/high/v3/callback.go index dc9349d..f534b4c 100644 --- a/datamodel/high/v3/callback.go +++ b/datamodel/high/v3/callback.go @@ -5,12 +5,15 @@ package v3 import low "github.com/pb33f/libopenapi/datamodel/low/v3" +// Callback represents a high-level Callback object for OpenAPI 3+. +// - https://spec.openapis.org/oas/v3.1.0#callback-object type Callback struct { Expression map[string]*PathItem Extensions map[string]any low *low.Callback } +// NewCallback creates a new high-level callback from a low-level one. func NewCallback(lowCallback *low.Callback) *Callback { n := new(Callback) n.low = lowCallback @@ -25,6 +28,7 @@ func NewCallback(lowCallback *low.Callback) *Callback { return n } +// GoLow returns the low-level Callback instance used to create the high-level one. func (c *Callback) GoLow() *low.Callback { return c.low } diff --git a/datamodel/high/v3/document.go b/datamodel/high/v3/document.go index d78d698..b2de451 100644 --- a/datamodel/high/v3/document.go +++ b/datamodel/high/v3/document.go @@ -1,6 +1,12 @@ // Copyright 2022 Princess B33f Heavy Industries / Dave Shanley // SPDX-License-Identifier: MIT +// Package v3 represents all OpenAPI 3+ high-level models. High-level models are easy to navigate +// and simple to extract what ever is required from an OpenAPI 3+ specification. +// +// High-level models are backed by low-level ones. There is a 'GoLow()' method available on every high level +// object. 'Going Low' allows engineers to transition from a high-level or 'porcelain' API, to a low-level 'plumbing' +// API, which provides fine grain detail to the underlying AST powering the data, lines, columns, raw nodes etc. package v3 import ( @@ -10,22 +16,61 @@ import ( "github.com/pb33f/libopenapi/index" ) +// Document represents a high-level OpenAPI 3 document (both 3.0 & 3.1). A Document is the root of the specification. type Document struct { - Version string - Info *base.Info - Servers []*Server - Paths *Paths - Components *Components - Security *SecurityRequirement - Tags []*base.Tag - ExternalDocs *base.ExternalDoc - Extensions map[string]any - Index *index.SpecIndex + + // Version is the version of OpenAPI being used, extracted from the 'openapi: x.x.x' definition. + // This is not a standard property of the OpenAPI model, it's a convenience mechanism only. + Version string + + // Info presents a specification Info definitions + // - https://spec.openapis.org/oas/v3.1.0#info-object + Info *base.Info + + // Servers is a slice of Server instances + // - https://spec.openapis.org/oas/v3.1.0#server-object + Servers []*Server + + // Paths contains all the PathItem definitions for the specification. + // - https://spec.openapis.org/oas/v3.1.0#paths-object + Paths *Paths + + // Components contains everything defined as a component (referenced by everything else) + // - https://spec.openapis.org/oas/v3.1.0#components-object + Components *Components + + // Security contains global security requirements/roles for the specification + // - https://spec.openapis.org/oas/v3.1.0#security-requirement-object + Security *SecurityRequirement + + // Tags is a slice of base.Tag instances defined by the specification + // - https://spec.openapis.org/oas/v3.1.0#tag-object + Tags []*base.Tag + + // ExternalDocs is an instance of base.ExternalDoc for.. well, obvious really, innit. + // - https://spec.openapis.org/oas/v3.1.0#external-documentation-object + ExternalDocs *base.ExternalDoc + + // Extensions contains all custom extensions defined for the top-level document. + Extensions map[string]any + + // JsonSchemaDialect is a 3.1+ property that sets the dialect to use for validating *base.Schema definitions + // - https://spec.openapis.org/oas/v3.1.0#schema-object JsonSchemaDialect string - Webhooks map[string]*PathItem - low *low.Document + + // Webhooks is a 3.1+ property that is similar to callbacks, except, this defines incoming webhooks. + Webhooks map[string]*PathItem + + // Index is a reference to the *index.SpecIndex that was created for the document and used + // as a guide when building out the Document. Ideal if further processing is required on the model and + // the original details are required to continue the work. + // + // This property is not a part of the OpenAPI schema, this is custom to libopenapi. + Index *index.SpecIndex + low *low.Document } +// NewDocument will create a new high-level Document from a low-level one. func NewDocument(document *low.Document) *Document { d := new(Document) d.low = document diff --git a/datamodel/high/v3/document_test.go b/datamodel/high/v3/document_test.go index 2627d3b..299aa27 100644 --- a/datamodel/high/v3/document_test.go +++ b/datamodel/high/v3/document_test.go @@ -12,13 +12,13 @@ import ( "testing" ) -var doc *lowv3.Document +var lowDoc *lowv3.Document func initTest() { data, _ := ioutil.ReadFile("../../../test_specs/burgershop.openapi.yaml") info, _ := datamodel.ExtractSpecInfo(data) var err []error - doc, err = lowv3.CreateDocument(info) + lowDoc, err = lowv3.CreateDocument(info) if err != nil { panic("broken something") } @@ -27,25 +27,25 @@ func initTest() { func BenchmarkNewDocument(b *testing.B) { initTest() for i := 0; i < b.N; i++ { - _ = NewDocument(doc) + _ = NewDocument(lowDoc) } } func TestNewDocument_Extensions(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Equal(t, "darkside", h.Extensions["x-something-something"]) } func TestNewDocument_ExternalDocs(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Equal(t, "https://pb33f.io", h.ExternalDocs.URL) } func TestNewDocument_Info(t *testing.T) { initTest() - highDoc := NewDocument(doc) + highDoc := NewDocument(lowDoc) assert.Equal(t, "3.1.0", highDoc.Version) assert.Equal(t, "Burger Shop", highDoc.Info.Title) assert.Equal(t, "https://pb33f.io", highDoc.Info.TermsOfService) @@ -77,7 +77,7 @@ func TestNewDocument_Info(t *testing.T) { func TestNewDocument_Servers(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Servers, 2) assert.Equal(t, "{scheme}://api.pb33f.io", h.Servers[0].URL) assert.Equal(t, "this is our main API server, for all fun API things.", h.Servers[0].Description) @@ -109,7 +109,7 @@ func TestNewDocument_Servers(t *testing.T) { func TestNewDocument_Tags(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Tags, 2) assert.Equal(t, "Burgers", h.Tags[0].Name) assert.Equal(t, "All kinds of yummy burgers.", h.Tags[0].Description) @@ -131,14 +131,14 @@ func TestNewDocument_Tags(t *testing.T) { func TestNewDocument_Webhooks(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Webhooks, 1) assert.Equal(t, "Information about a new burger", h.Webhooks["someHook"].Post.RequestBody.Description) } func TestNewDocument_Components_Links(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Links, 2) assert.Equal(t, "locateBurger", h.Components.Links["LocateBurger"].OperationId) assert.Equal(t, "$response.body#/id", h.Components.Links["LocateBurger"].Parameters["burgerId"]) @@ -151,7 +151,7 @@ func TestNewDocument_Components_Links(t *testing.T) { func TestNewDocument_Components_Callbacks(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Callbacks, 1) assert.Equal(t, "Callback payload", h.Components.Callbacks["BurgerCallback"].Expression["{$request.query.queryUrl}"].Post.RequestBody.Description) @@ -173,7 +173,7 @@ func TestNewDocument_Components_Callbacks(t *testing.T) { func TestNewDocument_Components_Schemas(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Schemas, 6) goLow := h.Components.GoLow() @@ -215,7 +215,7 @@ func TestNewDocument_Components_Schemas(t *testing.T) { func TestNewDocument_Components_Headers(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Headers, 1) assert.Equal(t, "this is a header", h.Components.Headers["UseOil"].Description) assert.Equal(t, 318, h.Components.Headers["UseOil"].GoLow().Description.ValueNode.Line) @@ -224,7 +224,7 @@ func TestNewDocument_Components_Headers(t *testing.T) { func TestNewDocument_Components_RequestBodies(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.RequestBodies, 1) assert.Equal(t, "Give us the new burger!", h.Components.RequestBodies["BurgerRequest"].Description) assert.Equal(t, 323, h.Components.RequestBodies["BurgerRequest"].GoLow().Description.ValueNode.Line) @@ -234,7 +234,7 @@ func TestNewDocument_Components_RequestBodies(t *testing.T) { func TestNewDocument_Components_Examples(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Examples, 1) assert.Equal(t, "A juicy two hander sammich", h.Components.Examples["QuarterPounder"].Summary) assert.Equal(t, 341, h.Components.Examples["QuarterPounder"].GoLow().Summary.ValueNode.Line) @@ -244,7 +244,7 @@ func TestNewDocument_Components_Examples(t *testing.T) { func TestNewDocument_Components_Responses(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Responses, 1) assert.Equal(t, "all the dressings for a burger.", h.Components.Responses["DressingResponse"].Description) assert.Equal(t, "array", h.Components.Responses["DressingResponse"].Content["application/json"].Schema.Schema().Type[0]) @@ -254,7 +254,7 @@ func TestNewDocument_Components_Responses(t *testing.T) { func TestNewDocument_Components_SecuritySchemes(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.SecuritySchemes, 3) api := h.Components.SecuritySchemes["APIKeyScheme"] @@ -285,7 +285,7 @@ func TestNewDocument_Components_SecuritySchemes(t *testing.T) { func TestNewDocument_Components_Parameters(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Components.Parameters, 2) bh := h.Components.Parameters["BurgerHeader"] assert.Equal(t, "burgerHeader", bh.Name) @@ -302,7 +302,7 @@ func TestNewDocument_Components_Parameters(t *testing.T) { func TestNewDocument_Paths(t *testing.T) { initTest() - h := NewDocument(doc) + h := NewDocument(lowDoc) assert.Len(t, h.Paths.PathItems, 5) burgersOp := h.Paths.PathItems["/burgers"] @@ -350,11 +350,11 @@ func TestStripeAsDoc(t *testing.T) { data, _ := ioutil.ReadFile("../../../test_specs/stripe.yaml") info, _ := datamodel.ExtractSpecInfo(data) var err []error - doc, err = lowv3.CreateDocument(info) + lowDoc, err = lowv3.CreateDocument(info) if err != nil { panic("broken something") } - d := NewDocument(doc) + d := NewDocument(lowDoc) fmt.Println(d) } @@ -362,11 +362,11 @@ func TestAsanaAsDoc(t *testing.T) { data, _ := ioutil.ReadFile("../../../test_specs/asana.yaml") info, _ := datamodel.ExtractSpecInfo(data) var err []error - doc, err = lowv3.CreateDocument(info) + lowDoc, err = lowv3.CreateDocument(info) if err != nil { panic("broken something") } - d := NewDocument(doc) + d := NewDocument(lowDoc) fmt.Println(d) } @@ -374,11 +374,11 @@ func TestPetstoreAsDoc(t *testing.T) { data, _ := ioutil.ReadFile("../../../test_specs/petstorev3.json") info, _ := datamodel.ExtractSpecInfo(data) var err []error - doc, err = lowv3.CreateDocument(info) + lowDoc, err = lowv3.CreateDocument(info) if err != nil { panic("broken something") } - d := NewDocument(doc) + d := NewDocument(lowDoc) fmt.Println(d) } @@ -386,11 +386,11 @@ func TestCircularReferencesDoc(t *testing.T) { data, _ := ioutil.ReadFile("../../../test_specs/circular-tests.yaml") info, _ := datamodel.ExtractSpecInfo(data) var err []error - doc, err = lowv3.CreateDocument(info) + lowDoc, err = lowv3.CreateDocument(info) if err != nil { panic("broken something") } - d := NewDocument(doc) + d := NewDocument(lowDoc) assert.Len(t, d.Components.Schemas, 9) assert.Len(t, d.Index.GetCircularReferences(), 3) } diff --git a/datamodel/high/v3/encoding.go b/datamodel/high/v3/encoding.go index 9af408d..aa6e54f 100644 --- a/datamodel/high/v3/encoding.go +++ b/datamodel/high/v3/encoding.go @@ -8,6 +8,8 @@ import ( low "github.com/pb33f/libopenapi/datamodel/low/v3" ) +// Encoding represents an OpenAPI 3+ Encoding object +// - https://spec.openapis.org/oas/v3.1.0#encoding-object type Encoding struct { ContentType string Headers map[string]*Header @@ -17,6 +19,7 @@ type Encoding struct { low *low.Encoding } +// NewEncoding creates a new instance of Encoding from a low-level one. func NewEncoding(encoding *low.Encoding) *Encoding { e := new(Encoding) e.low = encoding @@ -28,10 +31,12 @@ func NewEncoding(encoding *low.Encoding) *Encoding { return e } +// GoLow returns the low-level Encoding instance used to create the high-level one. func (e *Encoding) GoLow() *low.Encoding { return e.low } +// ExtractEncoding converts hard to navigate low-level plumbing Encoding definitions, into high-level simple map func ExtractEncoding(elements map[lowmodel.KeyReference[string]]lowmodel.ValueReference[*low.Encoding]) map[string]*Encoding { extracted := make(map[string]*Encoding) for k, v := range elements { diff --git a/datamodel/high/v3/package_test.go b/datamodel/high/v3/package_test.go new file mode 100644 index 0000000..a9e9f4e --- /dev/null +++ b/datamodel/high/v3/package_test.go @@ -0,0 +1,41 @@ +// Copyright 2022 Princess B33f Heavy Industries / Dave Shanley +// SPDX-License-Identifier: MIT + +package v3 + +import ( + "fmt" + "github.com/pb33f/libopenapi/datamodel" + lowv3 "github.com/pb33f/libopenapi/datamodel/low/v3" + "io/ioutil" +) + +// Creating a new high-level OpenAPI 3+ document from an OpenAPI specification. +func Example() { + // Load in an OpenAPI 3+ specification as a byte slice. + data, _ := ioutil.ReadFile("../../../test_specs/petstorev3.json") + + // Create a new *datamodel.SpecInfo from bytes. + info, _ := datamodel.ExtractSpecInfo(data) + + var err []error + + // Create a new low-level Document, capture any errors thrown during creation. + lowDoc, err = lowv3.CreateDocument(info) + + // Get upset if any errors were thrown. + if len(err) > 0 { + for i := range err { + fmt.Printf("error: %e", err[i]) + } + panic("something went wrong") + } + + // Create a high-level Document from the low-level one. + doc := NewDocument(lowDoc) + + // Print out some details + fmt.Printf("Petstore contains %d paths and %d component schemas", + len(doc.Paths.PathItems), len(doc.Components.Schemas)) + // Output: Petstore contains 13 paths and 8 component schemas +}