LukeParke/libopenapi
1
0
Fork 0
mirror of https://github.com/LukeHagar/libopenapi.git synced 2025-12-10 12:37:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Re-enabled JSON Parsing async channel

Browse Source 
The channel is used by vacuum and the validator, it is required for schema validation. but it also slows things down considerably when done synchronously. I have moved this code back to async, it cuts parsing time in half for vaccum, and restores super speed.

Signed-off-by: quobix <dave@quobix.com>
This commit is contained in:
quobix
2023-11-09 06:30:13 -05:00
parent 6a6d6d6e31
commit 8b9ef11270
2 changed files with 38 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
datamodel/spec_info.go
View File

@@ -101,17 +101,19 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro
spec.APISchema = OpenAPI2SchemaData spec.APISchema = OpenAPI2SchemaData
} }
if utils.IsYAML(string(bytes)) { go func() {
_ = parsedNode.Decode(&jsonSpec) if utils.IsYAML(string(bytes)) {
b, _ := json.Marshal(&jsonSpec) _ = parsedNode.Decode(&jsonSpec)
spec.SpecJSONBytes = &b b, _ := json.Marshal(&jsonSpec)
spec.SpecJSON = &jsonSpec spec.SpecJSONBytes = &b
} else { spec.SpecJSON = &jsonSpec
_ = json.Unmarshal(bytes, &jsonSpec) } else {
spec.SpecJSONBytes = &bytes _ = json.Unmarshal(bytes, &jsonSpec)
spec.SpecJSON = &jsonSpec spec.SpecJSONBytes = &bytes
} spec.SpecJSON = &jsonSpec
close(spec.JsonParsingChannel) // this needs removing at some point }
close(spec.JsonParsingChannel)
}()
} }
if !bypass { if !bypass {
@@ -177,23 +179,25 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro
if specInfo.SpecType == "" { if specInfo.SpecType == "" {
// parse JSON // parse JSON
parseJSON(spec, specInfo, &parsedSpec) go parseJSON(spec, specInfo, &parsedSpec)
specInfo.Error = errors.New("spec type not supported by libopenapi, sorry") specInfo.Error = errors.New("spec type not supported by libopenapi, sorry")
return specInfo, specInfo.Error return specInfo, specInfo.Error
} }
} else { } else {
var jsonSpec map[string]interface{} go func() {
if utils.IsYAML(string(spec)) { var jsonSpec map[string]interface{}
_ = parsedSpec.Decode(&jsonSpec) if utils.IsYAML(string(spec)) {
b, _ := json.Marshal(&jsonSpec) _ = parsedSpec.Decode(&jsonSpec)
specInfo.SpecJSONBytes = &b b, _ := json.Marshal(&jsonSpec)
specInfo.SpecJSON = &jsonSpec specInfo.SpecJSONBytes = &b
} else { specInfo.SpecJSON = &jsonSpec
_ = json.Unmarshal(spec, &jsonSpec) } else {
specInfo.SpecJSONBytes = &spec _ = json.Unmarshal(spec, &jsonSpec)
specInfo.SpecJSON = &jsonSpec specInfo.SpecJSONBytes = &spec
} specInfo.SpecJSON = &jsonSpec
close(specInfo.JsonParsingChannel) // this needs removing at some point }
close(specInfo.JsonParsingChannel) // this needs removing at some point
}()
} }
// detect the original whitespace indentation // detect the original whitespace indentation

10
datamodel/spec_info_test.go
View File

@@ -116,6 +116,7 @@ info:
func TestExtractSpecInfo_ValidJSON(t *testing.T) { func TestExtractSpecInfo_ValidJSON(t *testing.T) {
r, e := ExtractSpecInfo([]byte(goodJSON)) r, e := ExtractSpecInfo([]byte(goodJSON))
<-r.JsonParsingChannel
assert.Greater(t, len(*r.SpecJSONBytes), 0) assert.Greater(t, len(*r.SpecJSONBytes), 0)
assert.Error(t, e) assert.Error(t, e)
} }
@@ -132,6 +133,7 @@ func TestExtractSpecInfo_Nothing(t *testing.T) {
func TestExtractSpecInfo_ValidYAML(t *testing.T) { func TestExtractSpecInfo_ValidYAML(t *testing.T) {
r, e := ExtractSpecInfo([]byte(goodYAML)) r, e := ExtractSpecInfo([]byte(goodYAML))
<-r.JsonParsingChannel
assert.Greater(t, len(*r.SpecJSONBytes), 0) assert.Greater(t, len(*r.SpecJSONBytes), 0)
assert.Error(t, e) assert.Error(t, e)
} }
@@ -149,6 +151,7 @@ func TestExtractSpecInfo_InvalidOpenAPIVersion(t *testing.T) {
func TestExtractSpecInfo_OpenAPI3(t *testing.T) { func TestExtractSpecInfo_OpenAPI3(t *testing.T) {
r, e := ExtractSpecInfo([]byte(OpenApi3Spec)) r, e := ExtractSpecInfo([]byte(OpenApi3Spec))
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.Equal(t, utils.OpenApi3, r.SpecType) assert.Equal(t, utils.OpenApi3, r.SpecType)
assert.Equal(t, "3.0.1", r.Version) assert.Equal(t, "3.0.1", r.Version)
@@ -159,6 +162,7 @@ func TestExtractSpecInfo_OpenAPI3(t *testing.T) {
func TestExtractSpecInfo_OpenAPIWat(t *testing.T) { func TestExtractSpecInfo_OpenAPIWat(t *testing.T) {
r, e := ExtractSpecInfo([]byte(OpenApiWat)) r, e := ExtractSpecInfo([]byte(OpenApiWat))
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.Equal(t, OpenApi3, r.SpecType) assert.Equal(t, OpenApi3, r.SpecType)
assert.Equal(t, "3.2", r.Version) assert.Equal(t, "3.2", r.Version)
@@ -167,6 +171,7 @@ func TestExtractSpecInfo_OpenAPIWat(t *testing.T) {
func TestExtractSpecInfo_OpenAPI31(t *testing.T) { func TestExtractSpecInfo_OpenAPI31(t *testing.T) {
r, e := ExtractSpecInfo([]byte(OpenApi31)) r, e := ExtractSpecInfo([]byte(OpenApi31))
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.Equal(t, OpenApi3, r.SpecType) assert.Equal(t, OpenApi3, r.SpecType)
assert.Equal(t, "3.1", r.Version) assert.Equal(t, "3.1", r.Version)
@@ -183,6 +188,7 @@ why:
yes: no` yes: no`
r, e := ExtractSpecInfoWithDocumentCheck([]byte(random), true) r, e := ExtractSpecInfoWithDocumentCheck([]byte(random), true)
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.NotNil(t, r.RootNode) assert.NotNil(t, r.RootNode)
assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value) assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value)
@@ -194,6 +200,7 @@ func TestExtractSpecInfo_AnyDocument_JSON(t *testing.T) {
random := `{ "something" : "yeah"}` random := `{ "something" : "yeah"}`
r, e := ExtractSpecInfoWithDocumentCheck([]byte(random), true) r, e := ExtractSpecInfoWithDocumentCheck([]byte(random), true)
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.NotNil(t, r.RootNode) assert.NotNil(t, r.RootNode)
assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value) assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value)
@@ -212,6 +219,7 @@ why:
r, e := ExtractSpecInfoWithConfig([]byte(random), &DocumentConfiguration{ r, e := ExtractSpecInfoWithConfig([]byte(random), &DocumentConfiguration{
BypassDocumentCheck: true, BypassDocumentCheck: true,
}) })
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.NotNil(t, r.RootNode) assert.NotNil(t, r.RootNode)
assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value) assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value)
@@ -228,6 +236,7 @@ func TestExtractSpecInfo_OpenAPIFalse(t *testing.T) {
func TestExtractSpecInfo_OpenAPI2(t *testing.T) { func TestExtractSpecInfo_OpenAPI2(t *testing.T) {
r, e := ExtractSpecInfo([]byte(OpenApi2Spec)) r, e := ExtractSpecInfo([]byte(OpenApi2Spec))
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.Equal(t, OpenApi2, r.SpecType) assert.Equal(t, OpenApi2, r.SpecType)
assert.Equal(t, "2.0.1", r.Version) assert.Equal(t, "2.0.1", r.Version)
@@ -246,6 +255,7 @@ func TestExtractSpecInfo_OpenAPI2_OddVersion(t *testing.T) {
func TestExtractSpecInfo_AsyncAPI(t *testing.T) { func TestExtractSpecInfo_AsyncAPI(t *testing.T) {
r, e := ExtractSpecInfo([]byte(AsyncAPISpec)) r, e := ExtractSpecInfo([]byte(AsyncAPISpec))
<-r.JsonParsingChannel
assert.Nil(t, e) assert.Nil(t, e)
assert.Equal(t, AsyncApi, r.SpecType) assert.Equal(t, AsyncApi, r.SpecType)
assert.Equal(t, "2.0.0", r.Version) assert.Equal(t, "2.0.0", r.Version)