diff --git a/document_examples_test.go b/document_examples_test.go index 61201d8..c9f9de6 100644 --- a/document_examples_test.go +++ b/document_examples_test.go @@ -101,7 +101,7 @@ func ExampleNewDocument_fromWithDocumentConfigurationFailure() { if len(errors) > 0 { fmt.Println("Error building Digital Ocean spec errors reported") } - // Output: There are 474 errors logged + // Output: There are 475 errors logged //Error building Digital Ocean spec errors reported } diff --git a/index/extract_refs.go b/index/extract_refs.go index a5b68dc..373581f 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -225,7 +225,7 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, } else { // if the index has a base path, use that to resolve the path - if index.config.BasePath != "" { + if index.config.BasePath != "" && index.config.BaseURL == nil { abs, _ := filepath.Abs(filepath.Join(index.config.BasePath, uri[0])) if abs != defRoot { abs, _ = filepath.Abs(filepath.Join(defRoot, uri[0])) @@ -234,13 +234,19 @@ func (index *SpecIndex) ExtractRefs(node, parent *yaml.Node, seenPath []string, componentName = fmt.Sprintf("#/%s", uri[1]) } else { // if the index has a base URL, use that to resolve the path. - if index.config.BaseURL != nil { + if index.config.BaseURL != nil && !filepath.IsAbs(defRoot) { u := *index.config.BaseURL abs, _ := filepath.Abs(filepath.Join(u.Path, uri[0])) u.Path = abs fullDefinitionPath = fmt.Sprintf("%s#/%s", u.String(), uri[1]) componentName = fmt.Sprintf("#/%s", uri[1]) + + } else { + + abs, _ := filepath.Abs(filepath.Join(defRoot, uri[0])) + fullDefinitionPath = fmt.Sprintf("%s#/%s", abs, uri[1]) + componentName = fmt.Sprintf("#/%s", uri[1]) } } } diff --git a/index/find_component.go b/index/find_component.go index 83e795f..e05ae09 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -44,6 +44,7 @@ func (index *SpecIndex) FindComponent(componentId string) *Reference { // root search return index.FindComponentInRoot(componentId) } + return nil } func FindComponent(root *yaml.Node, componentId, absoluteFilePath string, index *SpecIndex) *Reference { diff --git a/index/resolver.go b/index/resolver.go index 38efc23..5006b00 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -330,6 +330,8 @@ func (resolver *Resolver) isInfiniteCircularDependency(ref *Reference, visitedDe return false, visitedDefinitions } + // TODO: pick up here, required ref properties are not extracted correctly. + for refDefinition := range ref.RequiredRefProperties { r, _ := resolver.specIndex.SearchIndexForReference(refDefinition) if initialRef != nil && initialRef.Definition == r.Definition { @@ -399,6 +401,7 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No u, _ := url.Parse(httpExp[0]) abs, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0])) u.Path = abs + u.Fragment = "" fullDef = fmt.Sprintf("%s#/%s", u.String(), exp[1]) } else { @@ -412,7 +415,9 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No fileDef := strings.Split(ref.FullDefinition, "#/") // extract the location of the ref and build a full def path. - fullDef, _ = filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0])) + abs, _ := filepath.Abs(filepath.Join(filepath.Dir(fileDef[0]), exp[0])) + fullDef = fmt.Sprintf("%s#/%s", abs, exp[1]) + } } @@ -518,11 +523,30 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No if _, v := utils.FindKeyNodeTop("items", node.Content[i+1].Content); v != nil { if utils.IsNodeMap(v) { if d, _, l := utils.IsNodeRefValue(v); d { - mappedRefs := resolver.specIndex.GetMappedReferences()[l] + + // create full definition lookup based on ref. + def := ref.FullDefinition + exp := strings.Split(ref.FullDefinition, "#/") + if len(exp) == 2 { + if exp[0] != "" { + if !strings.HasPrefix(ref.FullDefinition, "http") { + if !filepath.IsAbs(exp[0]) { + abs, _ := filepath.Abs(fmt.Sprintf("%s#/%s", filepath.Dir(ref.FullDefinition), exp[0])) + def = fmt.Sprintf("%s#/%s", abs, l) + } + } + } + } else { + if !strings.Contains(ref.FullDefinition, "#") { + + } + } + + mappedRefs := resolver.specIndex.GetMappedReferences()[def] if mappedRefs != nil && !mappedRefs.Circular { circ := false for f := range journey { - if journey[f].Definition == mappedRefs.Definition { + if journey[f].FullDefinition == mappedRefs.FullDefinition { circ = true break } @@ -560,11 +584,78 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No v := node.Content[i+1].Content[q] if utils.IsNodeMap(v) { if d, _, l := utils.IsNodeRefValue(v); d { - mappedRefs := resolver.specIndex.GetMappedReferences()[l] + + // create full definition lookup based on ref. + def := l + exp := strings.Split(l, "#/") + if len(exp) == 2 { + if exp[0] != "" { + if !strings.HasPrefix(exp[0], "http") { + if !filepath.IsAbs(exp[0]) { + + if strings.HasPrefix(ref.FullDefinition, "http") { + + u, _ := url.Parse(ref.FullDefinition) + p, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), exp[0])) + u.Path = p + def = fmt.Sprintf("%s#/%s", u.String(), exp[1]) + + } else { + abs, _ := filepath.Abs(filepath.Join(filepath.Dir(ref.FullDefinition), exp[0])) + def = fmt.Sprintf("%s#/%s", abs, exp[1]) + } + } + } else { + panic("mummmmma mia") + } + + } else { + if strings.HasPrefix(ref.FullDefinition, "http") { + u, _ := url.Parse(ref.FullDefinition) + u.Fragment = "" + def = fmt.Sprintf("%s#/%s", u.String(), exp[1]) + + } else { + if strings.HasPrefix(ref.FullDefinition, "#/") { + def = fmt.Sprintf("#/%s", exp[1]) + } else { + def = fmt.Sprintf("%s#/%s", ref.FullDefinition, exp[1]) + } + } + } + } else { + + if strings.HasPrefix(l, "http") { + def = l + } else { + if filepath.IsAbs(l) { + def = l + } else { + + // check if were dealing with a remote file + if strings.HasPrefix(ref.FullDefinition, "http") { + + // split the url. + u, _ := url.Parse(ref.FullDefinition) + abs, _ := filepath.Abs(filepath.Join(filepath.Dir(u.Path), l)) + u.Path = abs + u.Fragment = "" + def = u.String() + } else { + lookupRef := strings.Split(ref.FullDefinition, "#/") + abs, _ := filepath.Abs(filepath.Join(filepath.Dir(lookupRef[0]), l)) + def = abs + } + } + } + //panic("oh no") + } + + mappedRefs, _ := resolver.specIndex.SearchIndexForReference(def) if mappedRefs != nil && !mappedRefs.Circular { circ := false for f := range journey { - if journey[f].Definition == mappedRefs.Definition { + if journey[f].FullDefinition == mappedRefs.FullDefinition { circ = true break } diff --git a/index/rolodex.go b/index/rolodex.go index 9d3570a..f377d63 100644 --- a/index/rolodex.go +++ b/index/rolodex.go @@ -13,6 +13,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "sync" "time" ) @@ -272,21 +273,24 @@ func (r *Rolodex) IndexTheRolodex() error { copiedConfig.AvoidBuildIndex = true // we will build out everything in two steps. idx, err := idxFile.Index(&copiedConfig) - // for each index, we need a resolver - resolver := NewResolver(idx) - - // check if the config has been set to ignore circular references in arrays and polymorphic schemas - if copiedConfig.IgnoreArrayCircularReferences { - resolver.IgnoreArrayCircularReferences() - } - if copiedConfig.IgnorePolymorphicCircularReferences { - resolver.IgnorePolymorphicCircularReferences() - } - if err != nil { errChan <- err } - indexChan <- idx + + if err == nil { + // for each index, we need a resolver + resolver := NewResolver(idx) + + // check if the config has been set to ignore circular references in arrays and polymorphic schemas + if copiedConfig.IgnoreArrayCircularReferences { + resolver.IgnoreArrayCircularReferences() + } + if copiedConfig.IgnorePolymorphicCircularReferences { + resolver.IgnorePolymorphicCircularReferences() + } + indexChan <- idx + } + } if lfs, ok := fs.(RolodexFS); ok { @@ -339,6 +343,11 @@ func (r *Rolodex) IndexTheRolodex() error { // now that we have indexed all the files, we can build the index. r.indexes = indexBuildQueue //if !r.indexConfig.AvoidBuildIndex { + + sort.Slice(indexBuildQueue, func(i, j int) bool { + return indexBuildQueue[i].specAbsolutePath < indexBuildQueue[j].specAbsolutePath + }) + for _, idx := range indexBuildQueue { idx.BuildIndex() if r.indexConfig.AvoidCircularReferenceCheck { diff --git a/index/rolodex_remote_loader.go b/index/rolodex_remote_loader.go index 643ae6b..1744be9 100644 --- a/index/rolodex_remote_loader.go +++ b/index/rolodex_remote_loader.go @@ -264,11 +264,17 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { // can we block threads. if runtime.GOMAXPROCS(-1) > 2 { i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String()) - for { - if wf, ko := i.Files.Load(remoteParsedURL.Path); ko { - return wf.(*RemoteFile), nil + + f := make(chan *RemoteFile) + fwait := func(path string, c chan *RemoteFile) { + for { + if wf, ko := i.Files.Load(remoteParsedURL.Path); ko { + c <- wf.(*RemoteFile) + } } } + go fwait(remoteParsedURL.Path, f) + return <-f, nil } } @@ -366,12 +372,16 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { copiedCfg.BaseURL = newBaseURL } copiedCfg.SpecAbsolutePath = remoteParsedURL.String() - idx, idxError := remoteFile.Index(&copiedCfg) if len(remoteFile.data) > 0 { i.logger.Debug("successfully loaded file", "file", absolutePath) } //i.seekRelatives(remoteFile) + // remove from processing + i.ProcessingFiles.Delete(remoteParsedURL.Path) + i.Files.Store(absolutePath, remoteFile) + + idx, idxError := remoteFile.Index(&copiedCfg) if idxError != nil && idx == nil { i.remoteErrors = append(i.remoteErrors, idxError) @@ -383,10 +393,6 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) { idx.BuildIndex() } - // remove from processing - i.ProcessingFiles.Delete(remoteParsedURL.Path) - i.Files.Store(absolutePath, remoteFile) - //if !i.remoteRunning { return remoteFile, errors.Join(i.remoteErrors...) // } else { diff --git a/index/rolodex_test.go b/index/rolodex_test.go index 18905ba..e65b4a0 100644 --- a/index/rolodex_test.go +++ b/index/rolodex_test.go @@ -245,7 +245,7 @@ components: first := `openapi: 3.1.0 components: schemas: - CircleTest: + StartTest: type: object required: - muffins @@ -268,9 +268,9 @@ components: BaseDirectory: baseDir, DirFS: os.DirFS(baseDir), FileFilters: []string{ - // "first.yaml", - // "second.yaml", - // "third.yaml", + "first.yaml", + "second.yaml", + "third.yaml", "fourth.yaml", }, } @@ -300,10 +300,11 @@ components: assert.NoError(t, err) assert.Len(t, rolodex.GetCaughtErrors(), 0) - // there are two circles. Once when reading the journey from first.yaml, and then a second internal look in second.yaml + // there are three circles. Once when reading the journey from first.yaml, and then a second internal look in second.yaml // the index won't find three, because by the time that 'three' has been read, it's already been indexed and the journey - // discovered. - assert.Len(t, rolodex.GetIgnoredCircularReferences(), 2) + // discovered. The third is the entirely 'new' circle that is sucked down via `fourth.yaml` from the simulated remote server, which contains + // all the same specs, it's just they are now being sucked in remotely. + assert.Len(t, rolodex.GetIgnoredCircularReferences(), 3) } func test_rolodexDeepRefServer(a, b, c []byte) *httptest.Server { diff --git a/index/search_index.go b/index/search_index.go index 0633629..49b87f0 100644 --- a/index/search_index.go +++ b/index/search_index.go @@ -138,7 +138,18 @@ func (index *SpecIndex) SearchIndexForReferenceByReferenceWithContext(ctx contex // does component exist in the root? node, _ := rFile.GetContentAsYAMLNode() if node != nil { - found := idx.FindComponent(ref) + var found *Reference + exp := strings.Split(ref, "#/") + compId := ref + + if len(exp) == 2 { + compId = fmt.Sprintf("#/%s", exp[1]) + found = FindComponent(node, compId, exp[0], idx) + } + if found == nil { + found = idx.FindComponent(ref) + } + if found != nil { idx.cache.Store(ref, found) index.cache.Store(ref, found) diff --git a/index/utility_methods.go b/index/utility_methods.go index 5393b7e..a282bc5 100644 --- a/index/utility_methods.go +++ b/index/utility_methods.go @@ -103,11 +103,11 @@ func extractDefinitionRequiredRefProperties(schemaNode *yaml.Node, reqRefProps m // extractRequiredReferenceProperties returns a map of definition names to the property or properties which reference it within a node func extractRequiredReferenceProperties(fulldef string, requiredPropDefNode *yaml.Node, propName string, reqRefProps map[string][]string) map[string][]string { - isRef, _, defPath := utils.IsNodeRefValue(requiredPropDefNode) + isRef, _, _ := utils.IsNodeRefValue(requiredPropDefNode) if !isRef { _, defItems := utils.FindKeyNodeTop("items", requiredPropDefNode.Content) if defItems != nil { - isRef, _, defPath = utils.IsNodeRefValue(defItems) + isRef, _, _ = utils.IsNodeRefValue(defItems) } } @@ -115,8 +115,10 @@ func extractRequiredReferenceProperties(fulldef string, requiredPropDefNode *yam return reqRefProps } + defPath := fulldef + // explode defpath - exp := strings.Split(defPath, "#/") + exp := strings.Split(fulldef, "#/") if len(exp) == 2 { if exp[0] != "" { if !strings.HasPrefix(exp[0], "http") { @@ -141,6 +143,7 @@ func extractRequiredReferenceProperties(fulldef string, requiredPropDefNode *yam } } + } else { if strings.HasPrefix(exp[0], "http") {