Changed remote loader to use a timeout

rather than a hard block, it will wait 50ms then try again, regardless of cores, so it won’t ever block fully.

Signed-off-by: quobix <dave@quobix.com>

index/rolodex_remote_loader.go

@@ -4,17 +4,16 @@
 package index
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"github.com/pb33f/libopenapi/datamodel"
 	"github.com/pb33f/libopenapi/utils"
-	"log/slog"
-	"runtime"
-
 	"golang.org/x/sync/syncmap"
 	"gopkg.in/yaml.v3"
 	"io"
 	"io/fs"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"os"
@@ -261,11 +260,11 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 	// if we're processing, we need to block and wait for the file to be processed
 	// try path first
 	if _, ok := i.ProcessingFiles.Load(remoteParsedURL.Path); ok {
-		// we can't block if we only have a couple of CPUs, as we'll deadlock / run super slow, only when we're running in parallel
-		// can we block threads.
-		if runtime.GOMAXPROCS(-1) > 2 {
-			i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String())
 
+		i.logger.Debug("waiting for existing fetch to complete", "file", remoteURL, "remoteURL", remoteParsedURL.String())
+		// Create a context with a timeout of 50ms
+		ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Millisecond*50)
+		defer cancel()
 		f := make(chan *RemoteFile)
 		fwait := func(path string, c chan *RemoteFile) {
 			for {
@@ -275,7 +274,12 @@ func (i *RemoteFS) Open(remoteURL string) (fs.File, error) {
 			}
 		}
 		go fwait(remoteParsedURL.Path, f)
-			return <-f, nil
+
+		select {
+		case <-ctxTimeout.Done():
+			i.logger.Info("waiting for remote file timed out, trying again", "file", remoteURL, "remoteURL", remoteParsedURL.String())
+		case v := <-f:
+			return v, nil
 		}
 	}
 

index/spec_index.go

@@ -13,19 +13,16 @@
 package index
 
 import (
-	"context"
 	"fmt"
+	"github.com/pb33f/libopenapi/utils"
+	"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
 	"golang.org/x/sync/syncmap"
+	"gopkg.in/yaml.v3"
 	"log/slog"
 	"os"
 	"sort"
 	"strings"
 	"sync"
-	"time"
-
-	"github.com/pb33f/libopenapi/utils"
-	"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
-	"gopkg.in/yaml.v3"
 )
 
 // NewSpecIndexWithConfig will create a new index of an OpenAPI or Swagger spec. It uses the same logic as NewSpecIndex
@@ -696,29 +693,9 @@ func (index *SpecIndex) GetGlobalLinksCount() int {
 
             // look through method for links
             links, _ := yamlpath.NewPath("$..links")
-
-			// Channel used to receive the result from doSomething function
-			ch := make(chan string, 1)
-
-			// Create a context with a timeout of 5 seconds
-			ctxTimeout, cancel := context.WithTimeout(context.Background(), time.Millisecond*500)
-			defer cancel()
-
             var res []*yaml.Node
 
-			doSomething := func(ctx context.Context, ch chan<- string) {
             res, _ = links.Find(m.Node)
-				ch <- m.Definition
-			}
-
-			// Start the doSomething function
-			go doSomething(ctxTimeout, ch)
-
-			select {
-			case <-ctxTimeout.Done():
-				fmt.Printf("Global links %d ref: Context cancelled: %v\n", m.Node.Line, ctxTimeout.Err())
-			case <-ch:
-			}
 
             if len(res) > 0 {
                 for _, link := range res[0].Content {

index/spec_index_test.go

@@ -142,7 +142,7 @@ func TestSpecIndex_DigitalOcean(t *testing.T) {
 	cf.AllowRemoteLookup = true
 	cf.AvoidCircularReferenceCheck = true
 	cf.Logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
-		Level: slog.LevelError,
+		Level: slog.LevelInfo,
 	}))
 
 	// setting this baseURL will override the base