libopenapi/index/rolodex.go

// Copyright 2023 Princess B33f Heavy Industries / Dave Shanley
// SPDX-License-Identifier: MIT

package index

import (
	"errors"
	"fmt"
	"github.com/pb33f/libopenapi/datamodel"
	"gopkg.in/yaml.v3"
	"io"
	"io/fs"
	"log/slog"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type HasIndex interface {
	GetIndex() *SpecIndex
}

type CanBeIndexed interface {
	Index(config *SpecIndexConfig) (*SpecIndex, error)
}

type RolodexFile interface {
	GetContent() string
	GetFileExtension() FileExtension
	GetFullPath() string
	GetErrors() []error
	GetContentAsYAMLNode() (*yaml.Node, error)
	GetIndex() *SpecIndex
	Name() string
	ModTime() time.Time
	IsDir() bool
	Sys() any
	Size() int64
	Mode() os.FileMode
}

var logger *slog.Logger

func init() {
	logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
		Level: slog.LevelDebug,
	}))

}

type RolodexFS interface {
	Open(name string) (fs.File, error)
	GetFiles() map[string]RolodexFile
}

type Rolodex struct {
	localFS                   map[string]fs.FS
	remoteFS                  map[string]fs.FS
	indexed                   bool
	built                     bool
	resolved                  bool
	circChecked               bool
	indexConfig               *SpecIndexConfig
	indexingDuration          time.Duration
	indexes                   []*SpecIndex
	rootIndex                 *SpecIndex
	rootNode                  *yaml.Node
	caughtErrors              []error
	ignoredCircularReferences []*CircularReferenceResult
}

type rolodexFile struct {
	location   string
	rolodex    *Rolodex
	index      *SpecIndex
	localFile  *LocalFile
	remoteFile *RemoteFile
}

func (rf *rolodexFile) Name() string {
	if rf.localFile != nil {
		return rf.localFile.filename
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.filename
	}
	return ""
}

func (rf *rolodexFile) GetIndex() *SpecIndex {
	if rf.localFile != nil {
		return rf.localFile.GetIndex()
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.GetIndex()
	}
	return nil
}

func (rf *rolodexFile) Index(config *SpecIndexConfig) (*SpecIndex, error) {
	if rf.index != nil {
		return rf.index, nil
	}
	var content []byte
	if rf.localFile != nil {
		content = rf.localFile.data
	}
	if rf.remoteFile != nil {
		content = rf.remoteFile.data
	}

	// first, we must parse the content of the file
	info, err := datamodel.ExtractSpecInfo(content)
	if err != nil {
		return nil, err
	}

	// create a new index for this file and link it to this rolodex.
	config.Rolodex = rf.rolodex
	index := NewSpecIndexWithConfig(info.RootNode, config)
	rf.index = index
	return index, nil

}

func (rf *rolodexFile) GetContent() string {
	if rf.localFile != nil {
		return string(rf.localFile.data)
	}
	if rf.remoteFile != nil {
		return string(rf.remoteFile.data)
	}
	return ""
}

func (rf *rolodexFile) GetContentAsYAMLNode() (*yaml.Node, error) {
	if rf.localFile != nil {
		return rf.localFile.GetContentAsYAMLNode()
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.GetContentAsYAMLNode()
	}
	return nil, nil
}

func (rf *rolodexFile) GetFileExtension() FileExtension {
	if rf.localFile != nil {
		return rf.localFile.extension
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.extension
	}
	return UNSUPPORTED
}
func (rf *rolodexFile) GetFullPath() string {
	if rf.localFile != nil {
		return rf.localFile.fullPath
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.fullPath
	}
	return ""
}
func (rf *rolodexFile) ModTime() time.Time {
	if rf.localFile != nil {
		return rf.localFile.lastModified
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.lastModified
	}
	return time.Time{}
}

func (rf *rolodexFile) Size() int64 {
	if rf.localFile != nil {
		return rf.localFile.Size()
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.Size()
	}
	return 0
}

func (rf *rolodexFile) IsDir() bool {
	return false
}

func (rf *rolodexFile) Sys() interface{} {
	return nil
}

func (rf *rolodexFile) Mode() os.FileMode {
	if rf.localFile != nil {
		return rf.localFile.Mode()
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.Mode()
	}
	return os.FileMode(0)
}

func (rf *rolodexFile) GetErrors() []error {
	if rf.localFile != nil {
		return rf.localFile.readingErrors
	}
	if rf.remoteFile != nil {
		return rf.remoteFile.seekingErrors
	}
	return nil
}

func NewRolodex(indexConfig *SpecIndexConfig) *Rolodex {
	r := &Rolodex{
		indexConfig: indexConfig,
		localFS:     make(map[string]fs.FS),
		remoteFS:    make(map[string]fs.FS),
	}
	indexConfig.Rolodex = r
	return r
}

func (r *Rolodex) GetIgnoredCircularReferences() []*CircularReferenceResult {
	return r.ignoredCircularReferences
}

func (r *Rolodex) GetIndexingDuration() time.Duration {
	return r.indexingDuration
}

func (r *Rolodex) GetRootIndex() *SpecIndex {
	return r.rootIndex
}

func (r *Rolodex) GetIndexes() []*SpecIndex {
	return r.indexes
}

func (r *Rolodex) GetCaughtErrors() []error {
	return r.caughtErrors
}

func (r *Rolodex) AddLocalFS(baseDir string, fileSystem fs.FS) {
	absBaseDir, _ := filepath.Abs(baseDir)
	r.localFS[absBaseDir] = fileSystem
}

func (r *Rolodex) SetRootNode(node *yaml.Node) {
	r.rootNode = node
}

func (r *Rolodex) AddRemoteFS(baseURL string, fileSystem fs.FS) {
	r.remoteFS[baseURL] = fileSystem
}

func (r *Rolodex) IndexTheRolodex() error {
	if r.indexed {
		return nil
	}

	var caughtErrors []error

	var indexBuildQueue []*SpecIndex

	indexRolodexFile := func(
		location string, fs fs.FS,
		doneChan chan bool,
		errChan chan error,
		indexChan chan *SpecIndex) {

		var wg sync.WaitGroup

		indexFileFunc := func(idxFile CanBeIndexed, fullPath string) {
			defer wg.Done()

			// copy config and set the
			copiedConfig := *r.indexConfig
			copiedConfig.SpecAbsolutePath = fullPath
			copiedConfig.AvoidBuildIndex = true // we will build out everything in two steps.
			idx, err := idxFile.Index(&copiedConfig)

			// for each index, we need a resolver
			resolver := NewResolver(idx)

			// check if the config has been set to ignore circular references in arrays and polymorphic schemas
			if copiedConfig.IgnoreArrayCircularReferences {
				resolver.IgnoreArrayCircularReferences()
			}
			if copiedConfig.IgnorePolymorphicCircularReferences {
				resolver.IgnorePolymorphicCircularReferences()
			}

			if err != nil {
				errChan <- err
			}
			indexChan <- idx
		}

		if lfs, ok := fs.(RolodexFS); ok {
			wait := false
			for _, f := range lfs.GetFiles() {
				if idxFile, ko := f.(CanBeIndexed); ko {
					wg.Add(1)
					wait = true
					go indexFileFunc(idxFile, f.GetFullPath())
				}
			}
			if wait {
				wg.Wait()
			}
			doneChan <- true
			return
		} else {
			errChan <- errors.New("rolodex file system is not a RolodexFS")
			doneChan <- true
		}
	}

	indexingCompleted := 0
	totalToIndex := len(r.localFS) + len(r.remoteFS)
	doneChan := make(chan bool)
	errChan := make(chan error)
	indexChan := make(chan *SpecIndex)

	// run through every file system and index every file, fan out as many goroutines as possible.
	started := time.Now()
	for k, v := range r.localFS {
		go indexRolodexFile(k, v, doneChan, errChan, indexChan)
	}
	for k, v := range r.remoteFS {
		go indexRolodexFile(k, v, doneChan, errChan, indexChan)
	}

	for indexingCompleted < totalToIndex {
		select {
		case <-doneChan:
			indexingCompleted++
		case err := <-errChan:
			indexingCompleted++
			caughtErrors = append(caughtErrors, err)
		case idx := <-indexChan:
			indexBuildQueue = append(indexBuildQueue, idx)
		}
	}

	// now that we have indexed all the files, we can build the index.
	r.indexes = indexBuildQueue
	//if !r.indexConfig.AvoidBuildIndex {
	for _, idx := range indexBuildQueue {
		idx.BuildIndex()
		if r.indexConfig.AvoidCircularReferenceCheck {
			continue
		}
		errs := idx.resolver.CheckForCircularReferences()
		for e := range errs {
			caughtErrors = append(caughtErrors, errs[e])
		}
	}

	// indexed and built every supporting file, we can build the root index (our entry point)
	if r.rootNode != nil {

		// if there is a base path, then we need to set the root spec config to point to a theoretical root.yaml
		// which does not exist, but is used to formulate the absolute path to root references correctly.
		if r.indexConfig.BasePath != "" && r.indexConfig.BaseURL == nil {

			basePath := r.indexConfig.BasePath
			if !filepath.IsAbs(basePath) {
				basePath, _ = filepath.Abs(basePath)
			}

			if len(r.localFS) > 0 || len(r.remoteFS) > 0 {
				r.indexConfig.SpecAbsolutePath = filepath.Join(basePath, "root.yaml")
			}
		}

		// todo: variation with no base path, but a base URL.

		index := NewSpecIndexWithConfig(r.rootNode, r.indexConfig)
		resolver := NewResolver(index)

		if r.indexConfig.IgnoreArrayCircularReferences {
			resolver.IgnoreArrayCircularReferences()
		}
		if r.indexConfig.IgnorePolymorphicCircularReferences {
			resolver.IgnorePolymorphicCircularReferences()
		}

		index.BuildIndex()

		if !r.indexConfig.AvoidCircularReferenceCheck {
			resolvingErrors := resolver.CheckForCircularReferences()
			r.circChecked = true
			for e := range resolvingErrors {
				caughtErrors = append(caughtErrors, resolvingErrors[e])
			}
		}
		r.rootIndex = index
		if len(index.refErrors) > 0 {
			caughtErrors = append(caughtErrors, index.refErrors...)
		}
	}
	r.indexingDuration = time.Since(started)
	r.indexed = true
	r.caughtErrors = caughtErrors
	r.built = true
	return errors.Join(caughtErrors...)

}

func (r *Rolodex) CheckForCircularReferences() {
	if !r.circChecked {
		if r.rootIndex != nil && r.rootIndex.resolver != nil {
			resolvingErrors := r.rootIndex.resolver.CheckForCircularReferences()
			for e := range resolvingErrors {
				r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
			}
			if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
				r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
			}
			if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
				r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
			}
		}
		r.circChecked = true
	}
}

func (r *Rolodex) Resolve() {
	if r.rootIndex != nil && r.rootIndex.resolver != nil {
		resolvingErrors := r.rootIndex.resolver.Resolve()
		for e := range resolvingErrors {
			r.caughtErrors = append(r.caughtErrors, resolvingErrors[e])
		}
		if len(r.rootIndex.resolver.ignoredPolyReferences) > 0 {
			r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredPolyReferences...)
		}
		if len(r.rootIndex.resolver.ignoredArrayReferences) > 0 {
			r.ignoredCircularReferences = append(r.ignoredCircularReferences, r.rootIndex.resolver.ignoredArrayReferences...)
		}
	}
	r.resolved = true
}

func (r *Rolodex) BuildIndexes() {
	if r.built {
		return
	}
	for _, idx := range r.indexes {
		idx.BuildIndex()
	}
	if r.rootIndex != nil {
		r.rootIndex.BuildIndex()
	}
	r.built = true
}

func (r *Rolodex) Open(location string) (RolodexFile, error) {

	var errorStack []error

	var localFile *LocalFile
	var remoteFile *RemoteFile

	if r == nil || r.localFS == nil && r.remoteFS == nil {
		return nil, fmt.Errorf("rolodex has no file systems configured, cannot open '%s'", location)
	}

	fileLookup := location
	isUrl := false
	u, _ := url.Parse(location)
	if u != nil && u.Scheme != "" {
		isUrl = true
	}

	if !isUrl {

		for k, v := range r.localFS {

			// check if this is a URL or an abs/rel reference.

			if !filepath.IsAbs(location) {
				fileLookup, _ = filepath.Abs(filepath.Join(k, location))
			}

			f, err := v.Open(fileLookup)
			if err != nil {

				// try a lookup that is not absolute, but relative
				f, err = v.Open(location)
				if err != nil {
					errorStack = append(errorStack, err)
					continue
				}
			}
			// check if this is a native rolodex FS, then the work is done.
			if lrf, ok := interface{}(f).(*localRolodexFile); ok {

				if lf, ko := interface{}(lrf.f).(*LocalFile); ko {
					localFile = lf
					break
				}
			} else {
				// not a native FS, so we need to read the file and create a local file.
				bytes, rErr := io.ReadAll(f)
				if rErr != nil {
					errorStack = append(errorStack, rErr)
					continue
				}
				s, sErr := f.Stat()
				if sErr != nil {
					errorStack = append(errorStack, sErr)
					continue
				}
				if len(bytes) > 0 {
					localFile = &LocalFile{
						filename:     filepath.Base(fileLookup),
						name:         filepath.Base(fileLookup),
						extension:    ExtractFileType(fileLookup),
						data:         bytes,
						fullPath:     fileLookup,
						lastModified: s.ModTime(),
						index:        r.rootIndex,
					}
					break
				}
			}
		}

		if localFile == nil {

			// if there was no file found locally, then search the remote FS.
			for _, v := range r.remoteFS {
				f, err := v.Open(location)
				if err != nil {
					errorStack = append(errorStack, err)
					continue
				}
				if f != nil {
					return f.(*RemoteFile), nil
				}
			}
		}

	} else {

		if !r.indexConfig.AllowRemoteLookup {
			return nil, fmt.Errorf("remote lookup for '%s' not allowed, please set the index configuration to "+
				"AllowRemoteLookup to true", fileLookup)
		}

		for _, v := range r.remoteFS {
			f, err := v.Open(fileLookup)
			if err == nil {

				if rf, ok := interface{}(f).(*RemoteFile); ok {
					remoteFile = rf
					break
				} else {

					bytes, rErr := io.ReadAll(f)
					if rErr != nil {
						errorStack = append(errorStack, rErr)
						continue
					}
					s, sErr := f.Stat()
					if sErr != nil {
						errorStack = append(errorStack, sErr)
						continue
					}
					if len(bytes) > 0 {
						remoteFile = &RemoteFile{
							filename:     filepath.Base(fileLookup),
							name:         filepath.Base(fileLookup),
							extension:    ExtractFileType(fileLookup),
							data:         bytes,
							fullPath:     fileLookup,
							lastModified: s.ModTime(),
							index:        r.rootIndex,
						}
						break
					}
				}
			}
		}
	}

	if localFile != nil {
		return &rolodexFile{
			rolodex:   r,
			location:  localFile.fullPath,
			localFile: localFile,
		}, errors.Join(errorStack...)
	}

	if remoteFile != nil {
		return &rolodexFile{
			rolodex:    r,
			location:   remoteFile.fullPath,
			remoteFile: remoteFile,
		}, errors.Join(errorStack...)
	}

	return nil, errors.Join(errorStack...)
}