LukeParke/slinky
1
0
Fork 0
mirror of https://github.com/LukeHagar/slinky.git synced 2025-12-06 04:21:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor URL collection functions to utilize pre-loaded ignore configurations

Browse Source 
Enhance the CollectURLs and CollectURLsProgress functions in fsurls.go to accept pre-loaded ignore configurations, reducing redundant loading of .gitignore and .slinkignore files. Update the init function in check.go to load these configurations once for all targets, improving efficiency in URL collection processes.
This commit is contained in:
Luke Hagar
2025-09-19 18:26:40 +00:00
parent c42386c8e8
commit b31745a4da
2 changed files with 51 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/check.go
View File

@@ -83,6 +83,10 @@ func init() {
fmt.Printf("::debug:: Glob patterns: %s\n", strings.Join(globPatterns, ","))
}
// Load ignore configurations once for all targets
gitIgnore := fsurls.LoadGitIgnore(".")
slPathIgnore, slURLPatterns := fsurls.LoadSlinkyIgnore(".")
// Aggregate URL->files across all targets
agg := make(map[string]map[string]struct{})
merge := func(res map[string][]string, prefix string, isDir bool) {
@@ -109,7 +113,7 @@ func init() {
// 1) Collect for globs under current dir
if len(globPatterns) > 0 {
res, err := fsurls.CollectURLs(".", globPatterns, respectGitignore)
res, err := fsurls.CollectURLsWithIgnoreConfig(".", globPatterns, respectGitignore, gitIgnore, slPathIgnore, slURLPatterns)
if err != nil {
return err
}
@@ -120,13 +124,13 @@ func init() {
for _, r := range roots {
clean := toSlash(filepath.Clean(r.path))
if r.isDir {
res, err := fsurls.CollectURLs(r.path, []string{"**/*"}, respectGitignore)
res, err := fsurls.CollectURLsWithIgnoreConfig(r.path, []string{"**/*"}, respectGitignore, gitIgnore, slPathIgnore, slURLPatterns)
if err != nil {
return err
}
merge(res, clean, true)
} else {
res, err := fsurls.CollectURLs(r.path, nil, respectGitignore)
res, err := fsurls.CollectURLsWithIgnoreConfig(r.path, nil, respectGitignore, gitIgnore, slPathIgnore, slURLPatterns)
if err != nil {
return err
}

51
internal/fsurls/fsurls.go
View File

@@ -45,6 +45,18 @@ func isDebugEnv() bool {
// If globs is empty, all files are considered. Respects .gitignore if present and respectGitignore=true.
// Returns a map from URL -> sorted unique list of file paths that contained it.
func CollectURLs(rootPath string, globs []string, respectGitignore bool) (map[string][]string, error) {
return CollectURLsWithIgnore(rootPath, globs, respectGitignore, nil, nil)
}
// CollectURLsWithIgnore is like CollectURLs but accepts pre-loaded ignore configuration
// to avoid reloading .slinkignore and .gitignore multiple times.
func CollectURLsWithIgnore(rootPath string, globs []string, respectGitignore bool, slPathIgnore *ignore.GitIgnore, slURLPatterns []string) (map[string][]string, error) {
return CollectURLsWithIgnoreConfig(rootPath, globs, respectGitignore, nil, slPathIgnore, slURLPatterns)
}
// CollectURLsWithIgnoreConfig accepts all pre-loaded ignore configuration
// to avoid reloading .gitignore and .slinkignore multiple times.
func CollectURLsWithIgnoreConfig(rootPath string, globs []string, respectGitignore bool, gitIgnore *ignore.GitIgnore, slPathIgnore *ignore.GitIgnore, slURLPatterns []string) (map[string][]string, error) {
if strings.TrimSpace(rootPath) == "" {
rootPath = "."
}
@@ -55,10 +67,16 @@ func CollectURLs(rootPath string, globs []string, respectGitignore bool) (map[st
var ign *ignore.GitIgnore
if !isFileRoot && respectGitignore {
ign = loadGitIgnore(cleanRoot)
if gitIgnore != nil {
ign = gitIgnore
} else {
ign = LoadGitIgnore(cleanRoot)
}
}
// Load optional .slinkignore config if not provided
if slPathIgnore == nil {
slPathIgnore, slURLPatterns = LoadSlinkyIgnore(cleanRoot)
}
// Load optional .slinkignore config
slPathIgnore, slURLPatterns := loadSlinkyIgnore(cleanRoot)
var patterns []string
for _, g := range globs {
@@ -201,6 +219,18 @@ func CollectURLs(rootPath string, globs []string, respectGitignore bool) (map[st
// CollectURLsProgress is like CollectURLs but invokes onFile(relPath) for each included file.
func CollectURLsProgress(rootPath string, globs []string, respectGitignore bool, onFile func(string)) (map[string][]string, error) {
return CollectURLsProgressWithIgnore(rootPath, globs, respectGitignore, onFile, nil, nil)
}
// CollectURLsProgressWithIgnore is like CollectURLsProgress but accepts pre-loaded ignore configuration
// to avoid reloading .slinkignore and .gitignore multiple times.
func CollectURLsProgressWithIgnore(rootPath string, globs []string, respectGitignore bool, onFile func(string), slPathIgnore *ignore.GitIgnore, slURLPatterns []string) (map[string][]string, error) {
return CollectURLsProgressWithIgnoreConfig(rootPath, globs, respectGitignore, onFile, nil, slPathIgnore, slURLPatterns)
}
// CollectURLsProgressWithIgnoreConfig accepts all pre-loaded ignore configuration
// to avoid reloading .gitignore and .slinkignore multiple times.
func CollectURLsProgressWithIgnoreConfig(rootPath string, globs []string, respectGitignore bool, onFile func(string), gitIgnore *ignore.GitIgnore, slPathIgnore *ignore.GitIgnore, slURLPatterns []string) (map[string][]string, error) {
if strings.TrimSpace(rootPath) == "" {
rootPath = "."
}
@@ -211,9 +241,16 @@ func CollectURLsProgress(rootPath string, globs []string, respectGitignore bool,
var ign *ignore.GitIgnore
if !isFileRoot && respectGitignore {
ign = loadGitIgnore(cleanRoot)
if gitIgnore != nil {
ign = gitIgnore
} else {
ign = LoadGitIgnore(cleanRoot)
}
}
// Load optional .slinkignore config if not provided
if slPathIgnore == nil {
slPathIgnore, slURLPatterns = LoadSlinkyIgnore(cleanRoot)
}
slPathIgnore, slURLPatterns := loadSlinkyIgnore(cleanRoot)
var patterns []string
for _, g := range globs {
@@ -582,7 +619,7 @@ func extractCandidateMatches(content string) []matchCandidate {
return out
}
func loadGitIgnore(root string) *ignore.GitIgnore {
func LoadGitIgnore(root string) *ignore.GitIgnore {
var lines []string
gi := filepath.Join(root, ".gitignore")
if isDebugEnv() {
@@ -638,7 +675,7 @@ type slinkyIgnore struct {
IgnoreURLs []string `json:"ignoreURLs" optional:"true"`
}
func loadSlinkyIgnore(root string) (*ignore.GitIgnore, []string) {
func LoadSlinkyIgnore(root string) (*ignore.GitIgnore, []string) {
cfgPath := findSlinkyConfig(root)
if cfgPath == "" {
return nil, nil