Update CheckURLs function to treat HTTP 429 status as valid links alongside 401, 403, and 408. This enhancement improves the URL validation logic by accommodating scenarios where links may be valid but indicate rate limiting.

This commit is contained in:
Luke Hagar
2025-09-16 21:00:05 +00:00
parent 2c3a5b5c04
commit a3c7a2adf2

View File

@@ -63,8 +63,8 @@ func CheckURLs(ctx context.Context, urls []string, sources map[string][]string,
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
// Treat 401/403/408 as valid links
if status == http.StatusUnauthorized || status == http.StatusForbidden || status == http.StatusRequestTimeout {
// Treat 401/403/408/429 as valid links
if status == http.StatusUnauthorized || status == http.StatusForbidden || status == http.StatusRequestTimeout || status == http.StatusTooManyRequests {
ok = true
err = nil
}