fix: respect filtered result sets in unstable mode
This commit is contained in:
@@ -291,6 +291,7 @@ async function fetchHtml(
|
||||
): Promise<{ html: HTMLString; responseUrl: string }> {
|
||||
const maxRetries = opts?.maxRetries ?? 3;
|
||||
const retryBaseMs = opts?.retryBaseMs ?? 500;
|
||||
let lastRateLimitError: HttpError | null = null;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
@@ -326,12 +327,20 @@ async function fetchHtml(
|
||||
if (!res.ok) {
|
||||
// Respect 429 reset if provided
|
||||
if (res.status === 429) {
|
||||
lastRateLimitError = new HttpError(
|
||||
`Request failed with status ${res.status}`,
|
||||
res.status,
|
||||
url,
|
||||
);
|
||||
const resetSeconds = rateLimitReset
|
||||
? Number(rateLimitReset)
|
||||
: Number.NaN;
|
||||
const waitMs = Number.isFinite(resetSeconds)
|
||||
? Math.max(0, resetSeconds * 1000)
|
||||
: (attempt + 1) * retryBaseMs;
|
||||
if (attempt >= maxRetries) {
|
||||
throw lastRateLimitError;
|
||||
}
|
||||
await delay(waitMs);
|
||||
continue;
|
||||
}
|
||||
@@ -369,7 +378,7 @@ async function fetchHtml(
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Exhausted retries without response");
|
||||
throw lastRateLimitError ?? new Error("Exhausted retries without response");
|
||||
}
|
||||
|
||||
// ----------------------------- Parsing -----------------------------
|
||||
|
||||
@@ -959,10 +959,10 @@ export default async function fetchKijijiItems(
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\nParsed ${unstableMode.hideUnstableResults ? allListings.length : filteredListings.length} detailed listings.`,
|
||||
`\nParsed ${filteredListings.length} detailed listings.`,
|
||||
);
|
||||
return unstableMode.hideUnstableResults
|
||||
? finalizeResults(allListings)
|
||||
? finalizeResults(filteredListings)
|
||||
: finalizeResults(filteredListings);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user