fix: preserve free results and request pacing

This commit is contained in:
2026-04-23 05:40:42 -04:00
parent 13c0fec305
commit eb37e8814e
4 changed files with 162 additions and 10 deletions

View File

@@ -1205,7 +1205,8 @@ export default async function fetchFacebookItems(
// Filter to only priced items (already done in parseFacebookAds)
const pricedItems = items.filter(
(item) => item.listingPrice?.cents && item.listingPrice.cents > 0,
(item) =>
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
);
progressBar.update(totalProgress);

View File

@@ -889,15 +889,15 @@ export default async function fetchKijijiItems(
progressBar?.start(totalProgress, currentProgress);
// Process in batches for controlled concurrency
const CONCURRENT_REQUESTS = Math.max(1, Math.floor(requestsPerSecond * 2)); // 2x rate for faster processing
const CONCURRENT_REQUESTS = 1;
const results: (DetailedListing | null)[] = [];
for (let i = 0; i < newListingLinks.length; i += CONCURRENT_REQUESTS) {
const batch = newListingLinks.slice(i, i + CONCURRENT_REQUESTS);
const batchPromises = batch.map(async (link) => {
try {
const html = await fetchHtml(link, 0, {
// No per-request delay, batch handles rate limit
const html = await fetchHtml(link, DELAY_MS, {
// Per-request delay keeps detail fetches within REQUESTS_PER_SECOND.
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
@@ -936,12 +936,6 @@ export default async function fetchKijijiItems(
const batchResults = await Promise.all(batchPromises);
results.push(...batchResults);
// Wait between batches to respect rate limit
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
await new Promise((resolve) =>
setTimeout(resolve, DELAY_MS * batch.length),
);
}
}
allListings.push(