fix: preserve free results and request pacing
This commit is contained in:
@@ -1205,7 +1205,8 @@ export default async function fetchFacebookItems(
|
||||
|
||||
// Filter to only priced items (already done in parseFacebookAds)
|
||||
const pricedItems = items.filter(
|
||||
(item) => item.listingPrice?.cents && item.listingPrice.cents > 0,
|
||||
(item) =>
|
||||
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
|
||||
);
|
||||
|
||||
progressBar.update(totalProgress);
|
||||
|
||||
@@ -889,15 +889,15 @@ export default async function fetchKijijiItems(
|
||||
progressBar?.start(totalProgress, currentProgress);
|
||||
|
||||
// Process in batches for controlled concurrency
|
||||
const CONCURRENT_REQUESTS = Math.max(1, Math.floor(requestsPerSecond * 2)); // 2x rate for faster processing
|
||||
const CONCURRENT_REQUESTS = 1;
|
||||
const results: (DetailedListing | null)[] = [];
|
||||
|
||||
for (let i = 0; i < newListingLinks.length; i += CONCURRENT_REQUESTS) {
|
||||
const batch = newListingLinks.slice(i, i + CONCURRENT_REQUESTS);
|
||||
const batchPromises = batch.map(async (link) => {
|
||||
try {
|
||||
const html = await fetchHtml(link, 0, {
|
||||
// No per-request delay, batch handles rate limit
|
||||
const html = await fetchHtml(link, DELAY_MS, {
|
||||
// Per-request delay keeps detail fetches within REQUESTS_PER_SECOND.
|
||||
onRateInfo: (remaining, reset) => {
|
||||
if (remaining && reset) {
|
||||
console.log(
|
||||
@@ -936,12 +936,6 @@ export default async function fetchKijijiItems(
|
||||
const batchResults = await Promise.all(batchPromises);
|
||||
results.push(...batchResults);
|
||||
|
||||
// Wait between batches to respect rate limit
|
||||
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, DELAY_MS * batch.length),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
allListings.push(
|
||||
|
||||
Reference in New Issue
Block a user