old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: tighten item price and pacing behavior

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-23 10:59:33 -04:00
parent 9c8643086a
commit 10c2856bf6
4 changed files with 135 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
packages/core/src/scrapers/facebook.ts
View File

@@ -984,13 +984,13 @@ export function parseFacebookItem(
const url = `https://www.facebook.com/marketplace/item/${item.id}`;
// Extract price information
let cents = 0;
let cents: number | undefined;
let currency = "CAD"; // Default
let amountFormatted = item.formatted_price?.text || "FREE";
let amountFormatted = item.formatted_price?.text;
if (item.listing_price) {
currency = item.listing_price.currency || "CAD";
if (item.listing_price.amount && item.listing_price.amount !== "0.00") {
if (item.listing_price.amount != null) {
const amount = Number.parseFloat(item.listing_price.amount);
if (!Number.isNaN(amount)) {
cents = Math.round(amount * 100);
@@ -1037,6 +1037,13 @@ export function parseFacebookItem(
listingType = "vehicle";
}
if (cents == null || !amountFormatted) {
if (!listingStatus || listingStatus === "ACTIVE") return null;
cents = 0;
amountFormatted = item.formatted_price?.text || "PRICE_UNAVAILABLE";
}
const listingDetails: FacebookListingDetails = {
url,
title,

16
packages/core/src/scrapers/kijiji.ts
View File

@@ -889,15 +889,19 @@ export default async function fetchKijijiItems(
progressBar?.start(totalProgress, currentProgress);
// Process in batches for controlled concurrency
const CONCURRENT_REQUESTS = 1;
const CONCURRENT_REQUESTS = Math.max(1, Math.floor(requestsPerSecond));
const results: (DetailedListing | null)[] = [];
for (let i = 0; i < newListingLinks.length; i += CONCURRENT_REQUESTS) {
const batch = newListingLinks.slice(i, i + CONCURRENT_REQUESTS);
const batchPromises = batch.map(async (link) => {
const batchPromises = batch.map(async (link, batchIndex) => {
try {
const html = await fetchHtml(link, DELAY_MS, {
// Per-request delay keeps detail fetches within REQUESTS_PER_SECOND.
if (batchIndex > 0) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS * batchIndex));
}
const html = await fetchHtml(link, 0, {
// Staggered starts keep request pacing within REQUESTS_PER_SECOND.
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
@@ -936,6 +940,10 @@ export default async function fetchKijijiItems(
const batchResults = await Promise.all(batchPromises);
results.push(...batchResults);
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
}
allListings.push(