old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: respect scraper pacing details

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-27 00:13:42 -04:00
parent 0f77155c8d
commit b73faa35da
4 changed files with 95 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/core/src/scrapers/ebay.ts
View File

@@ -84,7 +84,8 @@ function parseEbayPrice(
currency = "CAD"; currency = "CAD";
} else if ( } else if (
cleaned.toUpperCase().includes("USD") || cleaned.toUpperCase().includes("USD") ||
cleaned.toUpperCase().includes("US $") cleaned.toUpperCase().includes("US $") ||
cleaned.toUpperCase().includes("US$")
) { ) {
currency = "USD"; currency = "USD";
} else if (cleaned.includes("£")) { } else if (cleaned.includes("£")) {

22
packages/core/src/scrapers/kijiji.ts
View File

@@ -462,18 +462,16 @@ async function fetchSellerDetails(
accountType?: string; accountType?: string;
}> { }> {
try { try {
const [reviewData, profileData] = await Promise.all([ const reviewData = await fetchGraphQLData(
fetchGraphQLData( GRAPHQL_QUERIES.getReviewSummary,
GRAPHQL_QUERIES.getReviewSummary, { userId: posterId },
{ userId: posterId }, BASE_URL,
BASE_URL, );
), const profileData = await fetchGraphQLData(
fetchGraphQLData( GRAPHQL_QUERIES.getProfileMetrics,
GRAPHQL_QUERIES.getProfileMetrics, { profileId: posterId },
{ profileId: posterId }, BASE_URL,
BASE_URL, );
),
]);
const reviewResponse = reviewData as GraphQLReviewResponse; const reviewResponse = reviewData as GraphQLReviewResponse;
const profileResponse = profileData as GraphQLProfileResponse; const profileResponse = profileData as GraphQLProfileResponse;

26
packages/core/test/ebay-core.test.ts
View File

@@ -234,6 +234,32 @@ describe("eBay Scraper Cookie Handling", () => {
]); ]);
}); });
test("treats US dollar prices without space as USD", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(`
<html><body>
<li class="s-item">
<a href="/itm/123"></a>
<h3>Stable Laptop Bundle</h3>
<span class="s-item__price">US$123.45</span>
</li>
</body></html>
`),
}),
) as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
}),
]);
});
test("maps pound prices to GBP", async () => { test("maps pound prices to GBP", async () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({

57
packages/core/test/kijiji-core.test.ts
View File

@@ -772,6 +772,63 @@ describe("fetchKijijiItems", () => {
); );
}); });
test("fetchSellerDetails does not fire concurrent GraphQL requests", async () => {
const html = `
<html>
<script id="__NEXT_DATA__" type="application/json">
${JSON.stringify({
props: {
pageProps: {
__APOLLO_STATE__: {
"Listing:detail": {
url: "/v-test/k0l0",
title: "Test Listing",
price: { amount: 10000, currency: "CAD", type: "FIXED" },
type: "OFFER",
status: "ACTIVE",
posterInfo: { posterId: "123" },
},
},
},
},
})}
</script>
</html>
`;
let activeAnvilRequests = 0;
let maxActiveAnvilRequests = 0;
global.fetch = mock(async (input: string | URL | Request) => {
const url = typeof input === "string" ? input : input.toString();
if (url.includes("/anvil/api")) {
activeAnvilRequests++;
maxActiveAnvilRequests = Math.max(
maxActiveAnvilRequests,
activeAnvilRequests,
);
await new Promise((resolve) => setTimeout(resolve, 50));
activeAnvilRequests--;
return {
ok: true,
json: () => Promise.resolve({ data: { user: {} } }),
headers: { get: () => null },
url,
};
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
await parseDetailedListing(html, "https://www.kijiji.ca", {
includeClientSideData: true,
sellerDataDepth: "detailed",
});
expect(maxActiveAnvilRequests).toBe(1);
});
test("returns results and unstableResults when unstable mode is enabled", async () => { test("returns results and unstableResults when unstable mode is enabled", async () => {
const searchHtml = ` const searchHtml = `
<html> <html>