old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: refine scraper output behavior

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-23 10:43:38 -04:00
parent 244a88e63c
commit 9c8643086a
4 changed files with 141 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/core/src/scrapers/ebay.ts
View File

@@ -87,6 +87,12 @@ function parseEbayPrice(
cleaned.toUpperCase().includes("US $") cleaned.toUpperCase().includes("US $")
) { ) {
currency = "USD"; currency = "USD";
} else if (cleaned.includes("£")) {
currency = "GBP";
} else if (cleaned.includes("€")) {
currency = "EUR";
} else if (cleaned.includes("¥")) {
currency = "JPY";
} }
return { cents, currency }; return { cents, currency };

14
packages/core/src/scrapers/facebook.ts
View File

@@ -1180,13 +1180,13 @@ export default async function fetchFacebookItems(
console.log(`\nFound ${ads.length} raw ads. Processing...`); console.log(`\nFound ${ads.length} raw ads. Processing...`);
const progressBar = new cliProgress.SingleBar( const isTTY = process.stdout?.isTTY ?? false;
{}, const progressBar = isTTY
cliProgress.Presets.shades_classic, ? new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
); : null;
const totalProgress = ads.length; const totalProgress = ads.length;
const currentProgress = 0; const currentProgress = 0;
progressBar.start(totalProgress, currentProgress); progressBar?.start(totalProgress, currentProgress);
const items = parseFacebookAds(ads); const items = parseFacebookAds(ads);
@@ -1196,8 +1196,8 @@ export default async function fetchFacebookItems(
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0, typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
); );
progressBar.update(totalProgress); progressBar?.update(totalProgress);
progressBar.stop(); progressBar?.stop();
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`); console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
return finalizeResults(pricedItems); return finalizeResults(pricedItems);

62
packages/core/test/ebay-core.test.ts
View File

@@ -234,6 +234,68 @@ describe("eBay Scraper Cookie Handling", () => {
]); ]);
}); });
test("maps pound prices to GBP", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(`
<html><body>
<li class="s-item">
<a href="/itm/123"></a>
<h3>Stable Laptop Bundle</h3>
<span class="s-item__price">£123.45</span>
</li>
</body></html>
`),
}),
) as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "GBP", cents: 12345 }),
}),
]);
});
test("maps euro and yen prices to the matching currency labels", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(`
<html><body>
<li class="s-item">
<a href="/itm/123"></a>
<h3>Euro Bundle</h3>
<span class="s-item__price">€123.45</span>
</li>
<li class="s-item">
<a href="/itm/456"></a>
<h3>Yen Bundle</h3>
<span class="s-item__price">¥123</span>
</li>
</body></html>
`),
}),
) as typeof fetch;
const results = await fetchEbayItems("bundle", 1000, {
keywords: ["bundle"],
});
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "EUR", cents: 12345 }),
}),
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "JPY", cents: 12300 }),
}),
]);
});
test("prefers the discounted Canadian-formatted price", async () => { test("prefers the discounted Canadian-formatted price", async () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({

66
packages/core/test/facebook-core.test.ts
View File

@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import cliProgress from "cli-progress";
import { import {
classifyFacebookResponse, classifyFacebookResponse,
type FacebookListingDetails, type FacebookListingDetails,
@@ -16,6 +17,8 @@ import type { UnstableListingBuckets } from "../src/types/common";
import { formatCookiesForHeader } from "../src/utils/cookies"; import { formatCookiesForHeader } from "../src/utils/cookies";
import { formatCentsToCurrency } from "../src/utils/format"; import { formatCentsToCurrency } from "../src/utils/format";
const originalStdoutIsTTY = process.stdout.isTTY;
type Assert<T extends true> = T; type Assert<T extends true> = T;
type IsExact<T, U> = type IsExact<T, U> =
(<G>() => G extends T ? 1 : 2) extends <G>() => G extends U ? 1 : 2 (<G>() => G extends T ? 1 : 2) extends <G>() => G extends U ? 1 : 2
@@ -50,6 +53,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
afterEach(() => { afterEach(() => {
global.fetch = originalFetch; global.fetch = originalFetch;
process.stdout.isTTY = originalStdoutIsTTY;
}); });
describe("Cookie Parsing", () => { describe("Cookie Parsing", () => {
@@ -621,6 +625,68 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
]); ]);
}); });
test("does not start a progress bar when stdout is not a TTY", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
payload: {
resultGroups: [
{
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Chair Listing",
listing_price: {
amount: "120.00",
formatted_amount: "CA$120",
currency: "CAD",
},
is_live: true,
},
},
},
],
},
],
},
})}</script></body></html>`;
process.stdout.isTTY = false;
const startSpy = mock(() => {});
const updateSpy = mock(() => {});
const stopSpy = mock(() => {});
const originalStart = cliProgress.SingleBar.prototype.start;
const originalUpdate = cliProgress.SingleBar.prototype.update;
const originalStop = cliProgress.SingleBar.prototype.stop;
try {
cliProgress.SingleBar.prototype.start = startSpy;
cliProgress.SingleBar.prototype.update = updateSpy;
cliProgress.SingleBar.prototype.stop = stopSpy;
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () => Promise.resolve(mockSearchHtml),
url: "https://www.facebook.com/marketplace/toronto/search?query=chair",
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
expect(results).toHaveLength(1);
expect(startSpy).not.toHaveBeenCalled();
expect(updateSpy).not.toHaveBeenCalled();
expect(stopSpy).not.toHaveBeenCalled();
} finally {
cliProgress.SingleBar.prototype.start = originalStart;
cliProgress.SingleBar.prototype.update = originalUpdate;
cliProgress.SingleBar.prototype.stop = originalStop;
}
});
test("returns results and unstableResults when unstable mode is enabled", async () => { test("returns results and unstableResults when unstable mode is enabled", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({ const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
payload: { payload: {