old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: biome lint and formatting

Browse Source 
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
This commit is contained in:
Dmytro Stanchiev
2026-04-28 19:21:16 -04:00
parent 49e90d45f8
commit 957e0f137b
10 changed files with 465 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/api-server/src/routes/kijiji.ts
View File

@@ -48,16 +48,16 @@ export async function kijijiRoute(req: Request): Promise<Response> {
location: reqUrl.searchParams.get("location") || undefined,
category: reqUrl.searchParams.get("category") || undefined,
keywords: reqUrl.searchParams.get("keywords") || undefined,
sortBy: (reqUrl.searchParams.get("sortBy") as
sortBy:
(reqUrl.searchParams.get("sortBy") as
| "relevancy"
| "date"
| "price"
| "distance"
| undefined) || undefined,
sortOrder: (reqUrl.searchParams.get("sortOrder") as
| "desc"
| "asc"
| undefined) || undefined,
sortOrder:
(reqUrl.searchParams.get("sortOrder") as "desc" | "asc" | undefined) ||
undefined,
maxPages,
priceMin,
priceMax,

113
packages/api-server/test/routes.test.ts
View File

@@ -1,4 +1,4 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import { beforeEach, describe, expect, mock, test } from "bun:test";
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
@@ -123,7 +123,10 @@ describe("API routes", () => {
),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
{
minPrice: undefined,
maxPrice: undefined,
strictMode: false,
@@ -131,9 +134,11 @@ describe("API routes", () => {
keywords: ["laptop"],
buyItNowOnly: true,
canadaOnly: true,
}, {
},
{
hideUnstableResults: true,
});
},
);
});
test("kijijiRoute forwards unstableFilter=true to core", async () => {
@@ -202,9 +207,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&buyItNowOnly=true",
),
new Request("http://localhost/api/ebay?q=laptop&buyItNowOnly=true"),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
@@ -242,9 +245,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=5",
),
new Request("http://localhost/api/kijiji?q=laptop&maxPages=5"),
);
expect(fetchKijijiItems).toHaveBeenCalledWith(
@@ -385,17 +386,17 @@ describe("API routes", () => {
test("ebayRoute forwards maxItems to core in default mode", async () => {
const { ebayRoute } = await import("../src/routes/ebay");
fetchEbayItems.mockImplementation(() =>
Promise.resolve([{ title: "a" }]),
);
fetchEbayItems.mockImplementation(() => Promise.resolve([{ title: "a" }]));
await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxItems=2",
),
new Request("http://localhost/api/ebay?q=laptop&maxItems=2"),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }));
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
expect.objectContaining({ maxItems: 2 }),
);
});
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
@@ -419,9 +420,14 @@ describe("API routes", () => {
expect(body.unstableResults).toHaveLength(2);
expect(body.results[0].title).toBe("a");
expect(body.unstableResults[0].title).toBe("d");
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 4 }), {
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
expect.objectContaining({ maxItems: 4 }),
{
hideUnstableResults: true,
});
},
);
});
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
@@ -440,9 +446,14 @@ describe("API routes", () => {
),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }), {
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
expect.objectContaining({ maxItems: 2 }),
{
hideUnstableResults: true,
});
},
);
});
test("ebayRoute returns 404 when unstable results are empty", async () => {
@@ -456,9 +467,7 @@ describe("API routes", () => {
);
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&unstableFilter=true",
),
new Request("http://localhost/api/ebay?q=laptop&unstableFilter=true"),
);
expect(response.status).toBe(404);
@@ -470,9 +479,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxItems=abc",
),
new Request("http://localhost/api/ebay?q=laptop&maxItems=abc"),
);
expect(response.status).toBe(400);
@@ -484,9 +491,7 @@ describe("API routes", () => {
const { facebookRoute } = await import("../src/routes/facebook");
const response = await facebookRoute(
new Request(
"http://localhost/api/facebook?q=laptop&maxItems=abc",
),
new Request("http://localhost/api/facebook?q=laptop&maxItems=abc"),
);
expect(response.status).toBe(400);
@@ -498,9 +503,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&minPrice=abc",
),
new Request("http://localhost/api/ebay?q=laptop&minPrice=abc"),
);
expect(response.status).toBe(400);
@@ -512,9 +515,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxPrice=abc",
),
new Request("http://localhost/api/ebay?q=laptop&maxPrice=abc"),
);
expect(response.status).toBe(400);
@@ -526,9 +527,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=abc",
),
new Request("http://localhost/api/kijiji?q=laptop&maxPages=abc"),
);
expect(response.status).toBe(400);
@@ -540,9 +539,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMin=abc",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMin=abc"),
);
expect(response.status).toBe(400);
@@ -554,9 +551,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMax=abc",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMax=abc"),
);
expect(response.status).toBe(400);
@@ -568,9 +563,7 @@ describe("API routes", () => {
const { facebookRoute } = await import("../src/routes/facebook");
const response = await facebookRoute(
new Request(
"http://localhost/api/facebook?q=laptop&maxItems=-1",
),
new Request("http://localhost/api/facebook?q=laptop&maxItems=-1"),
);
expect(response.status).toBe(400);
@@ -582,9 +575,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxItems=-1",
),
new Request("http://localhost/api/ebay?q=laptop&maxItems=-1"),
);
expect(response.status).toBe(400);
@@ -596,9 +587,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&minPrice=-5",
),
new Request("http://localhost/api/ebay?q=laptop&minPrice=-5"),
);
expect(response.status).toBe(400);
@@ -610,9 +599,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxPrice=-10",
),
new Request("http://localhost/api/ebay?q=laptop&maxPrice=-10"),
);
expect(response.status).toBe(400);
@@ -624,9 +611,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=-2",
),
new Request("http://localhost/api/kijiji?q=laptop&maxPages=-2"),
);
expect(response.status).toBe(400);
@@ -638,9 +623,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMin=-5",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMin=-5"),
);
expect(response.status).toBe(400);
@@ -652,9 +635,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMax=-10",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMax=-10"),
);
expect(response.status).toBe(400);

22
packages/core/src/scrapers/ebay.ts
View File

@@ -4,13 +4,13 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type CookieConfig,
ensureCookies,
formatCookiesForHeader,
} from "../utils/cookies";
import { delay } from "../utils/delay";
import { classifyUnstableListings } from "../utils/unstable";
// eBay cookie configuration
const EBAY_COOKIE_CONFIG: CookieConfig = {
@@ -44,7 +44,9 @@ function canonicalizeEbayItemUrl(url: string): string {
try {
const parsed = new URL(url, "https://www.ebay.ca");
const match = parsed.pathname.match(/\/itm\/(?:[^/?#]+\/)?\d+/);
return match ? `${parsed.origin}${match[0]}` : `${parsed.origin}${parsed.pathname}`;
return match
? `${parsed.origin}${match[0]}`
: `${parsed.origin}${parsed.pathname}`;
} catch {
return url;
}
@@ -275,11 +277,7 @@ function parseEbayListings(
const actualPrices: HTMLElement[] = [];
for (const el of allPriceElements) {
const text = el.textContent?.trim();
if (
text &&
EBAY_PRICE_TEXT_RE.test(text) &&
text.length < 50
) {
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
actualPrices.push(el);
}
}
@@ -386,7 +384,8 @@ async function loadEbayCookies(): Promise<string | undefined> {
export default async function fetchEbayItems(
SEARCH_QUERY: string,
REQUESTS_PER_SECOND: number | undefined,
opts: {
opts:
| {
minPrice?: number;
maxPrice?: number;
strictMode?: boolean;
@@ -395,7 +394,8 @@ export default async function fetchEbayItems(
buyItNowOnly?: boolean;
canadaOnly?: boolean;
maxItems?: number;
} | undefined,
}
| undefined,
unstableMode: { hideUnstableResults: true },
): Promise<UnstableListingBuckets<EbayListingDetails>>;
export default async function fetchEbayItems(
@@ -529,7 +529,9 @@ export default async function fetchEbayItems(
// Filter by price range (additional safety check)
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return typeof cents === "number" && cents >= minPrice && cents <= maxPrice;
return (
typeof cents === "number" && cents >= minPrice && cents <= maxPrice
);
});
console.log(`Parsed ${filteredListings.length} eBay listings.`);

62
packages/core/src/scrapers/facebook.ts
View File

@@ -5,7 +5,6 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type Cookie,
type CookieConfig,
@@ -16,6 +15,7 @@ import {
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http";
import { classifyUnstableListings } from "../utils/unstable";
/**
* Facebook Marketplace Scraper
@@ -408,7 +408,11 @@ export function classifyFacebookResponse(
htmlString.includes("This listing is no longer available") ||
htmlString.includes("listing has been removed");
if (unavailable) {
return { kind: "unavailable" as const, authGated: false, unavailable: true };
return {
kind: "unavailable" as const,
authGated: false,
unavailable: true,
};
}
if (responseUrl.includes("/marketplace/item/")) {
@@ -455,7 +459,8 @@ function isFacebookSearchEdgeArray(value: unknown): value is FacebookEdge[] {
Array.isArray(value) &&
value.length > 0 &&
value.every(
(edge) => isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
(edge) =>
isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
)
);
}
@@ -552,8 +557,7 @@ function scoreMarketplaceItemPath(path: string[]): number {
if (
path.some(
(segment) =>
segment.includes("recommend") || segment.includes("related"),
(segment) => segment.includes("recommend") || segment.includes("related"),
)
) {
score -= 10;
@@ -567,7 +571,9 @@ function collectMarketplaceItemCandidates(
path: string[] = [],
): FacebookMarketplaceItemMatch[] {
if (Array.isArray(candidate)) {
return candidate.flatMap((item) => collectMarketplaceItemCandidates(item, path));
return candidate.flatMap((item) =>
collectMarketplaceItemCandidates(item, path),
);
}
if (!isRecord(candidate)) {
@@ -628,7 +634,9 @@ function extractRenderedText(node: ParentNode, selector: string): string[] {
.filter((text): text is string => Boolean(text));
}
function extractMarketplaceItemIdFromElement(element: Element | null): string | null {
function extractMarketplaceItemIdFromElement(
element: Element | null,
): string | null {
const href = element?.getAttribute("href") || "";
return href.match(FACEBOOK_ITEM_HREF_RE)?.[1] ?? null;
}
@@ -666,7 +674,9 @@ function extractFacebookPermalinkItemId(document: Document): string | null {
return extractMarketplaceItemIdFromElement(itemLinks.at(-1) ?? null);
}
function extractFacebookDescriptionText(document: Document): string | undefined {
function extractFacebookDescriptionText(
document: Document,
): string | undefined {
const labels = Array.from(document.querySelectorAll("div, span, h2, h3, p"));
for (const label of labels) {
@@ -759,7 +769,10 @@ function extractFacebookItemHtmlFallback(
const priceText = texts.find((text) => FACEBOOK_PRICE_TEXT_RE.test(text));
const parsedPrice = priceText ? parseFacebookRenderedPrice(priceText) : null;
const location = texts.find(
(text) => text !== title && text !== priceText && FACEBOOK_LOCATION_TEXT_RE.test(text),
(text) =>
text !== title &&
text !== priceText &&
FACEBOOK_LOCATION_TEXT_RE.test(text),
);
const description = extractFacebookDescriptionText(document);
@@ -841,7 +854,8 @@ export function extractFacebookItemData(
if (
!bestMatch ||
match.score > bestMatch.score ||
(match.score === bestMatch.score && match.path.length < bestMatch.path.length)
(match.score === bestMatch.score &&
match.path.length < bestMatch.path.length)
) {
bestMatch = match;
}
@@ -1101,7 +1115,9 @@ export default async function fetchFacebookItems(
const finalizeResults = (
listings: FacebookListingDetails[],
): FacebookListingDetails[] | UnstableListingBuckets<FacebookListingDetails> => {
):
| FacebookListingDetails[]
| UnstableListingBuckets<FacebookListingDetails> => {
if (!unstableMode.hideUnstableResults) {
return listings.slice(0, MAX_ITEMS);
}
@@ -1166,9 +1182,14 @@ export default async function fetchFacebookItems(
throw err;
}
const classification = classifyFacebookResponse(searchHtml, searchResponseUrl);
const classification = classifyFacebookResponse(
searchHtml,
searchResponseUrl,
);
if (classification.authGated) {
console.warn("Facebook marketplace search redirected to login. Cookies may be expired.");
console.warn(
"Facebook marketplace search redirected to login. Cookies may be expired.",
);
return finalizeResults([]);
}
@@ -1204,7 +1225,8 @@ export default async function fetchFacebookItems(
// Filter to only priced items (already done in parseFacebookAds)
const pricedItems = items.filter(
(item) =>
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
typeof item.listingPrice?.cents === "number" &&
item.listingPrice.cents >= 0,
);
progressBar?.update(totalProgress);
@@ -1293,7 +1315,9 @@ export async function fetchFacebookItem(
if (classification.authGated) {
logExtractionMetrics(false, itemId);
console.warn(`Authentication failed for item ${itemId}. Cookies may be expired.`);
console.warn(
`Authentication failed for item ${itemId}. Cookies may be expired.`,
);
return null;
}
@@ -1301,7 +1325,9 @@ export async function fetchFacebookItem(
if (classification.unavailable && !itemData) {
logExtractionMetrics(false, itemId);
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}
@@ -1317,7 +1343,9 @@ export async function fetchFacebookItem(
logExtractionMetrics(false, itemId);
if (itemHtml.includes("This item has been sold")) {
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}

26
packages/core/src/scrapers/kijiji.ts
View File

@@ -6,7 +6,6 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type CookieConfig,
formatCookiesForHeader,
@@ -22,6 +21,7 @@ import {
RateLimitError,
ValidationError,
} from "../utils/http";
import { classifyUnstableListings } from "../utils/unstable";
// Kijiji cookie configuration
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
@@ -203,11 +203,17 @@ const SORT_MAPPINGS: Record<string, string> = {
};
const LOCATION_SLUGS = Object.fromEntries(
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [
id,
slug.replace(/\s+/g, "-"),
]),
) as Record<number, string>;
const CATEGORY_SLUGS = Object.fromEntries(
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [
id,
slug.replace(/\s+/g, "-"),
]),
) as Record<number, string>;
// ----------------------------- Utilities -----------------------------
@@ -816,7 +822,10 @@ export default async function fetchKijijiItems(
: undefined;
// Set defaults for configuration
const finalSearchOptions: Omit<Required<SearchOptions>, "priceMin" | "priceMax"> & {
const finalSearchOptions: Omit<
Required<SearchOptions>,
"priceMin" | "priceMax"
> & {
priceMin?: number;
priceMax?: number;
} = {
@@ -903,7 +912,9 @@ export default async function fetchKijijiItems(
const batchPromises = batch.map(async (link, batchIndex) => {
try {
if (batchIndex > 0) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS * batchIndex));
await new Promise((resolve) =>
setTimeout(resolve, DELAY_MS * batchIndex),
);
}
const html = await fetchHtml(link, 0, {
@@ -949,7 +960,6 @@ export default async function fetchKijijiItems(
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
}
allListings.push(
@@ -968,9 +978,7 @@ export default async function fetchKijijiItems(
matchesPriceFilters(listing, finalSearchOptions),
);
console.log(
`\nParsed ${filteredListings.length} detailed listings.`,
);
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
return finalizeResults(filteredListings);
}

29
packages/core/test/ebay-core.test.ts
View File

@@ -136,7 +136,9 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toHaveLength(1);
expect(results[0]).toEqual(
expect.objectContaining({ url: "https://www.ebay.ca/itm/123?_trkparms=foo" }),
expect.objectContaining({
url: "https://www.ebay.ca/itm/123?_trkparms=foo",
}),
);
});
@@ -229,7 +231,10 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "USD",
cents: 12345,
}),
}),
]);
});
@@ -255,7 +260,10 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "USD",
cents: 12345,
}),
}),
]);
});
@@ -281,7 +289,10 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "GBP", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "GBP",
cents: 12345,
}),
}),
]);
});
@@ -314,10 +325,16 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "EUR", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "EUR",
cents: 12345,
}),
}),
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "JPY", cents: 12300 }),
listingPrice: expect.objectContaining({
currency: "JPY",
cents: 12300,
}),
}),
]);
});

50
packages/core/test/facebook-core.test.ts
View File

@@ -2,13 +2,13 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import cliProgress from "cli-progress";
import {
classifyFacebookResponse,
type FacebookListingDetails,
ensureFacebookCookies,
extractFacebookBootstrapCandidates,
extractFacebookItemData,
extractFacebookMarketplaceData,
default as fetchFacebookItems,
type FacebookListingDetails,
fetchFacebookItem,
default as fetchFacebookItems,
parseFacebookAds,
parseFacebookCookieString,
parseFacebookItem,
@@ -30,9 +30,13 @@ type IsExact<T, U> =
const getDefaultFacebookItems = async () => fetchFacebookItems("chair");
const getUnstableFacebookItems = async (): Promise<
UnstableListingBuckets<FacebookListingDetails>
> => fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
> =>
fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
type _FacebookDefaultReturn = Assert<
IsExact<Awaited<ReturnType<typeof getDefaultFacebookItems>>, FacebookListingDetails[]>
IsExact<
Awaited<ReturnType<typeof getDefaultFacebookItems>>,
FacebookListingDetails[]
>
>;
type _FacebookUnstableReturn = Assert<
IsExact<
@@ -533,7 +537,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("returns an array by default", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -556,7 +561,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -576,7 +582,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("preserves free listings through the public fetch entrypoint", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -599,7 +606,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -626,7 +634,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("does not start a progress bar when stdout is not a TTY", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -649,7 +658,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
process.stdout.isTTY = false;
const startSpy = mock(() => {});
@@ -688,7 +698,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("returns results and unstableResults when unstable mode is enabled", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -739,7 +750,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -768,7 +780,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("unstable mode classifies before the final MAX_ITEMS limit", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -791,7 +804,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
node: {
listing: {
id: "2",
marketplace_listing_title: "Second Boundary Stable Chair",
marketplace_listing_title:
"Second Boundary Stable Chair",
listing_price: {
amount: "110.00",
formatted_amount: "CA$110",
@@ -819,7 +833,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -869,7 +884,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
redacted_description: { text: "Solid wood chair" },
location_text: { text: "Toronto, ON" },
marketplace_listing_seller: { id: "seller-1", name: "Alex" },
marketplace_listing_seller: {
id: "seller-1",
name: "Alex",
},
condition: "USED",
is_live: true,
},

10
packages/core/test/facebook-integration.test.ts
View File

@@ -1,5 +1,7 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import fetchFacebookItems, { fetchFacebookItem } from "../src/scrapers/facebook";
import fetchFacebookItems, {
fetchFacebookItem,
} from "../src/scrapers/facebook";
// Mock fetch globally
const originalFetch = global.fetch;
@@ -27,7 +29,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
describe("Main Search Function", () => {
test("should successfully fetch search results", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -55,7 +58,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({

95
packages/core/test/kijiji-core.test.ts
View File

@@ -1,12 +1,12 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import {
buildSearchUrl,
default as fetchKijijiItems,
type DetailedListing,
default as fetchKijijiItems,
NetworkError,
parseSearch,
parseDetailedListing,
ParseError,
parseDetailedListing,
parseSearch,
RateLimitError,
resolveCategoryId,
resolveLocationId,
@@ -282,7 +282,8 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-low/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
text: () =>
Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
headers: { get: () => null },
url,
});
@@ -291,7 +292,8 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-mid/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
text: () =>
Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
headers: { get: () => null },
url,
});
@@ -300,7 +302,8 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-high/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
text: () =>
Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
headers: { get: () => null },
url,
});
@@ -534,9 +537,18 @@ describe("fetchKijijiItems", () => {
props: {
pageProps: {
__APOLLO_STATE__: {
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
"Listing:3": { url: "/v-unstable/k0l0", title: "Unstable Listing" },
"Listing:1": {
url: "/v-stable-one/k0l0",
title: "Stable Listing One",
},
"Listing:2": {
url: "/v-stable-two/k0l0",
title: "Stable Listing Two",
},
"Listing:3": {
url: "/v-unstable/k0l0",
title: "Unstable Listing",
},
},
},
},
@@ -582,7 +594,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-one/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -591,7 +606,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-two/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -600,7 +618,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-unstable/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Unstable Listing", 7000, "v-unstable/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Unstable Listing", 7000, "v-unstable/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -635,10 +656,22 @@ describe("fetchKijijiItems", () => {
props: {
pageProps: {
__APOLLO_STATE__: {
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
"Listing:3": { url: "/v-out-of-range-high/k0l0", title: "Out Of Range High" },
"Listing:4": { url: "/v-out-of-range-low/k0l0", title: "Out Of Range Low" },
"Listing:1": {
url: "/v-stable-one/k0l0",
title: "Stable Listing One",
},
"Listing:2": {
url: "/v-stable-two/k0l0",
title: "Stable Listing Two",
},
"Listing:3": {
url: "/v-out-of-range-high/k0l0",
title: "Out Of Range High",
},
"Listing:4": {
url: "/v-out-of-range-low/k0l0",
title: "Out Of Range Low",
},
},
},
},
@@ -672,7 +705,11 @@ describe("fetchKijijiItems", () => {
global.fetch = mock((input: string | URL | Request) => {
const url = typeof input === "string" ? input : input.toString();
if (url.includes("/k0c0l1700272") && url.includes("priceMin=80") && url.includes("priceMax=150")) {
if (
url.includes("/k0c0l1700272") &&
url.includes("priceMin=80") &&
url.includes("priceMax=150")
) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(searchHtml),
@@ -684,7 +721,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-one/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -693,7 +733,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-two/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -702,7 +745,14 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-out-of-range-high/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Out Of Range High", 20000, "v-out-of-range-high/k0l0")),
text: () =>
Promise.resolve(
listingHtml(
"Out Of Range High",
20000,
"v-out-of-range-high/k0l0",
),
),
headers: { get: () => null },
url,
});
@@ -711,7 +761,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-out-of-range-low/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0"),
),
headers: { get: () => null },
url,
});

29
packages/core/test/unstable-listing-mode.test.ts
View File

@@ -31,8 +31,13 @@ describe("classifyUnstableListings", () => {
const buckets = classifyUnstableListings(listings);
expect(buckets.results.map((listing) => listing.id)).toEqual(["stable-1", "stable-2"]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
expect(buckets.results.map((listing) => listing.id)).toEqual([
"stable-1",
"stable-2",
]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
"unstable",
]);
});
test("uses the midpoint median for even-sized priced inputs", () => {
@@ -45,8 +50,14 @@ describe("classifyUnstableListings", () => {
const buckets = classifyUnstableListings(listings);
expect(buckets.results.map((listing) => listing.id)).toEqual(["mid-low", "mid-high", "high"]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["low"]);
expect(buckets.results.map((listing) => listing.id)).toEqual([
"mid-low",
"mid-high",
"high",
]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
"low",
]);
});
test("keeps non-positive prices in results and excludes them from the median input", () => {
@@ -66,7 +77,9 @@ describe("classifyUnstableListings", () => {
"stable-1",
"stable-2",
]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
"unstable",
]);
});
test("returns all listings in results when fewer than two valid prices are present", () => {
@@ -78,7 +91,11 @@ describe("classifyUnstableListings", () => {
const buckets = classifyUnstableListings(listings);
expect(buckets.results.map((listing) => listing.id)).toEqual(["zero", "negative", "only-valid"]);
expect(buckets.results.map((listing) => listing.id)).toEqual([
"zero",
"negative",
"only-valid",
]);
expect(buckets.unstableResults).toEqual([]);
});
});