old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: old4ever:df2635d92f27b885bcff0bd959d172b38f812c51
Branches Tags
old4ever:main
..
compare: old4ever:2a5701aeb9c45b01aa3d738a1302ed8883c202ff
Branches Tags
old4ever:main

4 Commits
df2635d92f ... 2a5701aeb9

Author SHA1 Message Date
Dmytro Stanchiev
2a5701aeb9 test: quiet and speed up test runs 2026-04-28 21:45:06 -04:00
Dmytro Stanchiev
c6c44a0914 fix(api): preserve unstable buckets 2026-04-28 21:34:47 -04:00
Dmytro Stanchiev
3fe5fdb63f fix(core): handle partial listing data 2026-04-28 21:34:45 -04:00
Dmytro Stanchiev
7966073bf8 fix(core): prefer explicit cookie source 2026-04-28 21:34:40 -04:00
26 changed files with 465 additions and 279 deletions
Expand all files Collapse all files

3
packages/api-server/src/index.ts
View File

@@ -1,3 +1,4 @@
import { logger } from "./logger";
import { ebayRoute } from "./routes/ebay";
import { facebookRoute } from "./routes/facebook";
import { kijijiRoute } from "./routes/kijiji";
@@ -27,4 +28,4 @@ const server = Bun.serve({
},
});
console.log(`API Server running on ${server.hostname}:${server.port}`);
logger.log(`API Server running on ${server.hostname}:${server.port}`);

10
packages/api-server/src/logger.ts Normal file
View File

@@ -0,0 +1,10 @@
const isTest = () => process.env.NODE_ENV === "test";
export const logger = {
log: (...args: Parameters<typeof console.log>) => {
if (!isTest()) console.log(...args);
},
error: (...args: Parameters<typeof console.error>) => {
if (!isTest()) console.error(...args);
},
};

30
packages/api-server/src/routes/ebay.ts
View File

@@ -1,4 +1,5 @@
import { fetchEbayItems } from "@marketplace-scrapers/core";
import { logger } from "../logger";
/**
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}
@@ -21,7 +22,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
const minPriceParam = reqUrl.searchParams.get("minPrice");
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
if (minPriceParam && (Number.isNaN(minPrice) || minPrice < 0)) {
if (minPriceParam && (Number.isNaN(minPrice) || (minPrice ?? 0) < 0)) {
return Response.json(
{ message: "Invalid minPrice parameter" },
{ status: 400 },
@@ -29,7 +30,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
}
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
if (maxPriceParam && (Number.isNaN(maxPrice) || maxPrice < 0)) {
if (maxPriceParam && (Number.isNaN(maxPrice) || (maxPrice ?? 0) < 0)) {
return Response.json(
{ message: "Invalid maxPrice parameter" },
{ status: 400 },
@@ -49,7 +50,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
const maxItemsParam = reqUrl.searchParams.get("maxItems");
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
if (maxItemsParam && (Number.isNaN(maxItems) || maxItems < 0)) {
if (maxItemsParam && (Number.isNaN(maxItems) || (maxItems ?? 0) < 0)) {
return Response.json(
{ message: "Invalid maxItems parameter" },
{ status: 400 },
@@ -67,25 +68,30 @@ export async function ebayRoute(req: Request): Promise<Response> {
canadaOnly,
maxItems,
};
const items = hideUnstableResults
? await fetchEbayItems(SEARCH_QUERY, 1, opts, {
if (hideUnstableResults) {
const items = await fetchEbayItems(SEARCH_QUERY, 1, opts, {
hideUnstableResults: true,
})
: await fetchEbayItems(SEARCH_QUERY, 1, opts);
});
if (items.results.length === 0 && items.unstableResults.length === 0) {
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
}
return Response.json(items, { status: 200 });
}
const isEmpty = hideUnstableResults
? items.results.length === 0 && items.unstableResults.length === 0
: !items || items.length === 0;
const items = await fetchEbayItems(SEARCH_QUERY, 1, opts);
const isEmpty = !items || items.length === 0;
if (isEmpty)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("eBay scraping error:", error);
logger.error("eBay scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });

31
packages/api-server/src/routes/facebook.ts
View File

@@ -1,4 +1,5 @@
import { fetchFacebookItems } from "@marketplace-scrapers/core";
import { logger } from "../logger";
/**
* GET /api/facebook?q={query}&location={location}
@@ -30,24 +31,34 @@ export async function facebookRoute(req: Request): Promise<Response> {
reqUrl.searchParams.get("unstableFilter") === "true";
try {
const items = hideUnstableResults
? await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems, {
if (hideUnstableResults) {
const items = await fetchFacebookItems(
SEARCH_QUERY,
1,
LOCATION,
maxItems,
{
hideUnstableResults: true,
})
: await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
},
);
if (items.results.length === 0 && items.unstableResults.length === 0) {
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
}
return Response.json(items, { status: 200 });
}
const isEmpty = hideUnstableResults
? items.results.length === 0 && items.unstableResults.length === 0
: !items || items.length === 0;
if (isEmpty)
const items = await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Facebook scraping error:", error);
logger.error("Facebook scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });

31
packages/api-server/src/routes/kijiji.ts
View File

@@ -1,4 +1,5 @@
import { fetchKijijiItems } from "@marketplace-scrapers/core";
import { logger } from "../logger";
/**
* GET /api/kijiji?q={query}
@@ -27,7 +28,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
}
const priceMinParam = reqUrl.searchParams.get("priceMin");
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
if (priceMinParam && (Number.isNaN(priceMin) || priceMin < 0)) {
if (priceMinParam && (Number.isNaN(priceMin) || (priceMin ?? 0) < 0)) {
return Response.json(
{ message: "Invalid priceMin parameter" },
{ status: 400 },
@@ -35,7 +36,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
}
const priceMaxParam = reqUrl.searchParams.get("priceMax");
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
if (priceMaxParam && (Number.isNaN(priceMax) || priceMax < 0)) {
if (priceMaxParam && (Number.isNaN(priceMax) || (priceMax ?? 0) < 0)) {
return Response.json(
{ message: "Invalid priceMax parameter" },
{ status: 400 },
@@ -65,35 +66,39 @@ export async function kijijiRoute(req: Request): Promise<Response> {
};
try {
const items = hideUnstableResults
? await fetchKijijiItems(
if (hideUnstableResults) {
const items = await fetchKijijiItems(
SEARCH_QUERY,
4, // 4 requests per second for faster scraping
"https://www.kijiji.ca",
searchOptions,
{},
{ hideUnstableResults: true },
)
: await fetchKijijiItems(
);
if (items.results.length === 0 && items.unstableResults.length === 0) {
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
}
return Response.json(items, { status: 200 });
}
const items = await fetchKijijiItems(
SEARCH_QUERY,
4, // 4 requests per second for faster scraping
"https://www.kijiji.ca",
searchOptions,
{},
);
const isEmpty = hideUnstableResults
? items.results.length === 0 && items.unstableResults.length === 0
: !items || items.length === 0;
if (isEmpty)
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Kijiji scraping error:", error);
logger.error("Kijiji scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });

36
packages/api-server/test/routes.test.ts
View File

@@ -1,8 +1,23 @@
import { beforeEach, describe, expect, mock, test } from "bun:test";
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchKijijiItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchFacebookItems = mock(
(): Promise<
| { title: string }[]
| { results: { title: string }[]; unstableResults: { title: string }[] }
> => Promise.resolve([{ title: "item" }]),
);
const fetchEbayItems = mock(
(): Promise<
| { title: string }[]
| { results: { title: string }[]; unstableResults: { title: string }[] }
> => Promise.resolve([{ title: "item" }]),
);
const fetchKijijiItems = mock(
(): Promise<
| { title: string }[]
| { results: { title: string }[]; unstableResults: { title: string }[] }
> => Promise.resolve([{ title: "item" }]),
);
mock.module("@marketplace-scrapers/core", () => ({
fetchFacebookItems,
@@ -13,16 +28,19 @@ mock.module("@marketplace-scrapers/core", () => ({
describe("API routes", () => {
beforeEach(() => {
fetchFacebookItems.mockReset();
fetchFacebookItems.mockImplementation(() =>
Promise.resolve([{ title: "item" }]),
fetchFacebookItems.mockImplementation(
() =>
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
);
fetchEbayItems.mockReset();
fetchEbayItems.mockImplementation(() =>
Promise.resolve([{ title: "item" }]),
fetchEbayItems.mockImplementation(
() =>
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
);
fetchKijijiItems.mockReset();
fetchKijijiItems.mockImplementation(() =>
Promise.resolve([{ title: "item" }]),
fetchKijijiItems.mockImplementation(
() =>
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
);
});

18
packages/core/src/scrapers/ebay.ts
View File

@@ -10,6 +10,7 @@ import {
formatCookiesForHeader,
} from "../utils/cookies";
import { delay } from "../utils/delay";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable";
// eBay cookie configuration
@@ -274,7 +275,7 @@ function parseEbayListings(
);
// Filter to only elements that actually contain prices (not labels)
const actualPrices: HTMLElement[] = [];
const actualPrices: Element[] = [];
for (const el of allPriceElements) {
const text = el.textContent?.trim();
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
@@ -301,11 +302,10 @@ function parseEbayListings(
if (nonStrikethroughPrices.length > 0) {
// Use the first non-strikethrough price (sale price)
priceElement = nonStrikethroughPrices[0];
priceElement = nonStrikethroughPrices[0] ?? null;
} else {
// Fallback: use the last price (likely the most current)
const lastPrice = actualPrices[actualPrices.length - 1];
priceElement = lastPrice;
priceElement = actualPrices[actualPrices.length - 1] ?? null;
}
}
}
@@ -355,7 +355,7 @@ function parseEbayListings(
results.push(listing);
seenUrls.add(canonicalUrl);
} catch (err) {
console.warn(`Error parsing eBay listing: ${err}`);
logger.warn(`Error parsing eBay listing: ${err}`);
}
}
@@ -372,7 +372,7 @@ async function loadEbayCookies(): Promise<string | undefined> {
const cookies = await ensureCookies(EBAY_COOKIE_CONFIG);
return formatCookiesForHeader(cookies, "www.ebay.ca");
} catch {
console.warn(
logger.warn(
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
);
return undefined;
@@ -475,7 +475,7 @@ export default async function fetchEbayItems(
const DELAY_MS = Math.max(1, Math.floor(1000 / requestsPerSecond));
console.log(`Fetching eBay search: ${searchUrl}`);
logger.log(`Fetching eBay search: ${searchUrl}`);
try {
// Use custom headers modeled after real browser requests to bypass bot detection
@@ -517,7 +517,7 @@ export default async function fetchEbayItems(
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS);
console.log(`\nParsing eBay listings...`);
logger.log(`\nParsing eBay listings...`);
const listings = parseEbayListings(
searchHtml,
@@ -534,7 +534,7 @@ export default async function fetchEbayItems(
);
});
console.log(`Parsed ${filteredListings.length} eBay listings.`);
logger.log(`Parsed ${filteredListings.length} eBay listings.`);
return finalizeResults(filteredListings);
} catch (err) {
if (err instanceof HttpError) {

97
packages/core/src/scrapers/facebook.ts
View File

@@ -15,6 +15,7 @@ import {
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable";
/**
@@ -86,7 +87,7 @@ interface FacebookMarketplaceItem {
__typename: "GroupCommerceProductItem";
// Listing content
marketplace_listing_title: string;
marketplace_listing_title?: string;
redacted_description?: {
text: string;
};
@@ -99,7 +100,7 @@ interface FacebookMarketplaceItem {
listing_price?: {
amount: string;
currency: string;
amount_with_offset: string;
amount_with_offset?: string;
};
// Location
@@ -127,9 +128,9 @@ interface FacebookMarketplaceItem {
// Seller information
marketplace_listing_seller?: {
__typename: "User";
id: string;
name: string;
__typename?: "User";
id?: string;
name?: string;
profile_picture?: {
uri: string;
};
@@ -260,14 +261,14 @@ function logExtractionMetrics(success: boolean, itemId?: string) {
successRate < 0.8 &&
!extractionStats.lastApiChangeDetected
) {
console.warn(
logger.warn(
"Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
);
extractionStats.lastApiChangeDetected = new Date();
}
if (!success && itemId) {
console.warn(`Facebook API extraction failed for item ${itemId}`);
logger.warn(`Facebook API extraction failed for item ${itemId}`);
}
}
@@ -820,18 +821,18 @@ export function extractFacebookMarketplaceData(
if (htmlString.includes("XCometMarketplaceSearchController")) {
const htmlFallback = extractFacebookMarketplaceHtmlFallback(htmlString);
if (htmlFallback?.length) {
console.log(
logger.log(
`Successfully parsed ${htmlFallback.length} Facebook marketplace listings from rendered HTML fallback`,
);
return htmlFallback;
}
}
console.warn("No marketplace data found in HTML response");
logger.warn("No marketplace data found in HTML response");
return null;
}
console.log(
logger.log(
`Successfully parsed ${bestEdges.length} Facebook marketplace listings`,
);
return bestEdges.map((edge) => ({ node: edge.node }));
@@ -982,7 +983,7 @@ export function parseFacebookAds(
results.push(listingDetails);
} catch (error) {
console.warn("Failed to parse Facebook ad:", error);
logger.warn("Failed to parse Facebook ad:", error);
}
}
@@ -1083,7 +1084,7 @@ export function parseFacebookItem(
return listingDetails;
} catch (error) {
console.warn(`Failed to parse Facebook item ${item.id}:`, error);
logger.warn(`Failed to parse Facebook item ${item.id}:`, error);
return null;
}
}
@@ -1148,8 +1149,8 @@ export default async function fetchFacebookItems(
// Facebook marketplace URL structure
const searchUrl = `https://www.facebook.com/marketplace/${LOCATION}/search?query=${encodedQuery}&sortBy=creation_time_descend&exact=false`;
console.log(`Fetching Facebook marketplace: ${searchUrl}`);
console.log(`Using ${cookies.length} cookies for authentication`);
logger.log(`Fetching Facebook marketplace: ${searchUrl}`);
logger.log(`Using ${cookies.length} cookies for authentication`);
let searchHtml: string;
let searchResponseUrl = searchUrl;
@@ -1158,7 +1159,7 @@ export default async function fetchFacebookItems(
maxRetries: 3,
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
logger.log(
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
@@ -1169,11 +1170,11 @@ export default async function fetchFacebookItems(
searchResponseUrl = response.responseUrl;
} catch (err) {
if (err instanceof HttpError) {
console.warn(
logger.warn(
`\nFacebook marketplace access failed (${err.status}): ${err.message}`,
);
if (err.status === 400 || err.status === 401 || err.status === 403) {
console.warn(
logger.warn(
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
);
}
@@ -1187,19 +1188,19 @@ export default async function fetchFacebookItems(
searchResponseUrl,
);
if (classification.authGated) {
console.warn(
logger.warn(
"Facebook marketplace search redirected to login. Cookies may be expired.",
);
return finalizeResults([]);
}
if (classification.unavailable) {
console.warn("Facebook marketplace search returned an unavailable route.");
logger.warn("Facebook marketplace search returned an unavailable route.");
return finalizeResults([]);
}
if (classification.kind !== "search") {
console.warn(
logger.warn(
`Facebook marketplace search returned unexpected route kind: ${classification.kind}.`,
);
return finalizeResults([]);
@@ -1207,11 +1208,11 @@ export default async function fetchFacebookItems(
const ads = extractFacebookMarketplaceData(searchHtml);
if (!ads || ads.length === 0) {
console.warn("No ads parsed from Facebook marketplace page.");
logger.warn("No ads parsed from Facebook marketplace page.");
return finalizeResults([]);
}
console.log(`\nFound ${ads.length} raw ads. Processing...`);
logger.log(`\nFound ${ads.length} raw ads. Processing...`);
const isTTY = process.stdout?.isTTY ?? false;
const progressBar = isTTY
@@ -1232,7 +1233,7 @@ export default async function fetchFacebookItems(
progressBar?.update(totalProgress);
progressBar?.stop();
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
logger.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
return finalizeResults(pricedItems);
}
@@ -1254,7 +1255,7 @@ export async function fetchFacebookItem(
const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`;
console.log(`Fetching Facebook marketplace item: ${itemUrl}`);
logger.log(`Fetching Facebook marketplace item: ${itemUrl}`);
let itemHtml: string;
let itemResponseUrl = itemUrl;
@@ -1262,7 +1263,7 @@ export async function fetchFacebookItem(
const response = await fetchHtml(itemUrl, 1000, {
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
logger.log(
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
@@ -1273,7 +1274,7 @@ export async function fetchFacebookItem(
itemResponseUrl = response.responseUrl;
} catch (err) {
if (err instanceof HttpError) {
console.warn(
logger.warn(
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
);
@@ -1282,29 +1283,29 @@ export async function fetchFacebookItem(
case 400:
case 401:
case 403:
console.warn(
logger.warn(
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
);
break;
case 404:
console.warn(
logger.warn(
"Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.",
);
break;
case 429:
console.warn(
logger.warn(
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
);
break;
case 500:
case 502:
case 503:
console.warn(
logger.warn(
"Facebook server error: Marketplace may be temporarily unavailable.",
);
break;
default:
console.warn(`Unexpected error status: ${err.status}`);
logger.warn(`Unexpected error status: ${err.status}`);
}
return null;
}
@@ -1315,17 +1316,25 @@ export async function fetchFacebookItem(
if (classification.authGated) {
logExtractionMetrics(false, itemId);
console.warn(
logger.warn(
`Authentication failed for item ${itemId}. Cookies may be expired.`,
);
return null;
}
if (itemResponseUrl.includes("unavailable_product=1")) {
logExtractionMetrics(false, itemId);
logger.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}
const itemData = extractFacebookItemData(itemHtml);
if (classification.unavailable && !itemData) {
logExtractionMetrics(false, itemId);
console.warn(
logger.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
@@ -1333,7 +1342,7 @@ export async function fetchFacebookItem(
if (classification.kind !== "item" && !itemData) {
logExtractionMetrics(false, itemId);
console.warn(
logger.warn(
`Item ${itemId} returned unexpected route kind: ${classification.kind}.`,
);
return null;
@@ -1343,38 +1352,38 @@ export async function fetchFacebookItem(
logExtractionMetrics(false, itemId);
if (itemHtml.includes("This item has been sold")) {
console.warn(
logger.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}
console.warn(
logger.warn(
`No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`,
);
console.warn(" - The listing was removed or sold");
console.warn(" - Authentication issues");
console.warn(" - Facebook changed their API structure");
console.warn(" - Network or parsing issues");
logger.warn(" - The listing was removed or sold");
logger.warn(" - Authentication issues");
logger.warn(" - Facebook changed their API structure");
logger.warn(" - Network or parsing issues");
return null;
}
logExtractionMetrics(true, itemId);
console.log(`Successfully extracted data for item ${itemId}`);
logger.log(`Successfully extracted data for item ${itemId}`);
const parsedItem = parseFacebookItem(itemData);
if (!parsedItem) {
console.warn(`Failed to parse item ${itemId}: Invalid data structure`);
logger.warn(`Failed to parse item ${itemId}: Invalid data structure`);
return null;
}
// Check for sold/removed status in the parsed data with proper precedence
if (itemData.is_sold) {
console.warn(`Item ${itemId} is marked as sold in the marketplace.`);
logger.warn(`Item ${itemId} is marked as sold in the marketplace.`);
// Still return the data but mark it as sold
parsedItem.listingStatus = "SOLD";
} else if (!itemData.is_live) {
console.warn(`Item ${itemId} is not live/active in the marketplace.`);
logger.warn(`Item ${itemId} is not live/active in the marketplace.`);
parsedItem.listingStatus = itemData.is_hidden
? "HIDDEN"
: itemData.is_pending

22
packages/core/src/scrapers/kijiji.ts
View File

@@ -21,6 +21,7 @@ import {
RateLimitError,
ValidationError,
} from "../utils/http";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable";
// Kijiji cookie configuration
@@ -28,7 +29,6 @@ const KIJIJI_COOKIE_CONFIG: CookieConfig = {
name: "Kijiji",
domain: ".kijiji.ca",
envVar: "KIJIJI_COOKIE",
filePath: "./cookies/kijiji.json",
};
// ----------------------------- Types -----------------------------
@@ -490,7 +490,7 @@ async function fetchSellerDetails(
};
} catch (err) {
// Silently fail for GraphQL errors - not critical for basic functionality
console.warn(
logger.warn(
`Failed to fetch seller details for ${posterId}:`,
err instanceof Error ? err.message : String(err),
);
@@ -724,7 +724,7 @@ export async function parseDetailedListing(
};
} catch {
// Silently fail - GraphQL data is optional
console.warn(
logger.warn(
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
);
}
@@ -861,11 +861,11 @@ export default async function fetchKijijiItems(
BASE_URL,
);
console.log(`Fetching search page ${page}: ${searchUrl}`);
logger.log(`Fetching search page ${page}: ${searchUrl}`);
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
logger.log(
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
@@ -875,9 +875,7 @@ export default async function fetchKijijiItems(
const searchResults = parseSearch(searchHtml, BASE_URL);
if (searchResults.length === 0) {
console.log(
`No more results found on page ${page}. Stopping pagination.`,
);
logger.log(`No more results found on page ${page}. Stopping pagination.`);
break;
}
@@ -890,7 +888,7 @@ export default async function fetchKijijiItems(
seenUrls.add(link);
}
console.log(
logger.log(
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
);
@@ -921,7 +919,7 @@ export default async function fetchKijijiItems(
// Staggered starts keep request pacing within REQUESTS_PER_SECOND.
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
logger.log(
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
@@ -949,7 +947,7 @@ export default async function fetchKijijiItems(
currentProgress++;
progressBar?.update(currentProgress);
if (!progressBar) {
console.log(`Progress: ${currentProgress}/${totalProgress}`);
logger.log(`Progress: ${currentProgress}/${totalProgress}`);
}
}
});
@@ -978,7 +976,7 @@ export default async function fetchKijijiItems(
matchesPriceFilters(listing, finalSearchOptions),
);
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
logger.log(`\nParsed ${filteredListings.length} detailed listings.`);
return finalizeResults(filteredListings);
}

23
packages/core/src/utils/cookies.ts
View File

@@ -2,6 +2,8 @@
* Shared cookie handling utilities for marketplace scrapers
*/
import { logger } from "./logger";
export interface Cookie {
name: string;
value: string;
@@ -41,9 +43,9 @@ export function parseCookieString(
.split(";")
.map((pair) => pair.trim())
.filter((pair) => pair.includes("="))
.map((pair) => {
.map((pair): Cookie | null => {
const [name, ...valueParts] = pair.split("=");
const trimmedName = name.trim();
const trimmedName = name?.trim();
const trimmedValue = valueParts.join("=").trim();
if (!trimmedName || !trimmedValue) {
@@ -98,14 +100,25 @@ export function formatCookiesForHeader(
}
/**
* Load cookies from the configured environment variable
* Load cookies from the configured environment variable or explicit cookie string
*/
export async function ensureCookies(config: CookieConfig): Promise<Cookie[]> {
export async function ensureCookies(
config: CookieConfig,
cookiesSource?: string,
): Promise<Cookie[]> {
// Explicit cookie string takes priority
if (cookiesSource) {
const cookies = parseCookieString(cookiesSource, config.domain);
if (cookies.length > 0) {
return cookies;
}
}
const envValue = process.env[config.envVar];
const cookies = parseCookieString(envValue ?? "", config.domain);
if (cookies.length > 0) {
console.log(
logger.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
);
return cookies;

2
packages/core/src/utils/delay.ts
View File

@@ -4,5 +4,7 @@
* @returns A promise that resolves after the specified delay
*/
export function delay(ms: number): Promise<void> {
if (process.env.NODE_ENV === "test") return Promise.resolve();
return new Promise((resolve) => setTimeout(resolve, ms));
}

19
packages/core/src/utils/http.ts
View File

@@ -1,56 +1,55 @@
import { delay } from "./delay";
/** Custom error class for HTTP-related failures */
export class HttpError extends Error {
override name = "HttpError";
constructor(
message: string,
public readonly statusCode: number,
public readonly url?: string,
) {
super(message);
this.name = "HttpError";
}
}
/** Error class for network failures (timeouts, connection issues) */
export class NetworkError extends Error {
override name = "NetworkError";
constructor(
message: string,
public readonly url: string,
public readonly cause?: Error,
public override readonly cause?: Error,
) {
super(message);
this.name = "NetworkError";
}
}
/** Error class for parsing failures */
export class ParseError extends Error {
override name = "ParseError";
constructor(
message: string,
public readonly data?: unknown,
) {
super(message);
this.name = "ParseError";
}
}
/** Error class for rate limiting */
export class RateLimitError extends Error {
override name = "RateLimitError";
constructor(
message: string,
public readonly url: string,
public readonly resetTime?: number,
) {
super(message);
this.name = "RateLimitError";
}
}
/** Error class for validation failures */
export class ValidationError extends Error {
constructor(message: string) {
super(message);
this.name = "ValidationError";
}
override name = "ValidationError";
}
/** Type guard to check if a value is a record (object) */
@@ -170,7 +169,7 @@ export async function fetchHtml(
const html = await res.text();
// Respect per-request delay to maintain rate limiting
await new Promise((resolve) => setTimeout(resolve, delayMs));
await delay(delayMs);
return html;
} catch (err) {
// Re-throw known errors

10
packages/core/src/utils/logger.ts Normal file
View File

@@ -0,0 +1,10 @@
const isTest = () => process.env.NODE_ENV === "test";
export const logger = {
log: (...args: Parameters<typeof console.log>) => {
if (!isTest()) console.log(...args);
},
warn: (...args: Parameters<typeof console.warn>) => {
if (!isTest()) console.warn(...args);
},
};

28
packages/core/src/utils/unstable.ts
View File

@@ -1,21 +1,29 @@
import type { ListingDetails, UnstableListingBuckets } from "../types/common";
import type { UnstableListingBuckets } from "../types/common";
interface HasListingPrice {
listingPrice?: { cents?: number } | null;
}
function getMedian(values: number[]): number {
const middleIndex = Math.floor(values.length / 2);
if (values.length % 2 === 0) {
return (values[middleIndex - 1] + values[middleIndex]) / 2;
const left = values[middleIndex - 1] ?? 0;
const right = values[middleIndex] ?? 0;
return (left + right) / 2;
}
return values[middleIndex];
return values[middleIndex] ?? 0;
}
export function classifyUnstableListings<T extends ListingDetails>(
export function classifyUnstableListings<T extends HasListingPrice>(
listings: T[],
): UnstableListingBuckets<T> {
const validPrices = listings
.map((listing) => listing.listingPrice.cents)
.filter((price) => Number.isFinite(price) && price > 0)
.map((listing) => listing.listingPrice?.cents)
.filter(
(price): price is number => Number.isFinite(price) && (price ?? 0) > 0,
)
.sort((left, right) => left - right);
if (validPrices.length < 2) {
@@ -32,9 +40,13 @@ export function classifyUnstableListings<T extends ListingDetails>(
};
for (const listing of listings) {
const price = listing.listingPrice.cents;
const price = listing.listingPrice?.cents;
if (Number.isFinite(price) && price > 0 && price < threshold) {
if (
Number.isFinite(price) &&
(price ?? 0) > 0 &&
(price ?? 0) < threshold
) {
buckets.unstableResults.push(listing);
continue;
}

24
packages/core/test/delay.test.ts Normal file
View File

@@ -0,0 +1,24 @@
import { afterEach, describe, expect, mock, test } from "bun:test";
import { delay } from "../src/utils/delay";
describe("delay", () => {
const originalNodeEnv = process.env.NODE_ENV;
const originalSetTimeout = globalThis.setTimeout;
afterEach(() => {
process.env.NODE_ENV = originalNodeEnv;
globalThis.setTimeout = originalSetTimeout;
});
test("does not schedule throttle timers during tests", async () => {
process.env.NODE_ENV = "test";
const setTimeoutMock = mock(() => {
throw new Error("setTimeout should not be called during tests");
});
globalThis.setTimeout = setTimeoutMock as unknown as typeof setTimeout;
await delay(1000);
expect(setTimeoutMock).not.toHaveBeenCalled();
});
});

52
packages/core/test/ebay-core.test.ts
View File

@@ -34,7 +34,7 @@ describe("eBay Scraper Cookie Handling", () => {
ok: true,
text: () => Promise.resolve("<html><body></body></html>"),
}),
) as typeof fetch;
) as unknown as typeof fetch;
});
afterEach(() => {
@@ -44,20 +44,20 @@ describe("eBay Scraper Cookie Handling", () => {
});
test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => {
const warnMock = mock(() => {});
console.warn = warnMock;
await fetchEbayItems("laptop", 1000);
expect(global.fetch).toHaveBeenCalledTimes(1);
const [, init] = (global.fetch as ReturnType<typeof mock>).mock.calls[0];
const firstFetchCall = (global.fetch as unknown as ReturnType<typeof mock>)
.mock.calls[0];
if (!firstFetchCall) {
throw new Error("Expected fetch to be called");
}
const [, init] = firstFetchCall;
const headers = (init as RequestInit).headers as Record<string, string>;
expect(headers.Cookie).toBeUndefined();
expect(warnMock).toHaveBeenCalledWith(
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
);
});
test("keeps relative item links on the ebay.ca host", async () => {
@@ -75,7 +75,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -100,7 +100,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -130,7 +130,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -167,7 +167,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -199,7 +199,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -225,7 +225,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -254,7 +254,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -283,7 +283,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -317,7 +317,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("bundle", 1000, {
keywords: ["bundle"],
@@ -357,7 +357,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -389,7 +389,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -421,7 +421,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000);
@@ -451,7 +451,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("bike", 1000);
@@ -478,7 +478,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("microphone", 1000, {
keywords: ["microphone"],
@@ -510,7 +510,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000, {
minPrice: 0,
@@ -550,7 +550,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems(
"laptop",
@@ -595,7 +595,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems("laptop", 1000, { maxItems: 2 });
@@ -633,7 +633,7 @@ describe("eBay Scraper Cookie Handling", () => {
</body></html>
`),
}),
) as typeof fetch;
) as unknown as typeof fetch;
const results = await fetchEbayItems(
"laptop",

92
packages/core/test/facebook-core.test.ts
View File

@@ -52,7 +52,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
beforeEach(() => {
global.fetch = mock(() => {
throw new Error("fetch should be mocked in individual tests");
});
}) as unknown as typeof fetch;
});
afterEach(() => {
@@ -93,8 +93,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
const result = parseFacebookCookieString(cookieString);
expect(result[0].value).toBe("123+456");
expect(result[1].value).toBe("abc=def");
expect(result[0]?.value).toBe("123+456");
expect(result[1]?.value).toBe("abc=def");
});
test("should filter out malformed cookies", () => {
@@ -115,10 +115,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const result = parseFacebookCookieString(cookieString);
expect(result).toHaveLength(2);
expect(result[0].name).toBe("c_user");
expect(result[0].value).toBe("123");
expect(result[1].name).toBe("xs");
expect(result[1].value).toBe("abc");
expect(result[0]?.name).toBe("c_user");
expect(result[0]?.value).toBe("123");
expect(result[1]?.name).toBe("xs");
expect(result[1]?.value).toBe("abc");
});
test("should load Facebook cookies from FACEBOOK_COOKIE env var", async () => {
@@ -177,10 +177,6 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("should handle authentication errors", async () => {
const originalWarn = console.warn;
const warnMock = mock(() => {});
console.warn = warnMock;
global.fetch = mock(() =>
Promise.resolve({
ok: false,
@@ -190,18 +186,11 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
try {
const result = await fetchFacebookItem("123");
expect(result).toBeNull();
expect(global.fetch).toHaveBeenCalledTimes(1);
expect(warnMock).toHaveBeenCalledWith(
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
);
} finally {
console.warn = originalWarn;
}
});
test("should handle item not found", async () => {
@@ -214,7 +203,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("nonexistent");
expect(result).toBeNull();
@@ -274,7 +263,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
});
});
}) as unknown as typeof fetch;
const _result = await fetchFacebookItem("123");
expect(attempts).toBe(2);
@@ -297,7 +286,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
text: () => Promise.resolve("Rate limited"),
});
});
}) as unknown as typeof fetch;
const result = await fetchFacebookItem("429-loop");
@@ -346,7 +335,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("456");
expect(result?.listingStatus).toBe("SOLD");
@@ -388,7 +377,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("457");
@@ -435,7 +424,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("458");
@@ -493,7 +482,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("789");
expect(result).not.toBeNull();
@@ -512,7 +501,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("error");
expect(result).toBeNull();
@@ -573,7 +562,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
@@ -618,7 +607,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
@@ -682,7 +671,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
@@ -762,7 +751,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("chair", 1, "toronto", 25, {
hideUnstableResults: true,
@@ -845,7 +834,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("chair", 1, "toronto", 2, {
hideUnstableResults: true,
@@ -1132,7 +1121,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const result = extractFacebookMarketplaceData(html);
expect(result).not.toBeNull();
expect(result).toHaveLength(2);
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
expect(result?.[0]?.node.listing.marketplace_listing_title).toBe(
"Item 1",
);
});
@@ -1153,11 +1142,11 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const result = extractFacebookMarketplaceData(html);
expect(result).not.toBeNull();
expect(result).toHaveLength(1);
expect(result?.[0].node.listing.id).toBe("987654321");
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
expect(result?.[0]?.node.listing.id).toBe("987654321");
expect(result?.[0]?.node.listing.marketplace_listing_title).toBe(
"Vintage Bike",
);
expect(result?.[0].node.listing.listing_price).toEqual({
expect(result?.[0]?.node.listing.listing_price).toEqual({
amount: "120.00",
formatted_amount: "CA$120",
currency: "CAD",
@@ -1385,7 +1374,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const ads = extractFacebookMarketplaceData(html);
expect(ads).toHaveLength(1);
expect(ads?.[0].node.listing.marketplace_listing_title).toBe("Bike");
expect(ads?.[0]?.node.listing.marketplace_listing_title).toBe("Bike");
});
test("prefers the strongest marketplace edge set when multiple edges arrays exist", () => {
@@ -1443,7 +1432,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const ads = extractFacebookMarketplaceData(html);
expect(ads).toHaveLength(1);
expect(ads?.[0].node.listing.id).toBe("right-1");
expect(ads?.[0]?.node.listing.id).toBe("right-1");
});
test("rejects mixed edge arrays that contain non-listing entries", () => {
@@ -1668,11 +1657,11 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const results = parseFacebookAds(ads);
expect(results).toHaveLength(2);
expect(results[0].title).toBe("Ad 1");
expect(results[0].listingPrice?.cents).toBe(5000);
expect(results[0].address).toBe("Toronto");
expect(results[1].title).toBe("Ad 2");
expect(results[1].address).toBe("Ottawa");
expect(results[0]?.title).toBe("Ad 1");
expect(results[0]?.listingPrice?.cents).toBe(5000);
expect(results[0]?.address).toBe("Toronto");
expect(results[1]?.title).toBe("Ad 2");
expect(results[1]?.address).toBe("Ottawa");
});
test("should filter out ads without price", () => {
@@ -1704,14 +1693,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
const results = parseFacebookAds(ads);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("With Price");
expect(results[0]?.title).toBe("With Price");
});
test("should handle malformed ads gracefully", () => {
const originalWarn = console.warn;
const warnMock = mock(() => {});
console.warn = warnMock;
const ads = [
{
node: {
@@ -1731,15 +1716,14 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
node: {
// Missing listing
},
} as { node: { listing?: unknown } },
} as unknown as { node: { listing?: unknown } },
];
const results = parseFacebookAds(ads);
const results = parseFacebookAds(
ads as unknown as Parameters<typeof parseFacebookAds>[0],
);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Valid Ad");
expect(warnMock).toHaveBeenCalledTimes(1);
console.warn = originalWarn;
expect(results[0]?.title).toBe("Valid Ad");
});
test("parses formatted fallback prices with multiple commas", () => {

52
packages/core/test/facebook-integration.test.ts
View File

@@ -15,7 +15,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
process.env.FACEBOOK_COOKIE = facebookCookie;
global.fetch = mock(() => {
throw new Error("fetch should be mocked in individual tests");
});
}) as unknown as typeof fetch;
});
afterEach(() => {
@@ -69,11 +69,11 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("iPhone", 1, "toronto", 25);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("iPhone 13");
expect(results[0]?.title).toBe("iPhone 13");
});
test("should filter out items without price", async () => {
@@ -135,11 +135,11 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 25);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("With Price");
expect(results[0]?.title).toBe("With Price");
});
test("should respect MAX_ITEMS parameter", async () => {
@@ -190,7 +190,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 5);
expect(results).toHaveLength(5);
@@ -231,7 +231,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems(
"nonexistent query",
@@ -252,7 +252,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 25);
expect(results).toEqual([]);
@@ -281,7 +281,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("lamp", 1, "toronto", 25);
expect(results).toEqual([]);
@@ -322,14 +322,16 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("lamp", 1, "toronto", 25);
expect(results).toEqual([]);
});
test("should handle network errors", async () => {
global.fetch = mock(() => Promise.reject(new Error("Network error")));
global.fetch = mock(() =>
Promise.reject(new Error("Network error")),
) as unknown as typeof fetch;
await expect(
fetchFacebookItems("test", 1, "toronto", 25),
@@ -400,7 +402,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
});
});
}) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 25);
expect(attempts).toBe(2);
@@ -473,13 +475,13 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("cars", 1, "toronto", 25);
expect(results).toHaveLength(2);
// Both should be classified as "item" type in search results (vehicle detection is for item details)
expect(results[0].title).toBe("2006 Honda Civic");
expect(results[1].title).toBe("iPhone 13");
expect(results[0]?.title).toBe("2006 Honda Civic");
expect(results[1]?.title).toBe("iPhone 13");
});
});
@@ -542,7 +544,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems(
"nintendo switch",
@@ -551,8 +553,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
25,
);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Nintendo Switch");
expect(results[0].categoryId).toBe("479353692612078");
expect(results[0]?.title).toBe("Nintendo Switch");
expect(results[0]?.categoryId).toBe("479353692612078");
});
test("should handle home goods/furniture listings", async () => {
@@ -613,12 +615,12 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("table", 1, "toronto", 25);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Dining Table");
expect(results[0].categoryId).toBe("1569171756675761");
expect(results[0]?.title).toBe("Dining Table");
expect(results[0]?.categoryId).toBe("1569171756675761");
});
});
@@ -635,7 +637,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 25);
expect(results).toEqual([]);
@@ -651,7 +653,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 25);
expect(results).toEqual([]);
@@ -667,7 +669,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const results = await fetchFacebookItems("test", 1, "toronto", 25);
expect(results).toEqual([]);
@@ -708,7 +710,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
get: () => null,
},
}),
);
) as unknown as typeof fetch;
const result = await fetchFacebookItem("123");
expect(result).toBeNull();

41
packages/core/test/http.test.ts Normal file
View File

@@ -0,0 +1,41 @@
import { afterEach, describe, expect, mock, test } from "bun:test";
import { fetchHtml } from "../src/utils/http";
describe("fetchHtml", () => {
const originalFetch = global.fetch;
const originalNodeEnv = process.env.NODE_ENV;
const originalSetTimeout = globalThis.setTimeout;
const originalClearTimeout = globalThis.clearTimeout;
afterEach(() => {
global.fetch = originalFetch;
process.env.NODE_ENV = originalNodeEnv;
globalThis.setTimeout = originalSetTimeout;
globalThis.clearTimeout = originalClearTimeout;
});
test("does not schedule throttle timers during tests", async () => {
process.env.NODE_ENV = "test";
const scheduledDelays: number[] = [];
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () => Promise.resolve("<html></html>"),
}),
) as unknown as typeof fetch;
globalThis.setTimeout = mock((handler: TimerHandler, timeout?: number) => {
scheduledDelays.push(Number(timeout));
if (timeout !== 30_000 && typeof handler === "function") {
handler();
}
return 0 as unknown as ReturnType<typeof setTimeout>;
}) as unknown as typeof setTimeout;
globalThis.clearTimeout = mock(() => {}) as unknown as typeof clearTimeout;
await fetchHtml("https://example.com", 1000, { timeoutMs: 30_000 });
expect(scheduledDelays).not.toContain(1000);
});
});

16
packages/core/test/kijiji-core.test.ts
View File

@@ -49,7 +49,7 @@ const originalFetch = global.fetch;
beforeEach(() => {
global.fetch = mock(() => {
throw new Error("fetch should be mocked in individual tests");
});
}) as unknown as typeof fetch;
});
afterEach(() => {
@@ -310,7 +310,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
const results = await fetchKijijiItems(
"phone",
@@ -418,7 +418,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
const results = await fetchKijijiItems(
"phone",
@@ -515,7 +515,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
const results = await fetchKijijiItems(
"phone",
@@ -628,7 +628,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
const results = await fetchKijijiItems(
"phone",
@@ -771,7 +771,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
const results = await fetchKijijiItems(
"phone",
@@ -872,7 +872,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
await parseDetailedListing(html, "https://www.kijiji.ca", {
includeClientSideData: true,
@@ -981,7 +981,7 @@ describe("fetchKijijiItems", () => {
}
throw new Error(`Unexpected URL: ${url}`);
}) as typeof fetch;
}) as unknown as typeof fetch;
const results = await fetchKijijiItems(
"phone",

6
packages/core/test/kijiji-integration.test.ts
View File

@@ -13,7 +13,7 @@ describe("HTML Parsing Integration", () => {
// Mock fetch for all tests
global.fetch = mock(() => {
throw new Error("fetch should be mocked in individual tests");
});
}) as unknown as typeof fetch;
});
afterEach(() => {
@@ -111,7 +111,7 @@ describe("HTML Parsing Integration", () => {
`;
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
expect(results[0].listingLink).toBe(
expect(results[0]?.listingLink).toBe(
"https://www.kijiji.ca/v-iphone/k0l0",
);
});
@@ -146,7 +146,7 @@ describe("HTML Parsing Integration", () => {
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
expect(results).toHaveLength(1);
expect(results[0].name).toBe("iPhone 13 Pro");
expect(results[0]?.name).toBe("iPhone 13 Pro");
});
test("should return empty array for invalid HTML", () => {

29
packages/core/test/logger.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { afterEach, describe, expect, mock, test } from "bun:test";
describe("logger", () => {
const originalNodeEnv = process.env.NODE_ENV;
const originalConsoleLog = console.log;
const originalConsoleWarn = console.warn;
afterEach(() => {
process.env.NODE_ENV = originalNodeEnv;
console.log = originalConsoleLog;
console.warn = originalConsoleWarn;
});
test("suppresses log and warn output during tests", async () => {
process.env.NODE_ENV = "test";
const logMock = mock(() => {});
const warnMock = mock(() => {});
console.log = logMock;
console.warn = warnMock;
const { logger } = await import("../src/utils/logger");
logger.log("hidden log");
logger.warn("hidden warn");
expect(logMock).not.toHaveBeenCalled();
expect(warnMock).not.toHaveBeenCalled();
});
});

3
packages/mcp-server/src/index.ts
View File

@@ -1,3 +1,4 @@
import { logger } from "./logger";
import { handleMcpRequest } from "./protocol/handler";
import { serverCard } from "./protocol/metadata";
@@ -33,4 +34,4 @@ const server = Bun.serve({
},
});
console.log(`MCP Server running on ${server.hostname}:${server.port}`);
logger.log(`MCP Server running on ${server.hostname}:${server.port}`);

10
packages/mcp-server/src/logger.ts Normal file
View File

@@ -0,0 +1,10 @@
const isTest = () => process.env.NODE_ENV === "test";
export const logger = {
log: (...args: Parameters<typeof console.log>) => {
if (!isTest()) console.log(...args);
},
error: (...args: Parameters<typeof console.error>) => {
if (!isTest()) console.error(...args);
},
};

19
packages/mcp-server/src/protocol/handler.ts
View File

@@ -1,3 +1,4 @@
import { logger } from "../logger";
import { tools } from "./tools";
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
@@ -119,7 +120,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (args.unstableFilter !== undefined)
params.append("unstableFilter", args.unstableFilter.toString());
console.log(
logger.log(
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
);
const response = await Promise.race([
@@ -135,13 +136,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (!response.ok) {
const errorText = await response.text();
console.error(
logger.error(
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
logger.log(
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else if (name === "search_facebook") {
@@ -160,7 +161,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (args.unstableFilter !== undefined)
params.append("unstableFilter", args.unstableFilter.toString());
console.log(
logger.log(
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
);
const response = await Promise.race([
@@ -176,13 +177,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (!response.ok) {
const errorText = await response.text();
console.error(
logger.error(
`[MCP] Facebook API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
logger.log(
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else if (name === "search_ebay") {
@@ -214,7 +215,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (args.unstableFilter !== undefined)
params.append("unstableFilter", args.unstableFilter.toString());
console.log(
logger.log(
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
);
const response = await Promise.race([
@@ -230,13 +231,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (!response.ok) {
const errorText = await response.text();
console.error(
logger.error(
`[MCP] eBay API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
logger.log(
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else {

12
packages/mcp-server/test/protocol.test.ts
View File

@@ -8,7 +8,7 @@ describe("MCP protocol cookie inputs", () => {
beforeEach(() => {
global.fetch = mock(() =>
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
) as typeof fetch;
) as unknown as typeof fetch;
});
afterEach(() => {
@@ -48,7 +48,7 @@ describe("MCP protocol cookie inputs", () => {
}),
);
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
.calls[0]?.[0];
expect(String(calledUrl)).toContain("/facebook?q=laptop");
expect(String(calledUrl)).not.toContain("cookies=");
@@ -59,7 +59,7 @@ describe("MCP protocol unstableFilter", () => {
beforeEach(() => {
global.fetch = mock(() =>
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
) as typeof fetch;
) as unknown as typeof fetch;
});
afterEach(() => {
@@ -103,7 +103,7 @@ describe("MCP protocol unstableFilter", () => {
}),
);
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
.calls[0]?.[0];
expect(String(calledUrl)).toContain("unstableFilter=true");
});
@@ -127,7 +127,7 @@ describe("MCP protocol unstableFilter", () => {
}),
);
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
.calls[0]?.[0];
expect(String(calledUrl)).toContain("unstableFilter=true");
});
@@ -151,7 +151,7 @@ describe("MCP protocol unstableFilter", () => {
}),
);
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
.calls[0]?.[0];
expect(String(calledUrl)).toContain("unstableFilter=true");
});