test: quiet and speed up test runs

This commit is contained in:
2026-04-28 21:45:06 -04:00
parent c6c44a0914
commit 2a5701aeb9
20 changed files with 212 additions and 99 deletions

View File

@@ -1,3 +1,4 @@
import { logger } from "./logger";
import { ebayRoute } from "./routes/ebay"; import { ebayRoute } from "./routes/ebay";
import { facebookRoute } from "./routes/facebook"; import { facebookRoute } from "./routes/facebook";
import { kijijiRoute } from "./routes/kijiji"; import { kijijiRoute } from "./routes/kijiji";
@@ -27,4 +28,4 @@ const server = Bun.serve({
}, },
}); });
console.log(`API Server running on ${server.hostname}:${server.port}`); logger.log(`API Server running on ${server.hostname}:${server.port}`);

View File

@@ -0,0 +1,10 @@
const isTest = () => process.env.NODE_ENV === "test";
export const logger = {
log: (...args: Parameters<typeof console.log>) => {
if (!isTest()) console.log(...args);
},
error: (...args: Parameters<typeof console.error>) => {
if (!isTest()) console.error(...args);
},
};

View File

@@ -1,4 +1,5 @@
import { fetchEbayItems } from "@marketplace-scrapers/core"; import { fetchEbayItems } from "@marketplace-scrapers/core";
import { logger } from "../logger";
/** /**
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly} * GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}
@@ -90,7 +91,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
); );
return Response.json(items, { status: 200 }); return Response.json(items, { status: 200 });
} catch (error) { } catch (error) {
console.error("eBay scraping error:", error); logger.error("eBay scraping error:", error);
const errorMessage = const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred"; error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 }); return Response.json({ message: errorMessage }, { status: 400 });

View File

@@ -1,4 +1,5 @@
import { fetchFacebookItems } from "@marketplace-scrapers/core"; import { fetchFacebookItems } from "@marketplace-scrapers/core";
import { logger } from "../logger";
/** /**
* GET /api/facebook?q={query}&location={location} * GET /api/facebook?q={query}&location={location}
@@ -57,7 +58,7 @@ export async function facebookRoute(req: Request): Promise<Response> {
); );
return Response.json(items, { status: 200 }); return Response.json(items, { status: 200 });
} catch (error) { } catch (error) {
console.error("Facebook scraping error:", error); logger.error("Facebook scraping error:", error);
const errorMessage = const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred"; error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 }); return Response.json({ message: errorMessage }, { status: 400 });

View File

@@ -1,4 +1,5 @@
import { fetchKijijiItems } from "@marketplace-scrapers/core"; import { fetchKijijiItems } from "@marketplace-scrapers/core";
import { logger } from "../logger";
/** /**
* GET /api/kijiji?q={query} * GET /api/kijiji?q={query}
@@ -97,7 +98,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
); );
return Response.json(items, { status: 200 }); return Response.json(items, { status: 200 });
} catch (error) { } catch (error) {
console.error("Kijiji scraping error:", error); logger.error("Kijiji scraping error:", error);
const errorMessage = const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred"; error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 }); return Response.json({ message: errorMessage }, { status: 400 });

View File

@@ -10,6 +10,7 @@ import {
formatCookiesForHeader, formatCookiesForHeader,
} from "../utils/cookies"; } from "../utils/cookies";
import { delay } from "../utils/delay"; import { delay } from "../utils/delay";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable"; import { classifyUnstableListings } from "../utils/unstable";
// eBay cookie configuration // eBay cookie configuration
@@ -354,7 +355,7 @@ function parseEbayListings(
results.push(listing); results.push(listing);
seenUrls.add(canonicalUrl); seenUrls.add(canonicalUrl);
} catch (err) { } catch (err) {
console.warn(`Error parsing eBay listing: ${err}`); logger.warn(`Error parsing eBay listing: ${err}`);
} }
} }
@@ -371,7 +372,7 @@ async function loadEbayCookies(): Promise<string | undefined> {
const cookies = await ensureCookies(EBAY_COOKIE_CONFIG); const cookies = await ensureCookies(EBAY_COOKIE_CONFIG);
return formatCookiesForHeader(cookies, "www.ebay.ca"); return formatCookiesForHeader(cookies, "www.ebay.ca");
} catch { } catch {
console.warn( logger.warn(
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.", "No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
); );
return undefined; return undefined;
@@ -474,7 +475,7 @@ export default async function fetchEbayItems(
const DELAY_MS = Math.max(1, Math.floor(1000 / requestsPerSecond)); const DELAY_MS = Math.max(1, Math.floor(1000 / requestsPerSecond));
console.log(`Fetching eBay search: ${searchUrl}`); logger.log(`Fetching eBay search: ${searchUrl}`);
try { try {
// Use custom headers modeled after real browser requests to bypass bot detection // Use custom headers modeled after real browser requests to bypass bot detection
@@ -516,7 +517,7 @@ export default async function fetchEbayItems(
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND // Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS); await delay(DELAY_MS);
console.log(`\nParsing eBay listings...`); logger.log(`\nParsing eBay listings...`);
const listings = parseEbayListings( const listings = parseEbayListings(
searchHtml, searchHtml,
@@ -533,7 +534,7 @@ export default async function fetchEbayItems(
); );
}); });
console.log(`Parsed ${filteredListings.length} eBay listings.`); logger.log(`Parsed ${filteredListings.length} eBay listings.`);
return finalizeResults(filteredListings); return finalizeResults(filteredListings);
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {

View File

@@ -15,6 +15,7 @@ import {
import { delay } from "../utils/delay"; import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format"; import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http"; import { isRecord } from "../utils/http";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable"; import { classifyUnstableListings } from "../utils/unstable";
/** /**
@@ -260,14 +261,14 @@ function logExtractionMetrics(success: boolean, itemId?: string) {
successRate < 0.8 && successRate < 0.8 &&
!extractionStats.lastApiChangeDetected !extractionStats.lastApiChangeDetected
) { ) {
console.warn( logger.warn(
"Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.", "Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
); );
extractionStats.lastApiChangeDetected = new Date(); extractionStats.lastApiChangeDetected = new Date();
} }
if (!success && itemId) { if (!success && itemId) {
console.warn(`Facebook API extraction failed for item ${itemId}`); logger.warn(`Facebook API extraction failed for item ${itemId}`);
} }
} }
@@ -820,18 +821,18 @@ export function extractFacebookMarketplaceData(
if (htmlString.includes("XCometMarketplaceSearchController")) { if (htmlString.includes("XCometMarketplaceSearchController")) {
const htmlFallback = extractFacebookMarketplaceHtmlFallback(htmlString); const htmlFallback = extractFacebookMarketplaceHtmlFallback(htmlString);
if (htmlFallback?.length) { if (htmlFallback?.length) {
console.log( logger.log(
`Successfully parsed ${htmlFallback.length} Facebook marketplace listings from rendered HTML fallback`, `Successfully parsed ${htmlFallback.length} Facebook marketplace listings from rendered HTML fallback`,
); );
return htmlFallback; return htmlFallback;
} }
} }
console.warn("No marketplace data found in HTML response"); logger.warn("No marketplace data found in HTML response");
return null; return null;
} }
console.log( logger.log(
`Successfully parsed ${bestEdges.length} Facebook marketplace listings`, `Successfully parsed ${bestEdges.length} Facebook marketplace listings`,
); );
return bestEdges.map((edge) => ({ node: edge.node })); return bestEdges.map((edge) => ({ node: edge.node }));
@@ -982,7 +983,7 @@ export function parseFacebookAds(
results.push(listingDetails); results.push(listingDetails);
} catch (error) { } catch (error) {
console.warn("Failed to parse Facebook ad:", error); logger.warn("Failed to parse Facebook ad:", error);
} }
} }
@@ -1083,7 +1084,7 @@ export function parseFacebookItem(
return listingDetails; return listingDetails;
} catch (error) { } catch (error) {
console.warn(`Failed to parse Facebook item ${item.id}:`, error); logger.warn(`Failed to parse Facebook item ${item.id}:`, error);
return null; return null;
} }
} }
@@ -1148,8 +1149,8 @@ export default async function fetchFacebookItems(
// Facebook marketplace URL structure // Facebook marketplace URL structure
const searchUrl = `https://www.facebook.com/marketplace/${LOCATION}/search?query=${encodedQuery}&sortBy=creation_time_descend&exact=false`; const searchUrl = `https://www.facebook.com/marketplace/${LOCATION}/search?query=${encodedQuery}&sortBy=creation_time_descend&exact=false`;
console.log(`Fetching Facebook marketplace: ${searchUrl}`); logger.log(`Fetching Facebook marketplace: ${searchUrl}`);
console.log(`Using ${cookies.length} cookies for authentication`); logger.log(`Using ${cookies.length} cookies for authentication`);
let searchHtml: string; let searchHtml: string;
let searchResponseUrl = searchUrl; let searchResponseUrl = searchUrl;
@@ -1158,7 +1159,7 @@ export default async function fetchFacebookItems(
maxRetries: 3, maxRetries: 3,
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
console.log( logger.log(
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`, `\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
); );
} }
@@ -1169,11 +1170,11 @@ export default async function fetchFacebookItems(
searchResponseUrl = response.responseUrl; searchResponseUrl = response.responseUrl;
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
console.warn( logger.warn(
`\nFacebook marketplace access failed (${err.status}): ${err.message}`, `\nFacebook marketplace access failed (${err.status}): ${err.message}`,
); );
if (err.status === 400 || err.status === 401 || err.status === 403) { if (err.status === 400 || err.status === 401 || err.status === 403) {
console.warn( logger.warn(
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.", "This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
); );
} }
@@ -1187,19 +1188,19 @@ export default async function fetchFacebookItems(
searchResponseUrl, searchResponseUrl,
); );
if (classification.authGated) { if (classification.authGated) {
console.warn( logger.warn(
"Facebook marketplace search redirected to login. Cookies may be expired.", "Facebook marketplace search redirected to login. Cookies may be expired.",
); );
return finalizeResults([]); return finalizeResults([]);
} }
if (classification.unavailable) { if (classification.unavailable) {
console.warn("Facebook marketplace search returned an unavailable route."); logger.warn("Facebook marketplace search returned an unavailable route.");
return finalizeResults([]); return finalizeResults([]);
} }
if (classification.kind !== "search") { if (classification.kind !== "search") {
console.warn( logger.warn(
`Facebook marketplace search returned unexpected route kind: ${classification.kind}.`, `Facebook marketplace search returned unexpected route kind: ${classification.kind}.`,
); );
return finalizeResults([]); return finalizeResults([]);
@@ -1207,11 +1208,11 @@ export default async function fetchFacebookItems(
const ads = extractFacebookMarketplaceData(searchHtml); const ads = extractFacebookMarketplaceData(searchHtml);
if (!ads || ads.length === 0) { if (!ads || ads.length === 0) {
console.warn("No ads parsed from Facebook marketplace page."); logger.warn("No ads parsed from Facebook marketplace page.");
return finalizeResults([]); return finalizeResults([]);
} }
console.log(`\nFound ${ads.length} raw ads. Processing...`); logger.log(`\nFound ${ads.length} raw ads. Processing...`);
const isTTY = process.stdout?.isTTY ?? false; const isTTY = process.stdout?.isTTY ?? false;
const progressBar = isTTY const progressBar = isTTY
@@ -1232,7 +1233,7 @@ export default async function fetchFacebookItems(
progressBar?.update(totalProgress); progressBar?.update(totalProgress);
progressBar?.stop(); progressBar?.stop();
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`); logger.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
return finalizeResults(pricedItems); return finalizeResults(pricedItems);
} }
@@ -1254,7 +1255,7 @@ export async function fetchFacebookItem(
const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`; const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`;
console.log(`Fetching Facebook marketplace item: ${itemUrl}`); logger.log(`Fetching Facebook marketplace item: ${itemUrl}`);
let itemHtml: string; let itemHtml: string;
let itemResponseUrl = itemUrl; let itemResponseUrl = itemUrl;
@@ -1262,7 +1263,7 @@ export async function fetchFacebookItem(
const response = await fetchHtml(itemUrl, 1000, { const response = await fetchHtml(itemUrl, 1000, {
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
console.log( logger.log(
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`, `\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
); );
} }
@@ -1273,7 +1274,7 @@ export async function fetchFacebookItem(
itemResponseUrl = response.responseUrl; itemResponseUrl = response.responseUrl;
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
console.warn( logger.warn(
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`, `\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
); );
@@ -1282,29 +1283,29 @@ export async function fetchFacebookItem(
case 400: case 400:
case 401: case 401:
case 403: case 403:
console.warn( logger.warn(
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.", "Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
); );
break; break;
case 404: case 404:
console.warn( logger.warn(
"Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.", "Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.",
); );
break; break;
case 429: case 429:
console.warn( logger.warn(
"Rate limited: Too many requests. Facebook is blocking access temporarily.", "Rate limited: Too many requests. Facebook is blocking access temporarily.",
); );
break; break;
case 500: case 500:
case 502: case 502:
case 503: case 503:
console.warn( logger.warn(
"Facebook server error: Marketplace may be temporarily unavailable.", "Facebook server error: Marketplace may be temporarily unavailable.",
); );
break; break;
default: default:
console.warn(`Unexpected error status: ${err.status}`); logger.warn(`Unexpected error status: ${err.status}`);
} }
return null; return null;
} }
@@ -1315,7 +1316,7 @@ export async function fetchFacebookItem(
if (classification.authGated) { if (classification.authGated) {
logExtractionMetrics(false, itemId); logExtractionMetrics(false, itemId);
console.warn( logger.warn(
`Authentication failed for item ${itemId}. Cookies may be expired.`, `Authentication failed for item ${itemId}. Cookies may be expired.`,
); );
return null; return null;
@@ -1323,7 +1324,7 @@ export async function fetchFacebookItem(
if (itemResponseUrl.includes("unavailable_product=1")) { if (itemResponseUrl.includes("unavailable_product=1")) {
logExtractionMetrics(false, itemId); logExtractionMetrics(false, itemId);
console.warn( logger.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`, `Item ${itemId} appears to be sold or removed from marketplace.`,
); );
return null; return null;
@@ -1333,7 +1334,7 @@ export async function fetchFacebookItem(
if (classification.unavailable && !itemData) { if (classification.unavailable && !itemData) {
logExtractionMetrics(false, itemId); logExtractionMetrics(false, itemId);
console.warn( logger.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`, `Item ${itemId} appears to be sold or removed from marketplace.`,
); );
return null; return null;
@@ -1341,7 +1342,7 @@ export async function fetchFacebookItem(
if (classification.kind !== "item" && !itemData) { if (classification.kind !== "item" && !itemData) {
logExtractionMetrics(false, itemId); logExtractionMetrics(false, itemId);
console.warn( logger.warn(
`Item ${itemId} returned unexpected route kind: ${classification.kind}.`, `Item ${itemId} returned unexpected route kind: ${classification.kind}.`,
); );
return null; return null;
@@ -1351,38 +1352,38 @@ export async function fetchFacebookItem(
logExtractionMetrics(false, itemId); logExtractionMetrics(false, itemId);
if (itemHtml.includes("This item has been sold")) { if (itemHtml.includes("This item has been sold")) {
console.warn( logger.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`, `Item ${itemId} appears to be sold or removed from marketplace.`,
); );
return null; return null;
} }
console.warn( logger.warn(
`No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`, `No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`,
); );
console.warn(" - The listing was removed or sold"); logger.warn(" - The listing was removed or sold");
console.warn(" - Authentication issues"); logger.warn(" - Authentication issues");
console.warn(" - Facebook changed their API structure"); logger.warn(" - Facebook changed their API structure");
console.warn(" - Network or parsing issues"); logger.warn(" - Network or parsing issues");
return null; return null;
} }
logExtractionMetrics(true, itemId); logExtractionMetrics(true, itemId);
console.log(`Successfully extracted data for item ${itemId}`); logger.log(`Successfully extracted data for item ${itemId}`);
const parsedItem = parseFacebookItem(itemData); const parsedItem = parseFacebookItem(itemData);
if (!parsedItem) { if (!parsedItem) {
console.warn(`Failed to parse item ${itemId}: Invalid data structure`); logger.warn(`Failed to parse item ${itemId}: Invalid data structure`);
return null; return null;
} }
// Check for sold/removed status in the parsed data with proper precedence // Check for sold/removed status in the parsed data with proper precedence
if (itemData.is_sold) { if (itemData.is_sold) {
console.warn(`Item ${itemId} is marked as sold in the marketplace.`); logger.warn(`Item ${itemId} is marked as sold in the marketplace.`);
// Still return the data but mark it as sold // Still return the data but mark it as sold
parsedItem.listingStatus = "SOLD"; parsedItem.listingStatus = "SOLD";
} else if (!itemData.is_live) { } else if (!itemData.is_live) {
console.warn(`Item ${itemId} is not live/active in the marketplace.`); logger.warn(`Item ${itemId} is not live/active in the marketplace.`);
parsedItem.listingStatus = itemData.is_hidden parsedItem.listingStatus = itemData.is_hidden
? "HIDDEN" ? "HIDDEN"
: itemData.is_pending : itemData.is_pending

View File

@@ -21,6 +21,7 @@ import {
RateLimitError, RateLimitError,
ValidationError, ValidationError,
} from "../utils/http"; } from "../utils/http";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable"; import { classifyUnstableListings } from "../utils/unstable";
// Kijiji cookie configuration // Kijiji cookie configuration
@@ -489,7 +490,7 @@ async function fetchSellerDetails(
}; };
} catch (err) { } catch (err) {
// Silently fail for GraphQL errors - not critical for basic functionality // Silently fail for GraphQL errors - not critical for basic functionality
console.warn( logger.warn(
`Failed to fetch seller details for ${posterId}:`, `Failed to fetch seller details for ${posterId}:`,
err instanceof Error ? err.message : String(err), err instanceof Error ? err.message : String(err),
); );
@@ -723,7 +724,7 @@ export async function parseDetailedListing(
}; };
} catch { } catch {
// Silently fail - GraphQL data is optional // Silently fail - GraphQL data is optional
console.warn( logger.warn(
`Failed to fetch additional seller data for ${posterInfo.posterId}`, `Failed to fetch additional seller data for ${posterInfo.posterId}`,
); );
} }
@@ -860,11 +861,11 @@ export default async function fetchKijijiItems(
BASE_URL, BASE_URL,
); );
console.log(`Fetching search page ${page}: ${searchUrl}`); logger.log(`Fetching search page ${page}: ${searchUrl}`);
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, { const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
console.log( logger.log(
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`, `\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
); );
} }
@@ -874,9 +875,7 @@ export default async function fetchKijijiItems(
const searchResults = parseSearch(searchHtml, BASE_URL); const searchResults = parseSearch(searchHtml, BASE_URL);
if (searchResults.length === 0) { if (searchResults.length === 0) {
console.log( logger.log(`No more results found on page ${page}. Stopping pagination.`);
`No more results found on page ${page}. Stopping pagination.`,
);
break; break;
} }
@@ -889,7 +888,7 @@ export default async function fetchKijijiItems(
seenUrls.add(link); seenUrls.add(link);
} }
console.log( logger.log(
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`, `\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
); );
@@ -920,7 +919,7 @@ export default async function fetchKijijiItems(
// Staggered starts keep request pacing within REQUESTS_PER_SECOND. // Staggered starts keep request pacing within REQUESTS_PER_SECOND.
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
console.log( logger.log(
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`, `\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
); );
} }
@@ -948,7 +947,7 @@ export default async function fetchKijijiItems(
currentProgress++; currentProgress++;
progressBar?.update(currentProgress); progressBar?.update(currentProgress);
if (!progressBar) { if (!progressBar) {
console.log(`Progress: ${currentProgress}/${totalProgress}`); logger.log(`Progress: ${currentProgress}/${totalProgress}`);
} }
} }
}); });
@@ -977,7 +976,7 @@ export default async function fetchKijijiItems(
matchesPriceFilters(listing, finalSearchOptions), matchesPriceFilters(listing, finalSearchOptions),
); );
console.log(`\nParsed ${filteredListings.length} detailed listings.`); logger.log(`\nParsed ${filteredListings.length} detailed listings.`);
return finalizeResults(filteredListings); return finalizeResults(filteredListings);
} }

View File

@@ -2,6 +2,8 @@
* Shared cookie handling utilities for marketplace scrapers * Shared cookie handling utilities for marketplace scrapers
*/ */
import { logger } from "./logger";
export interface Cookie { export interface Cookie {
name: string; name: string;
value: string; value: string;
@@ -116,7 +118,7 @@ export async function ensureCookies(
const cookies = parseCookieString(envValue ?? "", config.domain); const cookies = parseCookieString(envValue ?? "", config.domain);
if (cookies.length > 0) { if (cookies.length > 0) {
console.log( logger.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`, `Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
); );
return cookies; return cookies;

View File

@@ -4,5 +4,7 @@
* @returns A promise that resolves after the specified delay * @returns A promise that resolves after the specified delay
*/ */
export function delay(ms: number): Promise<void> { export function delay(ms: number): Promise<void> {
if (process.env.NODE_ENV === "test") return Promise.resolve();
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }

View File

@@ -1,3 +1,5 @@
import { delay } from "./delay";
/** Custom error class for HTTP-related failures */ /** Custom error class for HTTP-related failures */
export class HttpError extends Error { export class HttpError extends Error {
override name = "HttpError"; override name = "HttpError";
@@ -167,7 +169,7 @@ export async function fetchHtml(
const html = await res.text(); const html = await res.text();
// Respect per-request delay to maintain rate limiting // Respect per-request delay to maintain rate limiting
await new Promise((resolve) => setTimeout(resolve, delayMs)); await delay(delayMs);
return html; return html;
} catch (err) { } catch (err) {
// Re-throw known errors // Re-throw known errors

View File

@@ -0,0 +1,10 @@
const isTest = () => process.env.NODE_ENV === "test";
export const logger = {
log: (...args: Parameters<typeof console.log>) => {
if (!isTest()) console.log(...args);
},
warn: (...args: Parameters<typeof console.warn>) => {
if (!isTest()) console.warn(...args);
},
};

View File

@@ -0,0 +1,24 @@
import { afterEach, describe, expect, mock, test } from "bun:test";
import { delay } from "../src/utils/delay";
describe("delay", () => {
const originalNodeEnv = process.env.NODE_ENV;
const originalSetTimeout = globalThis.setTimeout;
afterEach(() => {
process.env.NODE_ENV = originalNodeEnv;
globalThis.setTimeout = originalSetTimeout;
});
test("does not schedule throttle timers during tests", async () => {
process.env.NODE_ENV = "test";
const setTimeoutMock = mock(() => {
throw new Error("setTimeout should not be called during tests");
});
globalThis.setTimeout = setTimeoutMock as unknown as typeof setTimeout;
await delay(1000);
expect(setTimeoutMock).not.toHaveBeenCalled();
});
});

View File

@@ -44,9 +44,6 @@ describe("eBay Scraper Cookie Handling", () => {
}); });
test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => { test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => {
const warnMock = mock(() => {});
console.warn = warnMock;
await fetchEbayItems("laptop", 1000); await fetchEbayItems("laptop", 1000);
expect(global.fetch).toHaveBeenCalledTimes(1); expect(global.fetch).toHaveBeenCalledTimes(1);
@@ -61,9 +58,6 @@ describe("eBay Scraper Cookie Handling", () => {
const headers = (init as RequestInit).headers as Record<string, string>; const headers = (init as RequestInit).headers as Record<string, string>;
expect(headers.Cookie).toBeUndefined(); expect(headers.Cookie).toBeUndefined();
expect(warnMock).toHaveBeenCalledWith(
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
);
}); });
test("keeps relative item links on the ebay.ca host", async () => { test("keeps relative item links on the ebay.ca host", async () => {

View File

@@ -177,10 +177,6 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
}); });
test("should handle authentication errors", async () => { test("should handle authentication errors", async () => {
const originalWarn = console.warn;
const warnMock = mock(() => {});
console.warn = warnMock;
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: false, ok: false,
@@ -192,16 +188,9 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
}), }),
) as unknown as typeof fetch; ) as unknown as typeof fetch;
try { const result = await fetchFacebookItem("123");
const result = await fetchFacebookItem("123"); expect(result).toBeNull();
expect(result).toBeNull(); expect(global.fetch).toHaveBeenCalledTimes(1);
expect(global.fetch).toHaveBeenCalledTimes(1);
expect(warnMock).toHaveBeenCalledWith(
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
);
} finally {
console.warn = originalWarn;
}
}); });
test("should handle item not found", async () => { test("should handle item not found", async () => {
@@ -1708,10 +1697,6 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
}); });
test("should handle malformed ads gracefully", () => { test("should handle malformed ads gracefully", () => {
const originalWarn = console.warn;
const warnMock = mock(() => {});
console.warn = warnMock;
const ads = [ const ads = [
{ {
node: { node: {
@@ -1739,9 +1724,6 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
); );
expect(results).toHaveLength(1); expect(results).toHaveLength(1);
expect(results[0]?.title).toBe("Valid Ad"); expect(results[0]?.title).toBe("Valid Ad");
expect(warnMock).toHaveBeenCalledTimes(1);
console.warn = originalWarn;
}); });
test("parses formatted fallback prices with multiple commas", () => { test("parses formatted fallback prices with multiple commas", () => {

View File

@@ -0,0 +1,41 @@
import { afterEach, describe, expect, mock, test } from "bun:test";
import { fetchHtml } from "../src/utils/http";
describe("fetchHtml", () => {
const originalFetch = global.fetch;
const originalNodeEnv = process.env.NODE_ENV;
const originalSetTimeout = globalThis.setTimeout;
const originalClearTimeout = globalThis.clearTimeout;
afterEach(() => {
global.fetch = originalFetch;
process.env.NODE_ENV = originalNodeEnv;
globalThis.setTimeout = originalSetTimeout;
globalThis.clearTimeout = originalClearTimeout;
});
test("does not schedule throttle timers during tests", async () => {
process.env.NODE_ENV = "test";
const scheduledDelays: number[] = [];
global.fetch = mock(() =>
Promise.resolve({
ok: true,
headers: { get: () => null },
text: () => Promise.resolve("<html></html>"),
}),
) as unknown as typeof fetch;
globalThis.setTimeout = mock((handler: TimerHandler, timeout?: number) => {
scheduledDelays.push(Number(timeout));
if (timeout !== 30_000 && typeof handler === "function") {
handler();
}
return 0 as unknown as ReturnType<typeof setTimeout>;
}) as unknown as typeof setTimeout;
globalThis.clearTimeout = mock(() => {}) as unknown as typeof clearTimeout;
await fetchHtml("https://example.com", 1000, { timeoutMs: 30_000 });
expect(scheduledDelays).not.toContain(1000);
});
});

View File

@@ -0,0 +1,29 @@
import { afterEach, describe, expect, mock, test } from "bun:test";
describe("logger", () => {
const originalNodeEnv = process.env.NODE_ENV;
const originalConsoleLog = console.log;
const originalConsoleWarn = console.warn;
afterEach(() => {
process.env.NODE_ENV = originalNodeEnv;
console.log = originalConsoleLog;
console.warn = originalConsoleWarn;
});
test("suppresses log and warn output during tests", async () => {
process.env.NODE_ENV = "test";
const logMock = mock(() => {});
const warnMock = mock(() => {});
console.log = logMock;
console.warn = warnMock;
const { logger } = await import("../src/utils/logger");
logger.log("hidden log");
logger.warn("hidden warn");
expect(logMock).not.toHaveBeenCalled();
expect(warnMock).not.toHaveBeenCalled();
});
});

View File

@@ -1,3 +1,4 @@
import { logger } from "./logger";
import { handleMcpRequest } from "./protocol/handler"; import { handleMcpRequest } from "./protocol/handler";
import { serverCard } from "./protocol/metadata"; import { serverCard } from "./protocol/metadata";
@@ -33,4 +34,4 @@ const server = Bun.serve({
}, },
}); });
console.log(`MCP Server running on ${server.hostname}:${server.port}`); logger.log(`MCP Server running on ${server.hostname}:${server.port}`);

View File

@@ -0,0 +1,10 @@
const isTest = () => process.env.NODE_ENV === "test";
export const logger = {
log: (...args: Parameters<typeof console.log>) => {
if (!isTest()) console.log(...args);
},
error: (...args: Parameters<typeof console.error>) => {
if (!isTest()) console.error(...args);
},
};

View File

@@ -1,3 +1,4 @@
import { logger } from "../logger";
import { tools } from "./tools"; import { tools } from "./tools";
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api"; const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
@@ -119,7 +120,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (args.unstableFilter !== undefined) if (args.unstableFilter !== undefined)
params.append("unstableFilter", args.unstableFilter.toString()); params.append("unstableFilter", args.unstableFilter.toString());
console.log( logger.log(
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`, `[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
); );
const response = await Promise.race([ const response = await Promise.race([
@@ -135,13 +136,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (!response.ok) { if (!response.ok) {
const errorText = await response.text(); const errorText = await response.text();
console.error( logger.error(
`[MCP] Kijiji API error ${response.status}: ${errorText}`, `[MCP] Kijiji API error ${response.status}: ${errorText}`,
); );
throw new Error(`API returned ${response.status}: ${errorText}`); throw new Error(`API returned ${response.status}: ${errorText}`);
} }
result = await response.json(); result = await response.json();
console.log( logger.log(
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`, `[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
); );
} else if (name === "search_facebook") { } else if (name === "search_facebook") {
@@ -160,7 +161,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (args.unstableFilter !== undefined) if (args.unstableFilter !== undefined)
params.append("unstableFilter", args.unstableFilter.toString()); params.append("unstableFilter", args.unstableFilter.toString());
console.log( logger.log(
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`, `[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
); );
const response = await Promise.race([ const response = await Promise.race([
@@ -176,13 +177,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (!response.ok) { if (!response.ok) {
const errorText = await response.text(); const errorText = await response.text();
console.error( logger.error(
`[MCP] Facebook API error ${response.status}: ${errorText}`, `[MCP] Facebook API error ${response.status}: ${errorText}`,
); );
throw new Error(`API returned ${response.status}: ${errorText}`); throw new Error(`API returned ${response.status}: ${errorText}`);
} }
result = await response.json(); result = await response.json();
console.log( logger.log(
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`, `[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
); );
} else if (name === "search_ebay") { } else if (name === "search_ebay") {
@@ -214,7 +215,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (args.unstableFilter !== undefined) if (args.unstableFilter !== undefined)
params.append("unstableFilter", args.unstableFilter.toString()); params.append("unstableFilter", args.unstableFilter.toString());
console.log( logger.log(
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`, `[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
); );
const response = await Promise.race([ const response = await Promise.race([
@@ -230,13 +231,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
if (!response.ok) { if (!response.ok) {
const errorText = await response.text(); const errorText = await response.text();
console.error( logger.error(
`[MCP] eBay API error ${response.status}: ${errorText}`, `[MCP] eBay API error ${response.status}: ${errorText}`,
); );
throw new Error(`API returned ${response.status}: ${errorText}`); throw new Error(`API returned ${response.status}: ${errorText}`);
} }
result = await response.json(); result = await response.json();
console.log( logger.log(
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`, `[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
); );
} else { } else {