Compare commits
4 Commits
df2635d92f
...
2a5701aeb9
| Author | SHA1 | Date | |
|---|---|---|---|
| 2a5701aeb9 | |||
| c6c44a0914 | |||
| 3fe5fdb63f | |||
| 7966073bf8 |
@@ -1,3 +1,4 @@
|
|||||||
|
import { logger } from "./logger";
|
||||||
import { ebayRoute } from "./routes/ebay";
|
import { ebayRoute } from "./routes/ebay";
|
||||||
import { facebookRoute } from "./routes/facebook";
|
import { facebookRoute } from "./routes/facebook";
|
||||||
import { kijijiRoute } from "./routes/kijiji";
|
import { kijijiRoute } from "./routes/kijiji";
|
||||||
@@ -27,4 +28,4 @@ const server = Bun.serve({
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`API Server running on ${server.hostname}:${server.port}`);
|
logger.log(`API Server running on ${server.hostname}:${server.port}`);
|
||||||
|
|||||||
10
packages/api-server/src/logger.ts
Normal file
10
packages/api-server/src/logger.ts
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
const isTest = () => process.env.NODE_ENV === "test";
|
||||||
|
|
||||||
|
export const logger = {
|
||||||
|
log: (...args: Parameters<typeof console.log>) => {
|
||||||
|
if (!isTest()) console.log(...args);
|
||||||
|
},
|
||||||
|
error: (...args: Parameters<typeof console.error>) => {
|
||||||
|
if (!isTest()) console.error(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { fetchEbayItems } from "@marketplace-scrapers/core";
|
import { fetchEbayItems } from "@marketplace-scrapers/core";
|
||||||
|
import { logger } from "../logger";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}
|
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}
|
||||||
@@ -21,7 +22,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
|
|
||||||
const minPriceParam = reqUrl.searchParams.get("minPrice");
|
const minPriceParam = reqUrl.searchParams.get("minPrice");
|
||||||
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
|
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
|
||||||
if (minPriceParam && (Number.isNaN(minPrice) || minPrice < 0)) {
|
if (minPriceParam && (Number.isNaN(minPrice) || (minPrice ?? 0) < 0)) {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Invalid minPrice parameter" },
|
{ message: "Invalid minPrice parameter" },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
@@ -29,7 +30,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
}
|
}
|
||||||
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
|
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
|
||||||
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
|
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
|
||||||
if (maxPriceParam && (Number.isNaN(maxPrice) || maxPrice < 0)) {
|
if (maxPriceParam && (Number.isNaN(maxPrice) || (maxPrice ?? 0) < 0)) {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Invalid maxPrice parameter" },
|
{ message: "Invalid maxPrice parameter" },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
@@ -49,7 +50,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
|
|
||||||
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
|
||||||
if (maxItemsParam && (Number.isNaN(maxItems) || maxItems < 0)) {
|
if (maxItemsParam && (Number.isNaN(maxItems) || (maxItems ?? 0) < 0)) {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Invalid maxItems parameter" },
|
{ message: "Invalid maxItems parameter" },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
@@ -67,25 +68,30 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
canadaOnly,
|
canadaOnly,
|
||||||
maxItems,
|
maxItems,
|
||||||
};
|
};
|
||||||
const items = hideUnstableResults
|
if (hideUnstableResults) {
|
||||||
? await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
const items = await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
||||||
hideUnstableResults: true,
|
hideUnstableResults: true,
|
||||||
})
|
});
|
||||||
: await fetchEbayItems(SEARCH_QUERY, 1, opts);
|
if (items.results.length === 0 && items.unstableResults.length === 0) {
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
}
|
||||||
|
|
||||||
const isEmpty = hideUnstableResults
|
const items = await fetchEbayItems(SEARCH_QUERY, 1, opts);
|
||||||
? items.results.length === 0 && items.unstableResults.length === 0
|
const isEmpty = !items || items.length === 0;
|
||||||
: !items || items.length === 0;
|
|
||||||
|
|
||||||
if (isEmpty)
|
if (isEmpty)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
|
|
||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("eBay scraping error:", error);
|
logger.error("eBay scraping error:", error);
|
||||||
const errorMessage =
|
const errorMessage =
|
||||||
error instanceof Error ? error.message : "Unknown error occurred";
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
return Response.json({ message: errorMessage }, { status: 400 });
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { fetchFacebookItems } from "@marketplace-scrapers/core";
|
import { fetchFacebookItems } from "@marketplace-scrapers/core";
|
||||||
|
import { logger } from "../logger";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* GET /api/facebook?q={query}&location={location}
|
* GET /api/facebook?q={query}&location={location}
|
||||||
@@ -30,24 +31,34 @@ export async function facebookRoute(req: Request): Promise<Response> {
|
|||||||
reqUrl.searchParams.get("unstableFilter") === "true";
|
reqUrl.searchParams.get("unstableFilter") === "true";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const items = hideUnstableResults
|
if (hideUnstableResults) {
|
||||||
? await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems, {
|
const items = await fetchFacebookItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
1,
|
||||||
|
LOCATION,
|
||||||
|
maxItems,
|
||||||
|
{
|
||||||
hideUnstableResults: true,
|
hideUnstableResults: true,
|
||||||
})
|
},
|
||||||
: await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
|
);
|
||||||
|
if (items.results.length === 0 && items.unstableResults.length === 0) {
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
}
|
||||||
|
|
||||||
const isEmpty = hideUnstableResults
|
const items = await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
|
||||||
? items.results.length === 0 && items.unstableResults.length === 0
|
if (!items || items.length === 0)
|
||||||
: !items || items.length === 0;
|
|
||||||
|
|
||||||
if (isEmpty)
|
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Facebook scraping error:", error);
|
logger.error("Facebook scraping error:", error);
|
||||||
const errorMessage =
|
const errorMessage =
|
||||||
error instanceof Error ? error.message : "Unknown error occurred";
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
return Response.json({ message: errorMessage }, { status: 400 });
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { fetchKijijiItems } from "@marketplace-scrapers/core";
|
import { fetchKijijiItems } from "@marketplace-scrapers/core";
|
||||||
|
import { logger } from "../logger";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* GET /api/kijiji?q={query}
|
* GET /api/kijiji?q={query}
|
||||||
@@ -27,7 +28,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
|||||||
}
|
}
|
||||||
const priceMinParam = reqUrl.searchParams.get("priceMin");
|
const priceMinParam = reqUrl.searchParams.get("priceMin");
|
||||||
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
|
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
|
||||||
if (priceMinParam && (Number.isNaN(priceMin) || priceMin < 0)) {
|
if (priceMinParam && (Number.isNaN(priceMin) || (priceMin ?? 0) < 0)) {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Invalid priceMin parameter" },
|
{ message: "Invalid priceMin parameter" },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
@@ -35,7 +36,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
|||||||
}
|
}
|
||||||
const priceMaxParam = reqUrl.searchParams.get("priceMax");
|
const priceMaxParam = reqUrl.searchParams.get("priceMax");
|
||||||
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
|
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
|
||||||
if (priceMaxParam && (Number.isNaN(priceMax) || priceMax < 0)) {
|
if (priceMaxParam && (Number.isNaN(priceMax) || (priceMax ?? 0) < 0)) {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Invalid priceMax parameter" },
|
{ message: "Invalid priceMax parameter" },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
@@ -65,35 +66,39 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const items = hideUnstableResults
|
if (hideUnstableResults) {
|
||||||
? await fetchKijijiItems(
|
const items = await fetchKijijiItems(
|
||||||
SEARCH_QUERY,
|
SEARCH_QUERY,
|
||||||
4, // 4 requests per second for faster scraping
|
4, // 4 requests per second for faster scraping
|
||||||
"https://www.kijiji.ca",
|
"https://www.kijiji.ca",
|
||||||
searchOptions,
|
searchOptions,
|
||||||
{},
|
{},
|
||||||
{ hideUnstableResults: true },
|
{ hideUnstableResults: true },
|
||||||
)
|
);
|
||||||
: await fetchKijijiItems(
|
if (items.results.length === 0 && items.unstableResults.length === 0) {
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const items = await fetchKijijiItems(
|
||||||
SEARCH_QUERY,
|
SEARCH_QUERY,
|
||||||
4, // 4 requests per second for faster scraping
|
4, // 4 requests per second for faster scraping
|
||||||
"https://www.kijiji.ca",
|
"https://www.kijiji.ca",
|
||||||
searchOptions,
|
searchOptions,
|
||||||
{},
|
{},
|
||||||
);
|
);
|
||||||
|
if (!items || items.length === 0)
|
||||||
const isEmpty = hideUnstableResults
|
|
||||||
? items.results.length === 0 && items.unstableResults.length === 0
|
|
||||||
: !items || items.length === 0;
|
|
||||||
|
|
||||||
if (isEmpty)
|
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Kijiji scraping error:", error);
|
logger.error("Kijiji scraping error:", error);
|
||||||
const errorMessage =
|
const errorMessage =
|
||||||
error instanceof Error ? error.message : "Unknown error occurred";
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
return Response.json({ message: errorMessage }, { status: 400 });
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
|||||||
@@ -1,8 +1,23 @@
|
|||||||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
|
||||||
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
|
const fetchFacebookItems = mock(
|
||||||
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
|
(): Promise<
|
||||||
const fetchKijijiItems = mock(() => Promise.resolve([{ title: "item" }]));
|
| { title: string }[]
|
||||||
|
| { results: { title: string }[]; unstableResults: { title: string }[] }
|
||||||
|
> => Promise.resolve([{ title: "item" }]),
|
||||||
|
);
|
||||||
|
const fetchEbayItems = mock(
|
||||||
|
(): Promise<
|
||||||
|
| { title: string }[]
|
||||||
|
| { results: { title: string }[]; unstableResults: { title: string }[] }
|
||||||
|
> => Promise.resolve([{ title: "item" }]),
|
||||||
|
);
|
||||||
|
const fetchKijijiItems = mock(
|
||||||
|
(): Promise<
|
||||||
|
| { title: string }[]
|
||||||
|
| { results: { title: string }[]; unstableResults: { title: string }[] }
|
||||||
|
> => Promise.resolve([{ title: "item" }]),
|
||||||
|
);
|
||||||
|
|
||||||
mock.module("@marketplace-scrapers/core", () => ({
|
mock.module("@marketplace-scrapers/core", () => ({
|
||||||
fetchFacebookItems,
|
fetchFacebookItems,
|
||||||
@@ -13,16 +28,19 @@ mock.module("@marketplace-scrapers/core", () => ({
|
|||||||
describe("API routes", () => {
|
describe("API routes", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
fetchFacebookItems.mockReset();
|
fetchFacebookItems.mockReset();
|
||||||
fetchFacebookItems.mockImplementation(() =>
|
fetchFacebookItems.mockImplementation(
|
||||||
Promise.resolve([{ title: "item" }]),
|
() =>
|
||||||
|
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
|
||||||
);
|
);
|
||||||
fetchEbayItems.mockReset();
|
fetchEbayItems.mockReset();
|
||||||
fetchEbayItems.mockImplementation(() =>
|
fetchEbayItems.mockImplementation(
|
||||||
Promise.resolve([{ title: "item" }]),
|
() =>
|
||||||
|
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
|
||||||
);
|
);
|
||||||
fetchKijijiItems.mockReset();
|
fetchKijijiItems.mockReset();
|
||||||
fetchKijijiItems.mockImplementation(() =>
|
fetchKijijiItems.mockImplementation(
|
||||||
Promise.resolve([{ title: "item" }]),
|
() =>
|
||||||
|
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import {
|
|||||||
formatCookiesForHeader,
|
formatCookiesForHeader,
|
||||||
} from "../utils/cookies";
|
} from "../utils/cookies";
|
||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
|
import { logger } from "../utils/logger";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
// eBay cookie configuration
|
// eBay cookie configuration
|
||||||
@@ -274,7 +275,7 @@ function parseEbayListings(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Filter to only elements that actually contain prices (not labels)
|
// Filter to only elements that actually contain prices (not labels)
|
||||||
const actualPrices: HTMLElement[] = [];
|
const actualPrices: Element[] = [];
|
||||||
for (const el of allPriceElements) {
|
for (const el of allPriceElements) {
|
||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
|
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
|
||||||
@@ -301,11 +302,10 @@ function parseEbayListings(
|
|||||||
|
|
||||||
if (nonStrikethroughPrices.length > 0) {
|
if (nonStrikethroughPrices.length > 0) {
|
||||||
// Use the first non-strikethrough price (sale price)
|
// Use the first non-strikethrough price (sale price)
|
||||||
priceElement = nonStrikethroughPrices[0];
|
priceElement = nonStrikethroughPrices[0] ?? null;
|
||||||
} else {
|
} else {
|
||||||
// Fallback: use the last price (likely the most current)
|
// Fallback: use the last price (likely the most current)
|
||||||
const lastPrice = actualPrices[actualPrices.length - 1];
|
priceElement = actualPrices[actualPrices.length - 1] ?? null;
|
||||||
priceElement = lastPrice;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -355,7 +355,7 @@ function parseEbayListings(
|
|||||||
results.push(listing);
|
results.push(listing);
|
||||||
seenUrls.add(canonicalUrl);
|
seenUrls.add(canonicalUrl);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn(`Error parsing eBay listing: ${err}`);
|
logger.warn(`Error parsing eBay listing: ${err}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -372,7 +372,7 @@ async function loadEbayCookies(): Promise<string | undefined> {
|
|||||||
const cookies = await ensureCookies(EBAY_COOKIE_CONFIG);
|
const cookies = await ensureCookies(EBAY_COOKIE_CONFIG);
|
||||||
return formatCookiesForHeader(cookies, "www.ebay.ca");
|
return formatCookiesForHeader(cookies, "www.ebay.ca");
|
||||||
} catch {
|
} catch {
|
||||||
console.warn(
|
logger.warn(
|
||||||
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
|
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
|
||||||
);
|
);
|
||||||
return undefined;
|
return undefined;
|
||||||
@@ -475,7 +475,7 @@ export default async function fetchEbayItems(
|
|||||||
|
|
||||||
const DELAY_MS = Math.max(1, Math.floor(1000 / requestsPerSecond));
|
const DELAY_MS = Math.max(1, Math.floor(1000 / requestsPerSecond));
|
||||||
|
|
||||||
console.log(`Fetching eBay search: ${searchUrl}`);
|
logger.log(`Fetching eBay search: ${searchUrl}`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Use custom headers modeled after real browser requests to bypass bot detection
|
// Use custom headers modeled after real browser requests to bypass bot detection
|
||||||
@@ -517,7 +517,7 @@ export default async function fetchEbayItems(
|
|||||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
||||||
await delay(DELAY_MS);
|
await delay(DELAY_MS);
|
||||||
|
|
||||||
console.log(`\nParsing eBay listings...`);
|
logger.log(`\nParsing eBay listings...`);
|
||||||
|
|
||||||
const listings = parseEbayListings(
|
const listings = parseEbayListings(
|
||||||
searchHtml,
|
searchHtml,
|
||||||
@@ -534,7 +534,7 @@ export default async function fetchEbayItems(
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
logger.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||||
return finalizeResults(filteredListings);
|
return finalizeResults(filteredListings);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
import { formatCentsToCurrency } from "../utils/format";
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
import { isRecord } from "../utils/http";
|
import { isRecord } from "../utils/http";
|
||||||
|
import { logger } from "../utils/logger";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -86,7 +87,7 @@ interface FacebookMarketplaceItem {
|
|||||||
__typename: "GroupCommerceProductItem";
|
__typename: "GroupCommerceProductItem";
|
||||||
|
|
||||||
// Listing content
|
// Listing content
|
||||||
marketplace_listing_title: string;
|
marketplace_listing_title?: string;
|
||||||
redacted_description?: {
|
redacted_description?: {
|
||||||
text: string;
|
text: string;
|
||||||
};
|
};
|
||||||
@@ -99,7 +100,7 @@ interface FacebookMarketplaceItem {
|
|||||||
listing_price?: {
|
listing_price?: {
|
||||||
amount: string;
|
amount: string;
|
||||||
currency: string;
|
currency: string;
|
||||||
amount_with_offset: string;
|
amount_with_offset?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Location
|
// Location
|
||||||
@@ -127,9 +128,9 @@ interface FacebookMarketplaceItem {
|
|||||||
|
|
||||||
// Seller information
|
// Seller information
|
||||||
marketplace_listing_seller?: {
|
marketplace_listing_seller?: {
|
||||||
__typename: "User";
|
__typename?: "User";
|
||||||
id: string;
|
id?: string;
|
||||||
name: string;
|
name?: string;
|
||||||
profile_picture?: {
|
profile_picture?: {
|
||||||
uri: string;
|
uri: string;
|
||||||
};
|
};
|
||||||
@@ -260,14 +261,14 @@ function logExtractionMetrics(success: boolean, itemId?: string) {
|
|||||||
successRate < 0.8 &&
|
successRate < 0.8 &&
|
||||||
!extractionStats.lastApiChangeDetected
|
!extractionStats.lastApiChangeDetected
|
||||||
) {
|
) {
|
||||||
console.warn(
|
logger.warn(
|
||||||
"Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
|
"Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
|
||||||
);
|
);
|
||||||
extractionStats.lastApiChangeDetected = new Date();
|
extractionStats.lastApiChangeDetected = new Date();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!success && itemId) {
|
if (!success && itemId) {
|
||||||
console.warn(`Facebook API extraction failed for item ${itemId}`);
|
logger.warn(`Facebook API extraction failed for item ${itemId}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -820,18 +821,18 @@ export function extractFacebookMarketplaceData(
|
|||||||
if (htmlString.includes("XCometMarketplaceSearchController")) {
|
if (htmlString.includes("XCometMarketplaceSearchController")) {
|
||||||
const htmlFallback = extractFacebookMarketplaceHtmlFallback(htmlString);
|
const htmlFallback = extractFacebookMarketplaceHtmlFallback(htmlString);
|
||||||
if (htmlFallback?.length) {
|
if (htmlFallback?.length) {
|
||||||
console.log(
|
logger.log(
|
||||||
`Successfully parsed ${htmlFallback.length} Facebook marketplace listings from rendered HTML fallback`,
|
`Successfully parsed ${htmlFallback.length} Facebook marketplace listings from rendered HTML fallback`,
|
||||||
);
|
);
|
||||||
return htmlFallback;
|
return htmlFallback;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.warn("No marketplace data found in HTML response");
|
logger.warn("No marketplace data found in HTML response");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(
|
logger.log(
|
||||||
`Successfully parsed ${bestEdges.length} Facebook marketplace listings`,
|
`Successfully parsed ${bestEdges.length} Facebook marketplace listings`,
|
||||||
);
|
);
|
||||||
return bestEdges.map((edge) => ({ node: edge.node }));
|
return bestEdges.map((edge) => ({ node: edge.node }));
|
||||||
@@ -982,7 +983,7 @@ export function parseFacebookAds(
|
|||||||
|
|
||||||
results.push(listingDetails);
|
results.push(listingDetails);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("Failed to parse Facebook ad:", error);
|
logger.warn("Failed to parse Facebook ad:", error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1083,7 +1084,7 @@ export function parseFacebookItem(
|
|||||||
|
|
||||||
return listingDetails;
|
return listingDetails;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn(`Failed to parse Facebook item ${item.id}:`, error);
|
logger.warn(`Failed to parse Facebook item ${item.id}:`, error);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1148,8 +1149,8 @@ export default async function fetchFacebookItems(
|
|||||||
// Facebook marketplace URL structure
|
// Facebook marketplace URL structure
|
||||||
const searchUrl = `https://www.facebook.com/marketplace/${LOCATION}/search?query=${encodedQuery}&sortBy=creation_time_descend&exact=false`;
|
const searchUrl = `https://www.facebook.com/marketplace/${LOCATION}/search?query=${encodedQuery}&sortBy=creation_time_descend&exact=false`;
|
||||||
|
|
||||||
console.log(`Fetching Facebook marketplace: ${searchUrl}`);
|
logger.log(`Fetching Facebook marketplace: ${searchUrl}`);
|
||||||
console.log(`Using ${cookies.length} cookies for authentication`);
|
logger.log(`Using ${cookies.length} cookies for authentication`);
|
||||||
|
|
||||||
let searchHtml: string;
|
let searchHtml: string;
|
||||||
let searchResponseUrl = searchUrl;
|
let searchResponseUrl = searchUrl;
|
||||||
@@ -1158,7 +1159,7 @@ export default async function fetchFacebookItems(
|
|||||||
maxRetries: 3,
|
maxRetries: 3,
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(
|
logger.log(
|
||||||
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1169,11 +1170,11 @@ export default async function fetchFacebookItems(
|
|||||||
searchResponseUrl = response.responseUrl;
|
searchResponseUrl = response.responseUrl;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
console.warn(
|
logger.warn(
|
||||||
`\nFacebook marketplace access failed (${err.status}): ${err.message}`,
|
`\nFacebook marketplace access failed (${err.status}): ${err.message}`,
|
||||||
);
|
);
|
||||||
if (err.status === 400 || err.status === 401 || err.status === 403) {
|
if (err.status === 400 || err.status === 401 || err.status === 403) {
|
||||||
console.warn(
|
logger.warn(
|
||||||
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1187,19 +1188,19 @@ export default async function fetchFacebookItems(
|
|||||||
searchResponseUrl,
|
searchResponseUrl,
|
||||||
);
|
);
|
||||||
if (classification.authGated) {
|
if (classification.authGated) {
|
||||||
console.warn(
|
logger.warn(
|
||||||
"Facebook marketplace search redirected to login. Cookies may be expired.",
|
"Facebook marketplace search redirected to login. Cookies may be expired.",
|
||||||
);
|
);
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (classification.unavailable) {
|
if (classification.unavailable) {
|
||||||
console.warn("Facebook marketplace search returned an unavailable route.");
|
logger.warn("Facebook marketplace search returned an unavailable route.");
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (classification.kind !== "search") {
|
if (classification.kind !== "search") {
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Facebook marketplace search returned unexpected route kind: ${classification.kind}.`,
|
`Facebook marketplace search returned unexpected route kind: ${classification.kind}.`,
|
||||||
);
|
);
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
@@ -1207,11 +1208,11 @@ export default async function fetchFacebookItems(
|
|||||||
|
|
||||||
const ads = extractFacebookMarketplaceData(searchHtml);
|
const ads = extractFacebookMarketplaceData(searchHtml);
|
||||||
if (!ads || ads.length === 0) {
|
if (!ads || ads.length === 0) {
|
||||||
console.warn("No ads parsed from Facebook marketplace page.");
|
logger.warn("No ads parsed from Facebook marketplace page.");
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`\nFound ${ads.length} raw ads. Processing...`);
|
logger.log(`\nFound ${ads.length} raw ads. Processing...`);
|
||||||
|
|
||||||
const isTTY = process.stdout?.isTTY ?? false;
|
const isTTY = process.stdout?.isTTY ?? false;
|
||||||
const progressBar = isTTY
|
const progressBar = isTTY
|
||||||
@@ -1232,7 +1233,7 @@ export default async function fetchFacebookItems(
|
|||||||
progressBar?.update(totalProgress);
|
progressBar?.update(totalProgress);
|
||||||
progressBar?.stop();
|
progressBar?.stop();
|
||||||
|
|
||||||
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
|
logger.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
|
||||||
return finalizeResults(pricedItems);
|
return finalizeResults(pricedItems);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1254,7 +1255,7 @@ export async function fetchFacebookItem(
|
|||||||
|
|
||||||
const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`;
|
const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`;
|
||||||
|
|
||||||
console.log(`Fetching Facebook marketplace item: ${itemUrl}`);
|
logger.log(`Fetching Facebook marketplace item: ${itemUrl}`);
|
||||||
|
|
||||||
let itemHtml: string;
|
let itemHtml: string;
|
||||||
let itemResponseUrl = itemUrl;
|
let itemResponseUrl = itemUrl;
|
||||||
@@ -1262,7 +1263,7 @@ export async function fetchFacebookItem(
|
|||||||
const response = await fetchHtml(itemUrl, 1000, {
|
const response = await fetchHtml(itemUrl, 1000, {
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(
|
logger.log(
|
||||||
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1273,7 +1274,7 @@ export async function fetchFacebookItem(
|
|||||||
itemResponseUrl = response.responseUrl;
|
itemResponseUrl = response.responseUrl;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
console.warn(
|
logger.warn(
|
||||||
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
|
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1282,29 +1283,29 @@ export async function fetchFacebookItem(
|
|||||||
case 400:
|
case 400:
|
||||||
case 401:
|
case 401:
|
||||||
case 403:
|
case 403:
|
||||||
console.warn(
|
logger.warn(
|
||||||
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
case 404:
|
case 404:
|
||||||
console.warn(
|
logger.warn(
|
||||||
"Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.",
|
"Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.",
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
case 429:
|
case 429:
|
||||||
console.warn(
|
logger.warn(
|
||||||
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
|
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
case 500:
|
case 500:
|
||||||
case 502:
|
case 502:
|
||||||
case 503:
|
case 503:
|
||||||
console.warn(
|
logger.warn(
|
||||||
"Facebook server error: Marketplace may be temporarily unavailable.",
|
"Facebook server error: Marketplace may be temporarily unavailable.",
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
console.warn(`Unexpected error status: ${err.status}`);
|
logger.warn(`Unexpected error status: ${err.status}`);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -1315,17 +1316,25 @@ export async function fetchFacebookItem(
|
|||||||
|
|
||||||
if (classification.authGated) {
|
if (classification.authGated) {
|
||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Authentication failed for item ${itemId}. Cookies may be expired.`,
|
`Authentication failed for item ${itemId}. Cookies may be expired.`,
|
||||||
);
|
);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (itemResponseUrl.includes("unavailable_product=1")) {
|
||||||
|
logExtractionMetrics(false, itemId);
|
||||||
|
logger.warn(
|
||||||
|
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const itemData = extractFacebookItemData(itemHtml);
|
const itemData = extractFacebookItemData(itemHtml);
|
||||||
|
|
||||||
if (classification.unavailable && !itemData) {
|
if (classification.unavailable && !itemData) {
|
||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||||
);
|
);
|
||||||
return null;
|
return null;
|
||||||
@@ -1333,7 +1342,7 @@ export async function fetchFacebookItem(
|
|||||||
|
|
||||||
if (classification.kind !== "item" && !itemData) {
|
if (classification.kind !== "item" && !itemData) {
|
||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Item ${itemId} returned unexpected route kind: ${classification.kind}.`,
|
`Item ${itemId} returned unexpected route kind: ${classification.kind}.`,
|
||||||
);
|
);
|
||||||
return null;
|
return null;
|
||||||
@@ -1343,38 +1352,38 @@ export async function fetchFacebookItem(
|
|||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
|
|
||||||
if (itemHtml.includes("This item has been sold")) {
|
if (itemHtml.includes("This item has been sold")) {
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||||
);
|
);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.warn(
|
logger.warn(
|
||||||
`No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`,
|
`No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`,
|
||||||
);
|
);
|
||||||
console.warn(" - The listing was removed or sold");
|
logger.warn(" - The listing was removed or sold");
|
||||||
console.warn(" - Authentication issues");
|
logger.warn(" - Authentication issues");
|
||||||
console.warn(" - Facebook changed their API structure");
|
logger.warn(" - Facebook changed their API structure");
|
||||||
console.warn(" - Network or parsing issues");
|
logger.warn(" - Network or parsing issues");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
logExtractionMetrics(true, itemId);
|
logExtractionMetrics(true, itemId);
|
||||||
console.log(`Successfully extracted data for item ${itemId}`);
|
logger.log(`Successfully extracted data for item ${itemId}`);
|
||||||
|
|
||||||
const parsedItem = parseFacebookItem(itemData);
|
const parsedItem = parseFacebookItem(itemData);
|
||||||
if (!parsedItem) {
|
if (!parsedItem) {
|
||||||
console.warn(`Failed to parse item ${itemId}: Invalid data structure`);
|
logger.warn(`Failed to parse item ${itemId}: Invalid data structure`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for sold/removed status in the parsed data with proper precedence
|
// Check for sold/removed status in the parsed data with proper precedence
|
||||||
if (itemData.is_sold) {
|
if (itemData.is_sold) {
|
||||||
console.warn(`Item ${itemId} is marked as sold in the marketplace.`);
|
logger.warn(`Item ${itemId} is marked as sold in the marketplace.`);
|
||||||
// Still return the data but mark it as sold
|
// Still return the data but mark it as sold
|
||||||
parsedItem.listingStatus = "SOLD";
|
parsedItem.listingStatus = "SOLD";
|
||||||
} else if (!itemData.is_live) {
|
} else if (!itemData.is_live) {
|
||||||
console.warn(`Item ${itemId} is not live/active in the marketplace.`);
|
logger.warn(`Item ${itemId} is not live/active in the marketplace.`);
|
||||||
parsedItem.listingStatus = itemData.is_hidden
|
parsedItem.listingStatus = itemData.is_hidden
|
||||||
? "HIDDEN"
|
? "HIDDEN"
|
||||||
: itemData.is_pending
|
: itemData.is_pending
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import {
|
|||||||
RateLimitError,
|
RateLimitError,
|
||||||
ValidationError,
|
ValidationError,
|
||||||
} from "../utils/http";
|
} from "../utils/http";
|
||||||
|
import { logger } from "../utils/logger";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
// Kijiji cookie configuration
|
// Kijiji cookie configuration
|
||||||
@@ -28,7 +29,6 @@ const KIJIJI_COOKIE_CONFIG: CookieConfig = {
|
|||||||
name: "Kijiji",
|
name: "Kijiji",
|
||||||
domain: ".kijiji.ca",
|
domain: ".kijiji.ca",
|
||||||
envVar: "KIJIJI_COOKIE",
|
envVar: "KIJIJI_COOKIE",
|
||||||
filePath: "./cookies/kijiji.json",
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
@@ -490,7 +490,7 @@ async function fetchSellerDetails(
|
|||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Silently fail for GraphQL errors - not critical for basic functionality
|
// Silently fail for GraphQL errors - not critical for basic functionality
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Failed to fetch seller details for ${posterId}:`,
|
`Failed to fetch seller details for ${posterId}:`,
|
||||||
err instanceof Error ? err.message : String(err),
|
err instanceof Error ? err.message : String(err),
|
||||||
);
|
);
|
||||||
@@ -724,7 +724,7 @@ export async function parseDetailedListing(
|
|||||||
};
|
};
|
||||||
} catch {
|
} catch {
|
||||||
// Silently fail - GraphQL data is optional
|
// Silently fail - GraphQL data is optional
|
||||||
console.warn(
|
logger.warn(
|
||||||
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
|
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -861,11 +861,11 @@ export default async function fetchKijijiItems(
|
|||||||
BASE_URL,
|
BASE_URL,
|
||||||
);
|
);
|
||||||
|
|
||||||
console.log(`Fetching search page ${page}: ${searchUrl}`);
|
logger.log(`Fetching search page ${page}: ${searchUrl}`);
|
||||||
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(
|
logger.log(
|
||||||
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -875,9 +875,7 @@ export default async function fetchKijijiItems(
|
|||||||
|
|
||||||
const searchResults = parseSearch(searchHtml, BASE_URL);
|
const searchResults = parseSearch(searchHtml, BASE_URL);
|
||||||
if (searchResults.length === 0) {
|
if (searchResults.length === 0) {
|
||||||
console.log(
|
logger.log(`No more results found on page ${page}. Stopping pagination.`);
|
||||||
`No more results found on page ${page}. Stopping pagination.`,
|
|
||||||
);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -890,7 +888,7 @@ export default async function fetchKijijiItems(
|
|||||||
seenUrls.add(link);
|
seenUrls.add(link);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(
|
logger.log(
|
||||||
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
|
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -921,7 +919,7 @@ export default async function fetchKijijiItems(
|
|||||||
// Staggered starts keep request pacing within REQUESTS_PER_SECOND.
|
// Staggered starts keep request pacing within REQUESTS_PER_SECOND.
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(
|
logger.log(
|
||||||
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -949,7 +947,7 @@ export default async function fetchKijijiItems(
|
|||||||
currentProgress++;
|
currentProgress++;
|
||||||
progressBar?.update(currentProgress);
|
progressBar?.update(currentProgress);
|
||||||
if (!progressBar) {
|
if (!progressBar) {
|
||||||
console.log(`Progress: ${currentProgress}/${totalProgress}`);
|
logger.log(`Progress: ${currentProgress}/${totalProgress}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -978,7 +976,7 @@ export default async function fetchKijijiItems(
|
|||||||
matchesPriceFilters(listing, finalSearchOptions),
|
matchesPriceFilters(listing, finalSearchOptions),
|
||||||
);
|
);
|
||||||
|
|
||||||
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
|
logger.log(`\nParsed ${filteredListings.length} detailed listings.`);
|
||||||
return finalizeResults(filteredListings);
|
return finalizeResults(filteredListings);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
* Shared cookie handling utilities for marketplace scrapers
|
* Shared cookie handling utilities for marketplace scrapers
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { logger } from "./logger";
|
||||||
|
|
||||||
export interface Cookie {
|
export interface Cookie {
|
||||||
name: string;
|
name: string;
|
||||||
value: string;
|
value: string;
|
||||||
@@ -41,9 +43,9 @@ export function parseCookieString(
|
|||||||
.split(";")
|
.split(";")
|
||||||
.map((pair) => pair.trim())
|
.map((pair) => pair.trim())
|
||||||
.filter((pair) => pair.includes("="))
|
.filter((pair) => pair.includes("="))
|
||||||
.map((pair) => {
|
.map((pair): Cookie | null => {
|
||||||
const [name, ...valueParts] = pair.split("=");
|
const [name, ...valueParts] = pair.split("=");
|
||||||
const trimmedName = name.trim();
|
const trimmedName = name?.trim();
|
||||||
const trimmedValue = valueParts.join("=").trim();
|
const trimmedValue = valueParts.join("=").trim();
|
||||||
|
|
||||||
if (!trimmedName || !trimmedValue) {
|
if (!trimmedName || !trimmedValue) {
|
||||||
@@ -98,14 +100,25 @@ export function formatCookiesForHeader(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load cookies from the configured environment variable
|
* Load cookies from the configured environment variable or explicit cookie string
|
||||||
*/
|
*/
|
||||||
export async function ensureCookies(config: CookieConfig): Promise<Cookie[]> {
|
export async function ensureCookies(
|
||||||
|
config: CookieConfig,
|
||||||
|
cookiesSource?: string,
|
||||||
|
): Promise<Cookie[]> {
|
||||||
|
// Explicit cookie string takes priority
|
||||||
|
if (cookiesSource) {
|
||||||
|
const cookies = parseCookieString(cookiesSource, config.domain);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const envValue = process.env[config.envVar];
|
const envValue = process.env[config.envVar];
|
||||||
const cookies = parseCookieString(envValue ?? "", config.domain);
|
const cookies = parseCookieString(envValue ?? "", config.domain);
|
||||||
|
|
||||||
if (cookies.length > 0) {
|
if (cookies.length > 0) {
|
||||||
console.log(
|
logger.log(
|
||||||
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
|
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
|
||||||
);
|
);
|
||||||
return cookies;
|
return cookies;
|
||||||
|
|||||||
@@ -4,5 +4,7 @@
|
|||||||
* @returns A promise that resolves after the specified delay
|
* @returns A promise that resolves after the specified delay
|
||||||
*/
|
*/
|
||||||
export function delay(ms: number): Promise<void> {
|
export function delay(ms: number): Promise<void> {
|
||||||
|
if (process.env.NODE_ENV === "test") return Promise.resolve();
|
||||||
|
|
||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,56 +1,55 @@
|
|||||||
|
import { delay } from "./delay";
|
||||||
|
|
||||||
/** Custom error class for HTTP-related failures */
|
/** Custom error class for HTTP-related failures */
|
||||||
export class HttpError extends Error {
|
export class HttpError extends Error {
|
||||||
|
override name = "HttpError";
|
||||||
constructor(
|
constructor(
|
||||||
message: string,
|
message: string,
|
||||||
public readonly statusCode: number,
|
public readonly statusCode: number,
|
||||||
public readonly url?: string,
|
public readonly url?: string,
|
||||||
) {
|
) {
|
||||||
super(message);
|
super(message);
|
||||||
this.name = "HttpError";
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Error class for network failures (timeouts, connection issues) */
|
/** Error class for network failures (timeouts, connection issues) */
|
||||||
export class NetworkError extends Error {
|
export class NetworkError extends Error {
|
||||||
|
override name = "NetworkError";
|
||||||
constructor(
|
constructor(
|
||||||
message: string,
|
message: string,
|
||||||
public readonly url: string,
|
public readonly url: string,
|
||||||
public readonly cause?: Error,
|
public override readonly cause?: Error,
|
||||||
) {
|
) {
|
||||||
super(message);
|
super(message);
|
||||||
this.name = "NetworkError";
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Error class for parsing failures */
|
/** Error class for parsing failures */
|
||||||
export class ParseError extends Error {
|
export class ParseError extends Error {
|
||||||
|
override name = "ParseError";
|
||||||
constructor(
|
constructor(
|
||||||
message: string,
|
message: string,
|
||||||
public readonly data?: unknown,
|
public readonly data?: unknown,
|
||||||
) {
|
) {
|
||||||
super(message);
|
super(message);
|
||||||
this.name = "ParseError";
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Error class for rate limiting */
|
/** Error class for rate limiting */
|
||||||
export class RateLimitError extends Error {
|
export class RateLimitError extends Error {
|
||||||
|
override name = "RateLimitError";
|
||||||
constructor(
|
constructor(
|
||||||
message: string,
|
message: string,
|
||||||
public readonly url: string,
|
public readonly url: string,
|
||||||
public readonly resetTime?: number,
|
public readonly resetTime?: number,
|
||||||
) {
|
) {
|
||||||
super(message);
|
super(message);
|
||||||
this.name = "RateLimitError";
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Error class for validation failures */
|
/** Error class for validation failures */
|
||||||
export class ValidationError extends Error {
|
export class ValidationError extends Error {
|
||||||
constructor(message: string) {
|
override name = "ValidationError";
|
||||||
super(message);
|
|
||||||
this.name = "ValidationError";
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Type guard to check if a value is a record (object) */
|
/** Type guard to check if a value is a record (object) */
|
||||||
@@ -170,7 +169,7 @@ export async function fetchHtml(
|
|||||||
const html = await res.text();
|
const html = await res.text();
|
||||||
|
|
||||||
// Respect per-request delay to maintain rate limiting
|
// Respect per-request delay to maintain rate limiting
|
||||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
await delay(delayMs);
|
||||||
return html;
|
return html;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Re-throw known errors
|
// Re-throw known errors
|
||||||
|
|||||||
10
packages/core/src/utils/logger.ts
Normal file
10
packages/core/src/utils/logger.ts
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
const isTest = () => process.env.NODE_ENV === "test";
|
||||||
|
|
||||||
|
export const logger = {
|
||||||
|
log: (...args: Parameters<typeof console.log>) => {
|
||||||
|
if (!isTest()) console.log(...args);
|
||||||
|
},
|
||||||
|
warn: (...args: Parameters<typeof console.warn>) => {
|
||||||
|
if (!isTest()) console.warn(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -1,21 +1,29 @@
|
|||||||
import type { ListingDetails, UnstableListingBuckets } from "../types/common";
|
import type { UnstableListingBuckets } from "../types/common";
|
||||||
|
|
||||||
|
interface HasListingPrice {
|
||||||
|
listingPrice?: { cents?: number } | null;
|
||||||
|
}
|
||||||
|
|
||||||
function getMedian(values: number[]): number {
|
function getMedian(values: number[]): number {
|
||||||
const middleIndex = Math.floor(values.length / 2);
|
const middleIndex = Math.floor(values.length / 2);
|
||||||
|
|
||||||
if (values.length % 2 === 0) {
|
if (values.length % 2 === 0) {
|
||||||
return (values[middleIndex - 1] + values[middleIndex]) / 2;
|
const left = values[middleIndex - 1] ?? 0;
|
||||||
|
const right = values[middleIndex] ?? 0;
|
||||||
|
return (left + right) / 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
return values[middleIndex];
|
return values[middleIndex] ?? 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function classifyUnstableListings<T extends ListingDetails>(
|
export function classifyUnstableListings<T extends HasListingPrice>(
|
||||||
listings: T[],
|
listings: T[],
|
||||||
): UnstableListingBuckets<T> {
|
): UnstableListingBuckets<T> {
|
||||||
const validPrices = listings
|
const validPrices = listings
|
||||||
.map((listing) => listing.listingPrice.cents)
|
.map((listing) => listing.listingPrice?.cents)
|
||||||
.filter((price) => Number.isFinite(price) && price > 0)
|
.filter(
|
||||||
|
(price): price is number => Number.isFinite(price) && (price ?? 0) > 0,
|
||||||
|
)
|
||||||
.sort((left, right) => left - right);
|
.sort((left, right) => left - right);
|
||||||
|
|
||||||
if (validPrices.length < 2) {
|
if (validPrices.length < 2) {
|
||||||
@@ -32,9 +40,13 @@ export function classifyUnstableListings<T extends ListingDetails>(
|
|||||||
};
|
};
|
||||||
|
|
||||||
for (const listing of listings) {
|
for (const listing of listings) {
|
||||||
const price = listing.listingPrice.cents;
|
const price = listing.listingPrice?.cents;
|
||||||
|
|
||||||
if (Number.isFinite(price) && price > 0 && price < threshold) {
|
if (
|
||||||
|
Number.isFinite(price) &&
|
||||||
|
(price ?? 0) > 0 &&
|
||||||
|
(price ?? 0) < threshold
|
||||||
|
) {
|
||||||
buckets.unstableResults.push(listing);
|
buckets.unstableResults.push(listing);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
24
packages/core/test/delay.test.ts
Normal file
24
packages/core/test/delay.test.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
import { afterEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import { delay } from "../src/utils/delay";
|
||||||
|
|
||||||
|
describe("delay", () => {
|
||||||
|
const originalNodeEnv = process.env.NODE_ENV;
|
||||||
|
const originalSetTimeout = globalThis.setTimeout;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.env.NODE_ENV = originalNodeEnv;
|
||||||
|
globalThis.setTimeout = originalSetTimeout;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not schedule throttle timers during tests", async () => {
|
||||||
|
process.env.NODE_ENV = "test";
|
||||||
|
const setTimeoutMock = mock(() => {
|
||||||
|
throw new Error("setTimeout should not be called during tests");
|
||||||
|
});
|
||||||
|
globalThis.setTimeout = setTimeoutMock as unknown as typeof setTimeout;
|
||||||
|
|
||||||
|
await delay(1000);
|
||||||
|
|
||||||
|
expect(setTimeoutMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -34,7 +34,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve("<html><body></body></html>"),
|
text: () => Promise.resolve("<html><body></body></html>"),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -44,20 +44,20 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => {
|
test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => {
|
||||||
const warnMock = mock(() => {});
|
|
||||||
console.warn = warnMock;
|
|
||||||
|
|
||||||
await fetchEbayItems("laptop", 1000);
|
await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
expect(global.fetch).toHaveBeenCalledTimes(1);
|
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
const [, init] = (global.fetch as ReturnType<typeof mock>).mock.calls[0];
|
const firstFetchCall = (global.fetch as unknown as ReturnType<typeof mock>)
|
||||||
|
.mock.calls[0];
|
||||||
|
if (!firstFetchCall) {
|
||||||
|
throw new Error("Expected fetch to be called");
|
||||||
|
}
|
||||||
|
|
||||||
|
const [, init] = firstFetchCall;
|
||||||
const headers = (init as RequestInit).headers as Record<string, string>;
|
const headers = (init as RequestInit).headers as Record<string, string>;
|
||||||
|
|
||||||
expect(headers.Cookie).toBeUndefined();
|
expect(headers.Cookie).toBeUndefined();
|
||||||
expect(warnMock).toHaveBeenCalledWith(
|
|
||||||
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps relative item links on the ebay.ca host", async () => {
|
test("keeps relative item links on the ebay.ca host", async () => {
|
||||||
@@ -75,7 +75,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -100,7 +100,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -130,7 +130,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -167,7 +167,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -199,7 +199,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -225,7 +225,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -254,7 +254,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -283,7 +283,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -317,7 +317,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("bundle", 1000, {
|
const results = await fetchEbayItems("bundle", 1000, {
|
||||||
keywords: ["bundle"],
|
keywords: ["bundle"],
|
||||||
@@ -357,7 +357,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -389,7 +389,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -421,7 +421,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000);
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
@@ -451,7 +451,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("bike", 1000);
|
const results = await fetchEbayItems("bike", 1000);
|
||||||
|
|
||||||
@@ -478,7 +478,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("microphone", 1000, {
|
const results = await fetchEbayItems("microphone", 1000, {
|
||||||
keywords: ["microphone"],
|
keywords: ["microphone"],
|
||||||
@@ -510,7 +510,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000, {
|
const results = await fetchEbayItems("laptop", 1000, {
|
||||||
minPrice: 0,
|
minPrice: 0,
|
||||||
@@ -550,7 +550,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems(
|
const results = await fetchEbayItems(
|
||||||
"laptop",
|
"laptop",
|
||||||
@@ -595,7 +595,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems("laptop", 1000, { maxItems: 2 });
|
const results = await fetchEbayItems("laptop", 1000, { maxItems: 2 });
|
||||||
|
|
||||||
@@ -633,7 +633,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
</body></html>
|
</body></html>
|
||||||
`),
|
`),
|
||||||
}),
|
}),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchEbayItems(
|
const results = await fetchEbayItems(
|
||||||
"laptop",
|
"laptop",
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
global.fetch = mock(() => {
|
global.fetch = mock(() => {
|
||||||
throw new Error("fetch should be mocked in individual tests");
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -93,8 +93,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
|
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
|
||||||
const result = parseFacebookCookieString(cookieString);
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
expect(result[0].value).toBe("123+456");
|
expect(result[0]?.value).toBe("123+456");
|
||||||
expect(result[1].value).toBe("abc=def");
|
expect(result[1]?.value).toBe("abc=def");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should filter out malformed cookies", () => {
|
test("should filter out malformed cookies", () => {
|
||||||
@@ -115,10 +115,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
const result = parseFacebookCookieString(cookieString);
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
expect(result).toHaveLength(2);
|
expect(result).toHaveLength(2);
|
||||||
expect(result[0].name).toBe("c_user");
|
expect(result[0]?.name).toBe("c_user");
|
||||||
expect(result[0].value).toBe("123");
|
expect(result[0]?.value).toBe("123");
|
||||||
expect(result[1].name).toBe("xs");
|
expect(result[1]?.name).toBe("xs");
|
||||||
expect(result[1].value).toBe("abc");
|
expect(result[1]?.value).toBe("abc");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should load Facebook cookies from FACEBOOK_COOKIE env var", async () => {
|
test("should load Facebook cookies from FACEBOOK_COOKIE env var", async () => {
|
||||||
@@ -177,10 +177,6 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("should handle authentication errors", async () => {
|
test("should handle authentication errors", async () => {
|
||||||
const originalWarn = console.warn;
|
|
||||||
const warnMock = mock(() => {});
|
|
||||||
console.warn = warnMock;
|
|
||||||
|
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: false,
|
ok: false,
|
||||||
@@ -190,18 +186,11 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
try {
|
|
||||||
const result = await fetchFacebookItem("123");
|
const result = await fetchFacebookItem("123");
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
expect(global.fetch).toHaveBeenCalledTimes(1);
|
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||||
expect(warnMock).toHaveBeenCalledWith(
|
|
||||||
"Authentication error: Invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
|
||||||
);
|
|
||||||
} finally {
|
|
||||||
console.warn = originalWarn;
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should handle item not found", async () => {
|
test("should handle item not found", async () => {
|
||||||
@@ -214,7 +203,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("nonexistent");
|
const result = await fetchFacebookItem("nonexistent");
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
@@ -274,7 +263,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const _result = await fetchFacebookItem("123");
|
const _result = await fetchFacebookItem("123");
|
||||||
expect(attempts).toBe(2);
|
expect(attempts).toBe(2);
|
||||||
@@ -297,7 +286,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
},
|
},
|
||||||
text: () => Promise.resolve("Rate limited"),
|
text: () => Promise.resolve("Rate limited"),
|
||||||
});
|
});
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("429-loop");
|
const result = await fetchFacebookItem("429-loop");
|
||||||
|
|
||||||
@@ -346,7 +335,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("456");
|
const result = await fetchFacebookItem("456");
|
||||||
expect(result?.listingStatus).toBe("SOLD");
|
expect(result?.listingStatus).toBe("SOLD");
|
||||||
@@ -388,7 +377,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("457");
|
const result = await fetchFacebookItem("457");
|
||||||
|
|
||||||
@@ -435,7 +424,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("458");
|
const result = await fetchFacebookItem("458");
|
||||||
|
|
||||||
@@ -493,7 +482,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("789");
|
const result = await fetchFacebookItem("789");
|
||||||
expect(result).not.toBeNull();
|
expect(result).not.toBeNull();
|
||||||
@@ -512,7 +501,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("error");
|
const result = await fetchFacebookItem("error");
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
@@ -573,7 +562,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
|
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
|
||||||
|
|
||||||
@@ -618,7 +607,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
|
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
|
||||||
|
|
||||||
@@ -682,7 +671,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
|
const results = await fetchFacebookItems("chair", 1, "toronto", 25);
|
||||||
|
|
||||||
@@ -762,7 +751,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("chair", 1, "toronto", 25, {
|
const results = await fetchFacebookItems("chair", 1, "toronto", 25, {
|
||||||
hideUnstableResults: true,
|
hideUnstableResults: true,
|
||||||
@@ -845,7 +834,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("chair", 1, "toronto", 2, {
|
const results = await fetchFacebookItems("chair", 1, "toronto", 2, {
|
||||||
hideUnstableResults: true,
|
hideUnstableResults: true,
|
||||||
@@ -1132,7 +1121,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
const result = extractFacebookMarketplaceData(html);
|
const result = extractFacebookMarketplaceData(html);
|
||||||
expect(result).not.toBeNull();
|
expect(result).not.toBeNull();
|
||||||
expect(result).toHaveLength(2);
|
expect(result).toHaveLength(2);
|
||||||
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
|
expect(result?.[0]?.node.listing.marketplace_listing_title).toBe(
|
||||||
"Item 1",
|
"Item 1",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -1153,11 +1142,11 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
const result = extractFacebookMarketplaceData(html);
|
const result = extractFacebookMarketplaceData(html);
|
||||||
expect(result).not.toBeNull();
|
expect(result).not.toBeNull();
|
||||||
expect(result).toHaveLength(1);
|
expect(result).toHaveLength(1);
|
||||||
expect(result?.[0].node.listing.id).toBe("987654321");
|
expect(result?.[0]?.node.listing.id).toBe("987654321");
|
||||||
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
|
expect(result?.[0]?.node.listing.marketplace_listing_title).toBe(
|
||||||
"Vintage Bike",
|
"Vintage Bike",
|
||||||
);
|
);
|
||||||
expect(result?.[0].node.listing.listing_price).toEqual({
|
expect(result?.[0]?.node.listing.listing_price).toEqual({
|
||||||
amount: "120.00",
|
amount: "120.00",
|
||||||
formatted_amount: "CA$120",
|
formatted_amount: "CA$120",
|
||||||
currency: "CAD",
|
currency: "CAD",
|
||||||
@@ -1385,7 +1374,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
|
|
||||||
const ads = extractFacebookMarketplaceData(html);
|
const ads = extractFacebookMarketplaceData(html);
|
||||||
expect(ads).toHaveLength(1);
|
expect(ads).toHaveLength(1);
|
||||||
expect(ads?.[0].node.listing.marketplace_listing_title).toBe("Bike");
|
expect(ads?.[0]?.node.listing.marketplace_listing_title).toBe("Bike");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("prefers the strongest marketplace edge set when multiple edges arrays exist", () => {
|
test("prefers the strongest marketplace edge set when multiple edges arrays exist", () => {
|
||||||
@@ -1443,7 +1432,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
|
|
||||||
const ads = extractFacebookMarketplaceData(html);
|
const ads = extractFacebookMarketplaceData(html);
|
||||||
expect(ads).toHaveLength(1);
|
expect(ads).toHaveLength(1);
|
||||||
expect(ads?.[0].node.listing.id).toBe("right-1");
|
expect(ads?.[0]?.node.listing.id).toBe("right-1");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("rejects mixed edge arrays that contain non-listing entries", () => {
|
test("rejects mixed edge arrays that contain non-listing entries", () => {
|
||||||
@@ -1668,11 +1657,11 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
|
|
||||||
const results = parseFacebookAds(ads);
|
const results = parseFacebookAds(ads);
|
||||||
expect(results).toHaveLength(2);
|
expect(results).toHaveLength(2);
|
||||||
expect(results[0].title).toBe("Ad 1");
|
expect(results[0]?.title).toBe("Ad 1");
|
||||||
expect(results[0].listingPrice?.cents).toBe(5000);
|
expect(results[0]?.listingPrice?.cents).toBe(5000);
|
||||||
expect(results[0].address).toBe("Toronto");
|
expect(results[0]?.address).toBe("Toronto");
|
||||||
expect(results[1].title).toBe("Ad 2");
|
expect(results[1]?.title).toBe("Ad 2");
|
||||||
expect(results[1].address).toBe("Ottawa");
|
expect(results[1]?.address).toBe("Ottawa");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should filter out ads without price", () => {
|
test("should filter out ads without price", () => {
|
||||||
@@ -1704,14 +1693,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
|
|
||||||
const results = parseFacebookAds(ads);
|
const results = parseFacebookAds(ads);
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].title).toBe("With Price");
|
expect(results[0]?.title).toBe("With Price");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should handle malformed ads gracefully", () => {
|
test("should handle malformed ads gracefully", () => {
|
||||||
const originalWarn = console.warn;
|
|
||||||
const warnMock = mock(() => {});
|
|
||||||
console.warn = warnMock;
|
|
||||||
|
|
||||||
const ads = [
|
const ads = [
|
||||||
{
|
{
|
||||||
node: {
|
node: {
|
||||||
@@ -1731,15 +1716,14 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
node: {
|
node: {
|
||||||
// Missing listing
|
// Missing listing
|
||||||
},
|
},
|
||||||
} as { node: { listing?: unknown } },
|
} as unknown as { node: { listing?: unknown } },
|
||||||
];
|
];
|
||||||
|
|
||||||
const results = parseFacebookAds(ads);
|
const results = parseFacebookAds(
|
||||||
|
ads as unknown as Parameters<typeof parseFacebookAds>[0],
|
||||||
|
);
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].title).toBe("Valid Ad");
|
expect(results[0]?.title).toBe("Valid Ad");
|
||||||
expect(warnMock).toHaveBeenCalledTimes(1);
|
|
||||||
|
|
||||||
console.warn = originalWarn;
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test("parses formatted fallback prices with multiple commas", () => {
|
test("parses formatted fallback prices with multiple commas", () => {
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
process.env.FACEBOOK_COOKIE = facebookCookie;
|
process.env.FACEBOOK_COOKIE = facebookCookie;
|
||||||
global.fetch = mock(() => {
|
global.fetch = mock(() => {
|
||||||
throw new Error("fetch should be mocked in individual tests");
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -69,11 +69,11 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("iPhone", 1, "toronto", 25);
|
const results = await fetchFacebookItems("iPhone", 1, "toronto", 25);
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].title).toBe("iPhone 13");
|
expect(results[0]?.title).toBe("iPhone 13");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should filter out items without price", async () => {
|
test("should filter out items without price", async () => {
|
||||||
@@ -135,11 +135,11 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].title).toBe("With Price");
|
expect(results[0]?.title).toBe("With Price");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should respect MAX_ITEMS parameter", async () => {
|
test("should respect MAX_ITEMS parameter", async () => {
|
||||||
@@ -190,7 +190,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 5);
|
const results = await fetchFacebookItems("test", 1, "toronto", 5);
|
||||||
expect(results).toHaveLength(5);
|
expect(results).toHaveLength(5);
|
||||||
@@ -231,7 +231,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems(
|
const results = await fetchFacebookItems(
|
||||||
"nonexistent query",
|
"nonexistent query",
|
||||||
@@ -252,7 +252,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
@@ -281,7 +281,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("lamp", 1, "toronto", 25);
|
const results = await fetchFacebookItems("lamp", 1, "toronto", 25);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
@@ -322,14 +322,16 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("lamp", 1, "toronto", 25);
|
const results = await fetchFacebookItems("lamp", 1, "toronto", 25);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should handle network errors", async () => {
|
test("should handle network errors", async () => {
|
||||||
global.fetch = mock(() => Promise.reject(new Error("Network error")));
|
global.fetch = mock(() =>
|
||||||
|
Promise.reject(new Error("Network error")),
|
||||||
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
fetchFacebookItems("test", 1, "toronto", 25),
|
fetchFacebookItems("test", 1, "toronto", 25),
|
||||||
@@ -400,7 +402,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||||
expect(attempts).toBe(2);
|
expect(attempts).toBe(2);
|
||||||
@@ -473,13 +475,13 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("cars", 1, "toronto", 25);
|
const results = await fetchFacebookItems("cars", 1, "toronto", 25);
|
||||||
expect(results).toHaveLength(2);
|
expect(results).toHaveLength(2);
|
||||||
// Both should be classified as "item" type in search results (vehicle detection is for item details)
|
// Both should be classified as "item" type in search results (vehicle detection is for item details)
|
||||||
expect(results[0].title).toBe("2006 Honda Civic");
|
expect(results[0]?.title).toBe("2006 Honda Civic");
|
||||||
expect(results[1].title).toBe("iPhone 13");
|
expect(results[1]?.title).toBe("iPhone 13");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -542,7 +544,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems(
|
const results = await fetchFacebookItems(
|
||||||
"nintendo switch",
|
"nintendo switch",
|
||||||
@@ -551,8 +553,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
25,
|
25,
|
||||||
);
|
);
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].title).toBe("Nintendo Switch");
|
expect(results[0]?.title).toBe("Nintendo Switch");
|
||||||
expect(results[0].categoryId).toBe("479353692612078");
|
expect(results[0]?.categoryId).toBe("479353692612078");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should handle home goods/furniture listings", async () => {
|
test("should handle home goods/furniture listings", async () => {
|
||||||
@@ -613,12 +615,12 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("table", 1, "toronto", 25);
|
const results = await fetchFacebookItems("table", 1, "toronto", 25);
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].title).toBe("Dining Table");
|
expect(results[0]?.title).toBe("Dining Table");
|
||||||
expect(results[0].categoryId).toBe("1569171756675761");
|
expect(results[0]?.categoryId).toBe("1569171756675761");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -635,7 +637,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
@@ -651,7 +653,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
@@ -667,7 +669,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
@@ -708,7 +710,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
get: () => null,
|
get: () => null,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
);
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
const result = await fetchFacebookItem("123");
|
const result = await fetchFacebookItem("123");
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
|
|||||||
41
packages/core/test/http.test.ts
Normal file
41
packages/core/test/http.test.ts
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
import { afterEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import { fetchHtml } from "../src/utils/http";
|
||||||
|
|
||||||
|
describe("fetchHtml", () => {
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
const originalNodeEnv = process.env.NODE_ENV;
|
||||||
|
const originalSetTimeout = globalThis.setTimeout;
|
||||||
|
const originalClearTimeout = globalThis.clearTimeout;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
process.env.NODE_ENV = originalNodeEnv;
|
||||||
|
globalThis.setTimeout = originalSetTimeout;
|
||||||
|
globalThis.clearTimeout = originalClearTimeout;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("does not schedule throttle timers during tests", async () => {
|
||||||
|
process.env.NODE_ENV = "test";
|
||||||
|
const scheduledDelays: number[] = [];
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
|
text: () => Promise.resolve("<html></html>"),
|
||||||
|
}),
|
||||||
|
) as unknown as typeof fetch;
|
||||||
|
globalThis.setTimeout = mock((handler: TimerHandler, timeout?: number) => {
|
||||||
|
scheduledDelays.push(Number(timeout));
|
||||||
|
if (timeout !== 30_000 && typeof handler === "function") {
|
||||||
|
handler();
|
||||||
|
}
|
||||||
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||||
|
}) as unknown as typeof setTimeout;
|
||||||
|
globalThis.clearTimeout = mock(() => {}) as unknown as typeof clearTimeout;
|
||||||
|
|
||||||
|
await fetchHtml("https://example.com", 1000, { timeoutMs: 30_000 });
|
||||||
|
|
||||||
|
expect(scheduledDelays).not.toContain(1000);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -49,7 +49,7 @@ const originalFetch = global.fetch;
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
global.fetch = mock(() => {
|
global.fetch = mock(() => {
|
||||||
throw new Error("fetch should be mocked in individual tests");
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -310,7 +310,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchKijijiItems(
|
const results = await fetchKijijiItems(
|
||||||
"phone",
|
"phone",
|
||||||
@@ -418,7 +418,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchKijijiItems(
|
const results = await fetchKijijiItems(
|
||||||
"phone",
|
"phone",
|
||||||
@@ -515,7 +515,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchKijijiItems(
|
const results = await fetchKijijiItems(
|
||||||
"phone",
|
"phone",
|
||||||
@@ -628,7 +628,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchKijijiItems(
|
const results = await fetchKijijiItems(
|
||||||
"phone",
|
"phone",
|
||||||
@@ -771,7 +771,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchKijijiItems(
|
const results = await fetchKijijiItems(
|
||||||
"phone",
|
"phone",
|
||||||
@@ -872,7 +872,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
await parseDetailedListing(html, "https://www.kijiji.ca", {
|
await parseDetailedListing(html, "https://www.kijiji.ca", {
|
||||||
includeClientSideData: true,
|
includeClientSideData: true,
|
||||||
@@ -981,7 +981,7 @@ describe("fetchKijijiItems", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
throw new Error(`Unexpected URL: ${url}`);
|
throw new Error(`Unexpected URL: ${url}`);
|
||||||
}) as typeof fetch;
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
const results = await fetchKijijiItems(
|
const results = await fetchKijijiItems(
|
||||||
"phone",
|
"phone",
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ describe("HTML Parsing Integration", () => {
|
|||||||
// Mock fetch for all tests
|
// Mock fetch for all tests
|
||||||
global.fetch = mock(() => {
|
global.fetch = mock(() => {
|
||||||
throw new Error("fetch should be mocked in individual tests");
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
});
|
}) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -111,7 +111,7 @@ describe("HTML Parsing Integration", () => {
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||||
expect(results[0].listingLink).toBe(
|
expect(results[0]?.listingLink).toBe(
|
||||||
"https://www.kijiji.ca/v-iphone/k0l0",
|
"https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -146,7 +146,7 @@ describe("HTML Parsing Integration", () => {
|
|||||||
|
|
||||||
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0].name).toBe("iPhone 13 Pro");
|
expect(results[0]?.name).toBe("iPhone 13 Pro");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should return empty array for invalid HTML", () => {
|
test("should return empty array for invalid HTML", () => {
|
||||||
|
|||||||
29
packages/core/test/logger.test.ts
Normal file
29
packages/core/test/logger.test.ts
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { afterEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
|
||||||
|
describe("logger", () => {
|
||||||
|
const originalNodeEnv = process.env.NODE_ENV;
|
||||||
|
const originalConsoleLog = console.log;
|
||||||
|
const originalConsoleWarn = console.warn;
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
process.env.NODE_ENV = originalNodeEnv;
|
||||||
|
console.log = originalConsoleLog;
|
||||||
|
console.warn = originalConsoleWarn;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("suppresses log and warn output during tests", async () => {
|
||||||
|
process.env.NODE_ENV = "test";
|
||||||
|
const logMock = mock(() => {});
|
||||||
|
const warnMock = mock(() => {});
|
||||||
|
console.log = logMock;
|
||||||
|
console.warn = warnMock;
|
||||||
|
|
||||||
|
const { logger } = await import("../src/utils/logger");
|
||||||
|
|
||||||
|
logger.log("hidden log");
|
||||||
|
logger.warn("hidden warn");
|
||||||
|
|
||||||
|
expect(logMock).not.toHaveBeenCalled();
|
||||||
|
expect(warnMock).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { logger } from "./logger";
|
||||||
import { handleMcpRequest } from "./protocol/handler";
|
import { handleMcpRequest } from "./protocol/handler";
|
||||||
import { serverCard } from "./protocol/metadata";
|
import { serverCard } from "./protocol/metadata";
|
||||||
|
|
||||||
@@ -33,4 +34,4 @@ const server = Bun.serve({
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`MCP Server running on ${server.hostname}:${server.port}`);
|
logger.log(`MCP Server running on ${server.hostname}:${server.port}`);
|
||||||
|
|||||||
10
packages/mcp-server/src/logger.ts
Normal file
10
packages/mcp-server/src/logger.ts
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
const isTest = () => process.env.NODE_ENV === "test";
|
||||||
|
|
||||||
|
export const logger = {
|
||||||
|
log: (...args: Parameters<typeof console.log>) => {
|
||||||
|
if (!isTest()) console.log(...args);
|
||||||
|
},
|
||||||
|
error: (...args: Parameters<typeof console.error>) => {
|
||||||
|
if (!isTest()) console.error(...args);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { logger } from "../logger";
|
||||||
import { tools } from "./tools";
|
import { tools } from "./tools";
|
||||||
|
|
||||||
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
|
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
|
||||||
@@ -119,7 +120,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
if (args.unstableFilter !== undefined)
|
if (args.unstableFilter !== undefined)
|
||||||
params.append("unstableFilter", args.unstableFilter.toString());
|
params.append("unstableFilter", args.unstableFilter.toString());
|
||||||
|
|
||||||
console.log(
|
logger.log(
|
||||||
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
|
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
|
||||||
);
|
);
|
||||||
const response = await Promise.race([
|
const response = await Promise.race([
|
||||||
@@ -135,13 +136,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const errorText = await response.text();
|
const errorText = await response.text();
|
||||||
console.error(
|
logger.error(
|
||||||
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
|
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
|
||||||
);
|
);
|
||||||
throw new Error(`API returned ${response.status}: ${errorText}`);
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
}
|
}
|
||||||
result = await response.json();
|
result = await response.json();
|
||||||
console.log(
|
logger.log(
|
||||||
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
|
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
);
|
);
|
||||||
} else if (name === "search_facebook") {
|
} else if (name === "search_facebook") {
|
||||||
@@ -160,7 +161,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
if (args.unstableFilter !== undefined)
|
if (args.unstableFilter !== undefined)
|
||||||
params.append("unstableFilter", args.unstableFilter.toString());
|
params.append("unstableFilter", args.unstableFilter.toString());
|
||||||
|
|
||||||
console.log(
|
logger.log(
|
||||||
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
|
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
|
||||||
);
|
);
|
||||||
const response = await Promise.race([
|
const response = await Promise.race([
|
||||||
@@ -176,13 +177,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const errorText = await response.text();
|
const errorText = await response.text();
|
||||||
console.error(
|
logger.error(
|
||||||
`[MCP] Facebook API error ${response.status}: ${errorText}`,
|
`[MCP] Facebook API error ${response.status}: ${errorText}`,
|
||||||
);
|
);
|
||||||
throw new Error(`API returned ${response.status}: ${errorText}`);
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
}
|
}
|
||||||
result = await response.json();
|
result = await response.json();
|
||||||
console.log(
|
logger.log(
|
||||||
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
|
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
);
|
);
|
||||||
} else if (name === "search_ebay") {
|
} else if (name === "search_ebay") {
|
||||||
@@ -214,7 +215,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
if (args.unstableFilter !== undefined)
|
if (args.unstableFilter !== undefined)
|
||||||
params.append("unstableFilter", args.unstableFilter.toString());
|
params.append("unstableFilter", args.unstableFilter.toString());
|
||||||
|
|
||||||
console.log(
|
logger.log(
|
||||||
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
|
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
|
||||||
);
|
);
|
||||||
const response = await Promise.race([
|
const response = await Promise.race([
|
||||||
@@ -230,13 +231,13 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const errorText = await response.text();
|
const errorText = await response.text();
|
||||||
console.error(
|
logger.error(
|
||||||
`[MCP] eBay API error ${response.status}: ${errorText}`,
|
`[MCP] eBay API error ${response.status}: ${errorText}`,
|
||||||
);
|
);
|
||||||
throw new Error(`API returned ${response.status}: ${errorText}`);
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
}
|
}
|
||||||
result = await response.json();
|
result = await response.json();
|
||||||
console.log(
|
logger.log(
|
||||||
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
|
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ describe("MCP protocol cookie inputs", () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
|
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -48,7 +48,7 @@ describe("MCP protocol cookie inputs", () => {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
|
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
|
||||||
.calls[0]?.[0];
|
.calls[0]?.[0];
|
||||||
expect(String(calledUrl)).toContain("/facebook?q=laptop");
|
expect(String(calledUrl)).toContain("/facebook?q=laptop");
|
||||||
expect(String(calledUrl)).not.toContain("cookies=");
|
expect(String(calledUrl)).not.toContain("cookies=");
|
||||||
@@ -59,7 +59,7 @@ describe("MCP protocol unstableFilter", () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
|
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
|
||||||
) as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
});
|
});
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
@@ -103,7 +103,7 @@ describe("MCP protocol unstableFilter", () => {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
|
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
|
||||||
.calls[0]?.[0];
|
.calls[0]?.[0];
|
||||||
expect(String(calledUrl)).toContain("unstableFilter=true");
|
expect(String(calledUrl)).toContain("unstableFilter=true");
|
||||||
});
|
});
|
||||||
@@ -127,7 +127,7 @@ describe("MCP protocol unstableFilter", () => {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
|
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
|
||||||
.calls[0]?.[0];
|
.calls[0]?.[0];
|
||||||
expect(String(calledUrl)).toContain("unstableFilter=true");
|
expect(String(calledUrl)).toContain("unstableFilter=true");
|
||||||
});
|
});
|
||||||
@@ -151,7 +151,7 @@ describe("MCP protocol unstableFilter", () => {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
|
const calledUrl = (global.fetch as unknown as ReturnType<typeof mock>).mock
|
||||||
.calls[0]?.[0];
|
.calls[0]?.[0];
|
||||||
expect(String(calledUrl)).toContain("unstableFilter=true");
|
expect(String(calledUrl)).toContain("unstableFilter=true");
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user