diff --git a/packages/api-server/src/routes/kijiji.ts b/packages/api-server/src/routes/kijiji.ts index 227d992..6b8ea18 100644 --- a/packages/api-server/src/routes/kijiji.ts +++ b/packages/api-server/src/routes/kijiji.ts @@ -41,6 +41,7 @@ export async function kijijiRoute(req: Request): Promise { maxPages, priceMin, priceMax, + cookies: reqUrl.searchParams.get("cookies") || undefined, }; try { diff --git a/packages/core/src/scrapers/kijiji.ts b/packages/core/src/scrapers/kijiji.ts index 465dc85..d6d28fd 100644 --- a/packages/core/src/scrapers/kijiji.ts +++ b/packages/core/src/scrapers/kijiji.ts @@ -2,6 +2,11 @@ import cliProgress from "cli-progress"; import { parseHTML } from "linkedom"; import unidecode from "unidecode"; import type { HTMLString } from "../types/common"; +import { + type CookieConfig, + formatCookiesForHeader, + loadCookiesOptional, +} from "../utils/cookies"; import { formatCentsToCurrency } from "../utils/format"; import { fetchHtml, @@ -13,6 +18,14 @@ import { ValidationError, } from "../utils/http"; +// Kijiji cookie configuration +const KIJIJI_COOKIE_CONFIG: CookieConfig = { + name: "Kijiji", + domain: ".kijiji.ca", + envVar: "KIJIJI_COOKIE", + filePath: "./cookies/kijiji.json", +}; + // ----------------------------- Types ----------------------------- type SearchListing = { @@ -110,6 +123,7 @@ export interface SearchOptions { maxPages?: number; // Default: 5 priceMin?: number; priceMax?: number; + cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection) } export interface ListingFetchOptions { @@ -691,6 +705,16 @@ export default async function fetchKijijiItems( ) { const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND)); + // Load Kijiji cookies (optional - helps bypass bot detection) + const cookies = await loadCookiesOptional( + KIJIJI_COOKIE_CONFIG, + searchOptions.cookies, + ); + const cookieHeader = + cookies.length > 0 + ? formatCookiesForHeader(cookies, "www.kijiji.ca") + : undefined; + // Set defaults for configuration const finalSearchOptions: Required = { location: searchOptions.location ?? 1700272, // Default to GTA @@ -701,6 +725,7 @@ export default async function fetchKijijiItems( maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages priceMin: searchOptions.priceMin as number, priceMax: searchOptions.priceMax as number, + cookies: searchOptions.cookies ?? "", }; const finalListingOptions: Required = { @@ -733,6 +758,7 @@ export default async function fetchKijijiItems( ); } }, + headers: cookieHeader ? { cookie: cookieHeader } : undefined, }); const searchResults = parseSearch(searchHtml, BASE_URL); @@ -782,6 +808,7 @@ export default async function fetchKijijiItems( ); } }, + headers: cookieHeader ? { cookie: cookieHeader } : undefined, }); const parsed = await parseDetailedListing( html, diff --git a/packages/mcp-server/src/protocol/handler.ts b/packages/mcp-server/src/protocol/handler.ts index 2a41249..410b392 100644 --- a/packages/mcp-server/src/protocol/handler.ts +++ b/packages/mcp-server/src/protocol/handler.ts @@ -115,6 +115,7 @@ export async function handleMcpRequest(req: Request): Promise { params.append("priceMin", args.priceMin.toString()); if (args.priceMax) params.append("priceMax", args.priceMax.toString()); + if (args.cookies) params.append("cookies", args.cookies); console.log( `[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`, diff --git a/packages/mcp-server/src/protocol/tools.ts b/packages/mcp-server/src/protocol/tools.ts index 60a52fc..643459d 100644 --- a/packages/mcp-server/src/protocol/tools.ts +++ b/packages/mcp-server/src/protocol/tools.ts @@ -52,6 +52,11 @@ export const tools = [ type: "number", description: "Maximum price in cents", }, + cookies: { + type: "string", + description: + "Optional: Kijiji session cookies to bypass bot detection (JSON array or 'name1=value1; name2=value2')", + }, }, required: ["query"], },