From 3e4e35c9ae3e4de99966d99a62b6b8c2d527d8cc Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 00:32:23 -0400 Subject: [PATCH 01/14] fix: tighten route integer parsing and test coverage --- packages/api-server/src/routes/ebay.ts | 105 +++---- packages/api-server/src/routes/helpers.ts | 17 +- packages/api-server/test/routes.test.ts | 316 +++++++++++++++++++++- 3 files changed, 377 insertions(+), 61 deletions(-) diff --git a/packages/api-server/src/routes/ebay.ts b/packages/api-server/src/routes/ebay.ts index 1fc6a6c..bad0225 100644 --- a/packages/api-server/src/routes/ebay.ts +++ b/packages/api-server/src/routes/ebay.ts @@ -11,59 +11,60 @@ import { * Search eBay for listings (default: Buy It Now only, Canada only) */ export async function ebayRoute(req: Request): Promise { + const reqUrl = new URL(req.url); + + const SEARCH_QUERY = getRequiredSearchQuery(req); + if (SEARCH_QUERY instanceof Response) { + return SEARCH_QUERY; + } + + const minPrice = parseNonNegativeIntegerParam( + reqUrl.searchParams, + "minPrice", + ); + if (minPrice instanceof Response) { + return minPrice; + } + const maxPrice = parseNonNegativeIntegerParam( + reqUrl.searchParams, + "maxPrice", + ); + if (maxPrice instanceof Response) { + return maxPrice; + } + const strictMode = reqUrl.searchParams.get("strictMode") === "true"; + const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false"; + const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false"; + const exclusionsParam = reqUrl.searchParams.get("exclusions"); + const exclusions = exclusionsParam + ? exclusionsParam.split(",").map((s) => s.trim()) + : []; + const keywordsParam = reqUrl.searchParams.get("keywords"); + const keywords = keywordsParam + ? keywordsParam.split(",").map((s) => s.trim()) + : [SEARCH_QUERY]; + + const maxItems = parseNonNegativeIntegerParam( + reqUrl.searchParams, + "maxItems", + ); + if (maxItems instanceof Response) { + return maxItems; + } + const hideUnstableResults = + reqUrl.searchParams.get("unstableFilter") === "true"; + const opts = { + minPrice, + maxPrice, + strictMode, + exclusions, + keywords, + buyItNowOnly, + canadaOnly, + maxItems, + }; + try { - const reqUrl = new URL(req.url); - - const SEARCH_QUERY = getRequiredSearchQuery(req); - if (SEARCH_QUERY instanceof Response) { - return SEARCH_QUERY; - } - - const minPrice = parseNonNegativeIntegerParam( - reqUrl.searchParams, - "minPrice", - ); - if (minPrice instanceof Response) { - return minPrice; - } - const maxPrice = parseNonNegativeIntegerParam( - reqUrl.searchParams, - "maxPrice", - ); - if (maxPrice instanceof Response) { - return maxPrice; - } - const strictMode = reqUrl.searchParams.get("strictMode") === "true"; - const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false"; - const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false"; - const exclusionsParam = reqUrl.searchParams.get("exclusions"); - const exclusions = exclusionsParam - ? exclusionsParam.split(",").map((s) => s.trim()) - : []; - const keywordsParam = reqUrl.searchParams.get("keywords"); - const keywords = keywordsParam - ? keywordsParam.split(",").map((s) => s.trim()) - : [SEARCH_QUERY]; - - const maxItems = parseNonNegativeIntegerParam( - reqUrl.searchParams, - "maxItems", - ); - if (maxItems instanceof Response) { - return maxItems; - } - const hideUnstableResults = - reqUrl.searchParams.get("unstableFilter") === "true"; - const opts = { - minPrice, - maxPrice, - strictMode, - exclusions, - keywords, - buyItNowOnly, - canadaOnly, - maxItems, - }; if (hideUnstableResults) { const items = await fetchEbayItems(SEARCH_QUERY, 1, opts, { hideUnstableResults: true, diff --git a/packages/api-server/src/routes/helpers.ts b/packages/api-server/src/routes/helpers.ts index 3b13092..0455bc9 100644 --- a/packages/api-server/src/routes/helpers.ts +++ b/packages/api-server/src/routes/helpers.ts @@ -12,6 +12,15 @@ export function getRequiredSearchQuery(req: Request): string | Response { return query; } +export function parseNonNegativeIntegerParam( + searchParams: URLSearchParams, + name: string, + defaultValue: number, +): number | Response; +export function parseNonNegativeIntegerParam( + searchParams: URLSearchParams, + name: string, +): number | undefined | Response; export function parseNonNegativeIntegerParam( searchParams: URLSearchParams, name: string, @@ -21,8 +30,14 @@ export function parseNonNegativeIntegerParam( if (rawValue === null) { return defaultValue; } + if (!/^\d+$/.test(rawValue)) { + return Response.json( + { message: `Invalid ${name} parameter` }, + { status: 400 }, + ); + } const value = Number(rawValue); - if (!Number.isInteger(value) || value < 0) { + if (value < 0) { return Response.json( { message: `Invalid ${name} parameter` }, { status: 400 }, diff --git a/packages/api-server/test/routes.test.ts b/packages/api-server/test/routes.test.ts index 64909ad..9be15ae 100644 --- a/packages/api-server/test/routes.test.ts +++ b/packages/api-server/test/routes.test.ts @@ -501,6 +501,66 @@ describe("API routes", () => { expect(body.message).toBe("Invalid maxItems parameter"); }); + test("ebayRoute returns 400 for non-integer maxItems", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxItems=10abc"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("ebayRoute returns 400 for decimal maxItems", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxItems=1.5"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("ebayRoute returns 400 for empty maxItems", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxItems="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("ebayRoute returns 400 for whitespace maxItems", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxItems=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("ebayRoute returns 400 for hex maxItems", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxItems=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + test("facebookRoute returns 400 for invalid maxItems", async () => { const { facebookRoute } = await import("../src/routes/facebook"); @@ -513,7 +573,67 @@ describe("API routes", () => { expect(body.message).toBe("Invalid maxItems parameter"); }); - test("ebayRoute rejects non-integer minPrice", async () => { + test("facebookRoute returns 400 for non-integer maxItems", async () => { + const { facebookRoute } = await import("../src/routes/facebook"); + + const response = await facebookRoute( + new Request("http://localhost/api/facebook?q=laptop&maxItems=10abc"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("facebookRoute returns 400 for decimal maxItems", async () => { + const { facebookRoute } = await import("../src/routes/facebook"); + + const response = await facebookRoute( + new Request("http://localhost/api/facebook?q=laptop&maxItems=1.5"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("facebookRoute returns 400 for empty maxItems", async () => { + const { facebookRoute } = await import("../src/routes/facebook"); + + const response = await facebookRoute( + new Request("http://localhost/api/facebook?q=laptop&maxItems="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("facebookRoute returns 400 for whitespace maxItems", async () => { + const { facebookRoute } = await import("../src/routes/facebook"); + + const response = await facebookRoute( + new Request("http://localhost/api/facebook?q=laptop&maxItems=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("facebookRoute returns 400 for hex maxItems", async () => { + const { facebookRoute } = await import("../src/routes/facebook"); + + const response = await facebookRoute( + new Request("http://localhost/api/facebook?q=laptop&maxItems=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxItems parameter"); + }); + + test("ebayRoute returns 400 for non-integer minPrice", async () => { const { ebayRoute } = await import("../src/routes/ebay"); const response = await ebayRoute( @@ -537,7 +657,19 @@ describe("API routes", () => { expect(body.message).toBe("Invalid minPrice parameter"); }); - test("ebayRoute rejects non-integer maxPrice", async () => { + test("ebayRoute returns 400 for decimal minPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&minPrice=1.5"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid minPrice parameter"); + }); + + test("ebayRoute returns 400 for non-integer maxPrice", async () => { const { ebayRoute } = await import("../src/routes/ebay"); const response = await ebayRoute( @@ -561,7 +693,19 @@ describe("API routes", () => { expect(body.message).toBe("Invalid maxPrice parameter"); }); - test("kijijiRoute rejects decimal maxPages", async () => { + test("ebayRoute returns 400 for decimal maxPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxPrice=1.5"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPrice parameter"); + }); + + test("kijijiRoute returns 400 for decimal maxPages", async () => { const { kijijiRoute } = await import("../src/routes/kijiji"); const response = await kijijiRoute( @@ -585,6 +729,54 @@ describe("API routes", () => { expect(body.message).toBe("Invalid maxPages parameter"); }); + test("kijijiRoute returns 400 for non-integer maxPages", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&maxPages=10abc"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPages parameter"); + }); + + test("kijijiRoute returns 400 for empty maxPages", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&maxPages="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPages parameter"); + }); + + test("kijijiRoute returns 400 for whitespace maxPages", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&maxPages=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPages parameter"); + }); + + test("kijijiRoute returns 400 for hex maxPages", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&maxPages=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPages parameter"); + }); + test("kijijiRoute returns 400 for invalid priceMin", async () => { const { kijijiRoute } = await import("../src/routes/kijiji"); @@ -597,6 +789,66 @@ describe("API routes", () => { expect(body.message).toBe("Invalid priceMin parameter"); }); + test("kijijiRoute returns 400 for decimal priceMin", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMin=1.5"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMin parameter"); + }); + + test("kijijiRoute returns 400 for non-integer priceMin", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMin=10abc"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMin parameter"); + }); + + test("kijijiRoute returns 400 for empty priceMin", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMin="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMin parameter"); + }); + + test("kijijiRoute returns 400 for whitespace priceMin", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMin=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMin parameter"); + }); + + test("kijijiRoute returns 400 for hex priceMin", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMin=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMin parameter"); + }); + test("kijijiRoute returns 400 for invalid priceMax", async () => { const { kijijiRoute } = await import("../src/routes/kijiji"); @@ -609,16 +861,64 @@ describe("API routes", () => { expect(body.message).toBe("Invalid priceMax parameter"); }); - test("facebookRoute rejects non-integer maxItems", async () => { - const { facebookRoute } = await import("../src/routes/facebook"); + test("kijijiRoute returns 400 for decimal priceMax", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); - const response = await facebookRoute( - new Request("http://localhost/api/facebook?q=laptop&maxItems=10abc"), + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMax=1.5"), ); expect(response.status).toBe(400); const body = await response.json(); - expect(body.message).toBe("Invalid maxItems parameter"); + expect(body.message).toBe("Invalid priceMax parameter"); + }); + + test("kijijiRoute returns 400 for non-integer priceMax", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMax=10abc"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMax parameter"); + }); + + test("kijijiRoute returns 400 for empty priceMax", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMax="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMax parameter"); + }); + + test("kijijiRoute returns 400 for whitespace priceMax", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMax=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMax parameter"); + }); + + test("kijijiRoute returns 400 for hex priceMax", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + const response = await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&priceMax=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid priceMax parameter"); }); test("facebookRoute returns 400 for negative maxItems", async () => { From abdd39d65c31c651e7d710f9e3b8a1e26e8cf5d5 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 00:56:37 -0400 Subject: [PATCH 02/14] fix: complete ebay integer validation test coverage --- packages/api-server/src/routes/helpers.ts | 9 +-- packages/api-server/test/routes.test.ts | 72 +++++++++++++++++++++++ 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/packages/api-server/src/routes/helpers.ts b/packages/api-server/src/routes/helpers.ts index 0455bc9..4275b8f 100644 --- a/packages/api-server/src/routes/helpers.ts +++ b/packages/api-server/src/routes/helpers.ts @@ -36,14 +36,7 @@ export function parseNonNegativeIntegerParam( { status: 400 }, ); } - const value = Number(rawValue); - if (value < 0) { - return Response.json( - { message: `Invalid ${name} parameter` }, - { status: 400 }, - ); - } - return value; + return Number(rawValue); } export function emptySearchResponse(): Response { diff --git a/packages/api-server/test/routes.test.ts b/packages/api-server/test/routes.test.ts index 9be15ae..f1ae498 100644 --- a/packages/api-server/test/routes.test.ts +++ b/packages/api-server/test/routes.test.ts @@ -633,6 +633,78 @@ describe("API routes", () => { expect(body.message).toBe("Invalid maxItems parameter"); }); + test("ebayRoute returns 400 for empty minPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&minPrice="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid minPrice parameter"); + }); + + test("ebayRoute returns 400 for whitespace minPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&minPrice=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid minPrice parameter"); + }); + + test("ebayRoute returns 400 for hex minPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&minPrice=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid minPrice parameter"); + }); + + test("ebayRoute returns 400 for empty maxPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxPrice="), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPrice parameter"); + }); + + test("ebayRoute returns 400 for whitespace maxPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxPrice=%20%20"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPrice parameter"); + }); + + test("ebayRoute returns 400 for hex maxPrice", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + const response = await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&maxPrice=0x10"), + ); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body.message).toBe("Invalid maxPrice parameter"); + }); + test("ebayRoute returns 400 for non-integer minPrice", async () => { const { ebayRoute } = await import("../src/routes/ebay"); From 22eb65d4a2d5ae36d4c6c29a9aeb4f1d577646d6 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 05:37:24 -0400 Subject: [PATCH 03/14] refactor: share mcp api calls --- packages/mcp-server/src/protocol/handler.ts | 103 +++++--------------- packages/mcp-server/test/protocol.test.ts | 27 +++++ 2 files changed, 54 insertions(+), 76 deletions(-) diff --git a/packages/mcp-server/src/protocol/handler.ts b/packages/mcp-server/src/protocol/handler.ts index 80150e4..c5f9a09 100644 --- a/packages/mcp-server/src/protocol/handler.ts +++ b/packages/mcp-server/src/protocol/handler.ts @@ -2,7 +2,30 @@ import { logger } from "../logger"; import { tools } from "./tools"; const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api"; -const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; // 3 minutes default +const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; + +async function callMarketplaceApi( + marketplace: string, + params: URLSearchParams, +): Promise { + const url = `${API_BASE_URL}/${marketplace}?${params.toString()}`; + logger.log(`[MCP] Calling ${marketplace} API`); + const response = await Promise.race([ + fetch(url), + new Promise((_, reject) => + setTimeout( + () => reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)), + API_TIMEOUT, + ), + ), + ]); + if (!response.ok) { + const errorText = await response.text(); + logger.error(`[MCP] ${marketplace} API error ${response.status}: ${errorText}`); + throw new Error(`API returned ${response.status}: ${errorText}`); + } + return response.json(); +} /** * Handle MCP JSON-RPC 2.0 protocol requests @@ -119,31 +142,7 @@ export async function handleMcpRequest(req: Request): Promise { if (args.unstableFilter !== undefined) params.append("unstableFilter", args.unstableFilter.toString()); - logger.log( - `[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`, - ); - const response = await Promise.race([ - fetch(`${API_BASE_URL}/kijiji?${params.toString()}`), - new Promise((_, reject) => - setTimeout( - () => - reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)), - API_TIMEOUT, - ), - ), - ]); - - if (!response.ok) { - const errorText = await response.text(); - logger.error( - `[MCP] Kijiji API error ${response.status}: ${errorText}`, - ); - throw new Error(`API returned ${response.status}: ${errorText}`); - } - result = await response.json(); - logger.log( - `[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`, - ); + result = await callMarketplaceApi("kijiji", params); } else if (name === "search_facebook") { const query = args.query; if (!query) { @@ -160,31 +159,7 @@ export async function handleMcpRequest(req: Request): Promise { if (args.unstableFilter !== undefined) params.append("unstableFilter", args.unstableFilter.toString()); - logger.log( - `[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`, - ); - const response = await Promise.race([ - fetch(`${API_BASE_URL}/facebook?${params.toString()}`), - new Promise((_, reject) => - setTimeout( - () => - reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)), - API_TIMEOUT, - ), - ), - ]); - - if (!response.ok) { - const errorText = await response.text(); - logger.error( - `[MCP] Facebook API error ${response.status}: ${errorText}`, - ); - throw new Error(`API returned ${response.status}: ${errorText}`); - } - result = await response.json(); - logger.log( - `[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`, - ); + result = await callMarketplaceApi("facebook", params); } else if (name === "search_ebay") { const query = args.query; if (!query) { @@ -214,31 +189,7 @@ export async function handleMcpRequest(req: Request): Promise { if (args.unstableFilter !== undefined) params.append("unstableFilter", args.unstableFilter.toString()); - logger.log( - `[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`, - ); - const response = await Promise.race([ - fetch(`${API_BASE_URL}/ebay?${params.toString()}`), - new Promise((_, reject) => - setTimeout( - () => - reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)), - API_TIMEOUT, - ), - ), - ]); - - if (!response.ok) { - const errorText = await response.text(); - logger.error( - `[MCP] eBay API error ${response.status}: ${errorText}`, - ); - throw new Error(`API returned ${response.status}: ${errorText}`); - } - result = await response.json(); - logger.log( - `[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`, - ); + result = await callMarketplaceApi("ebay", params); } else { return Response.json({ jsonrpc: "2.0", diff --git a/packages/mcp-server/test/protocol.test.ts b/packages/mcp-server/test/protocol.test.ts index 36bf633..69bddcd 100644 --- a/packages/mcp-server/test/protocol.test.ts +++ b/packages/mcp-server/test/protocol.test.ts @@ -152,6 +152,33 @@ describe("MCP protocol unstableFilter", () => { expect(String(calledUrl)).toContain("unstableFilter=true"); }); + test("tools/call returns API JSON as text content", async () => { + global.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify([{ title: "item" }]), { status: 200 }), + ), + ) as unknown as typeof fetch; + + const response = await handleMcpRequest( + new Request("http://localhost", { + method: "POST", + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "search_facebook", + arguments: { query: "laptop" }, + }, + }), + }), + ); + + const body = await response.json(); + expect(body.result.content[0].type).toBe("text"); + expect(JSON.parse(body.result.content[0].text)).toEqual([{ title: "item" }]); + }); + test("handler should forward unstableFilter=true for search_ebay", async () => { await handleMcpRequest( new Request("http://localhost", { From 6e50ebf90106d65b0e5cea4fbf72f77d82682083 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 13:14:20 -0400 Subject: [PATCH 04/14] refactor: share scraper http fetching --- packages/core/src/scrapers/ebay.ts | 34 +---- packages/core/src/scrapers/facebook.ts | 166 ++++++------------------- packages/core/src/utils/http.ts | 53 +++++--- packages/core/src/utils/logger.ts | 3 + packages/core/test/ebay-core.test.ts | 19 +++ packages/core/test/http.test.ts | 19 +++ 6 files changed, 121 insertions(+), 173 deletions(-) diff --git a/packages/core/src/scrapers/ebay.ts b/packages/core/src/scrapers/ebay.ts index 25254d7..a0d0864 100644 --- a/packages/core/src/scrapers/ebay.ts +++ b/packages/core/src/scrapers/ebay.ts @@ -9,7 +9,7 @@ import { ensureCookies, formatCookiesForHeader, } from "../utils/cookies"; -import { delay } from "../utils/delay"; +import { fetchHtml, HttpError } from "../utils/http"; import { logger } from "../utils/logger"; import { classifyUnstableListings } from "../utils/unstable"; @@ -102,17 +102,6 @@ function parseEbayPrice( return { cents, currency }; } -class HttpError extends Error { - constructor( - message: string, - public readonly status: number, - public readonly url: string, - ) { - super(message); - this.name = "HttpError"; - } -} - // ----------------------------- Parsing ----------------------------- /** @@ -500,22 +489,7 @@ export default async function fetchEbayItems( headers.Cookie = cookies; } - const res = await fetch(searchUrl, { - method: "GET", - headers, - }); - - if (!res.ok) { - throw new HttpError( - `Request failed with status ${res.status}`, - res.status, - searchUrl, - ); - } - - const searchHtml = await res.text(); - // Respect per-request delay to keep at or under REQUESTS_PER_SECOND - await delay(DELAY_MS); + const searchHtml = await fetchHtml(searchUrl, DELAY_MS, { headers }); logger.log(`\nParsing eBay listings...`); @@ -538,8 +512,8 @@ export default async function fetchEbayItems( return finalizeResults(filteredListings); } catch (err) { if (err instanceof HttpError) { - console.error( - `Failed to fetch eBay search (${err.status}): ${err.message}`, + logger.error( + `Failed to fetch eBay search (${err.statusCode}): ${err.message}`, ); return finalizeResults([]); } diff --git a/packages/core/src/scrapers/facebook.ts b/packages/core/src/scrapers/facebook.ts index b70d98c..12fb129 100644 --- a/packages/core/src/scrapers/facebook.ts +++ b/packages/core/src/scrapers/facebook.ts @@ -12,9 +12,8 @@ import { formatCookiesForHeader, parseCookieString, } from "../utils/cookies"; -import { delay } from "../utils/delay"; import { formatCentsToCurrency } from "../utils/format"; -import { isRecord } from "../utils/http"; +import { fetchHtml, HttpError, isRecord, RateLimitError } from "../utils/http"; import { logger } from "../utils/logger"; import { classifyUnstableListings } from "../utils/unstable"; @@ -219,17 +218,6 @@ export async function ensureFacebookCookies(): Promise { return ensureCookies(FACEBOOK_COOKIE_CONFIG); } -class HttpError extends Error { - constructor( - message: string, - public readonly status: number, - public readonly url: string, - ) { - super(message); - this.name = "HttpError"; - } -} - // ----------------------------- Extraction Metrics ----------------------------- /** @@ -274,112 +262,21 @@ function logExtractionMetrics(success: boolean, itemId?: string) { // ----------------------------- HTTP Client ----------------------------- -/** - Fetch HTML with a basic retry strategy and simple rate-limit delay between calls. - - Retries on 429 and 5xx - - Respects X-RateLimit-Reset when present (seconds) - - Supports custom cookies for Facebook authentication -*/ -async function fetchHtml( - url: string, - DELAY_MS: number, - opts?: { - maxRetries?: number; - retryBaseMs?: number; - onRateInfo?: (remaining: string | null, reset: string | null) => void; - cookies?: string; - }, -): Promise<{ html: HTMLString; responseUrl: string }> { - const maxRetries = opts?.maxRetries ?? 3; - const retryBaseMs = opts?.retryBaseMs ?? 500; - let lastRateLimitError: HttpError | null = null; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - const headers: Record = { - accept: - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "accept-language": "en-GB,en-US;q=0.9,en;q=0.8", - "accept-encoding": "gzip, deflate, br", - "cache-control": "no-cache", - "upgrade-insecure-requests": "1", - "sec-fetch-dest": "document", - "sec-fetch-mode": "navigate", - "sec-fetch-site": "none", - "sec-fetch-user": "?1", - "user-agent": - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - }; - - // Add cookies if provided - if (opts?.cookies) { - headers.cookie = opts.cookies; - } - - const res = await fetch(url, { - method: "GET", - headers, - }); - - const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining"); - const rateLimitReset = res.headers.get("X-RateLimit-Reset"); - opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset); - - if (!res.ok) { - // Respect 429 reset if provided - if (res.status === 429) { - lastRateLimitError = new HttpError( - `Request failed with status ${res.status}`, - res.status, - url, - ); - const resetSeconds = rateLimitReset - ? Number(rateLimitReset) - : Number.NaN; - const waitMs = Number.isFinite(resetSeconds) - ? Math.max(0, resetSeconds * 1000) - : (attempt + 1) * retryBaseMs; - if (attempt >= maxRetries) { - throw lastRateLimitError; - } - await delay(waitMs); - continue; - } - // For Facebook, 400 often means authentication required - // Don't retry 4xx client errors except 429 - if (res.status >= 400 && res.status < 500 && res.status !== 429) { - throw new HttpError( - `Request failed with status ${res.status} (Facebook may require authentication cookies for access)`, - res.status, - url, - ); - } - // Retry on 5xx - if (res.status >= 500 && res.status < 600 && attempt < maxRetries) { - await delay((attempt + 1) * retryBaseMs); - continue; - } - throw new HttpError( - `Request failed with status ${res.status}`, - res.status, - url, - ); - } - - const html = await res.text(); - // Respect per-request delay to keep at or under REQUESTS_PER_SECOND - await delay(DELAY_MS); - return { html, responseUrl: res.url || url }; - } catch (err) { - if (err instanceof HttpError) { - throw err; - } - if (attempt >= maxRetries) throw err; - await delay((attempt + 1) * retryBaseMs); - } - } - - throw lastRateLimitError ?? new Error("Exhausted retries without response"); +function createFacebookHeaders(cookies: string): Record { + return { + accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "accept-language": "en-GB,en-US;q=0.9,en;q=0.8", + "cache-control": "no-cache", + "upgrade-insecure-requests": "1", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-user": "?1", + "user-agent": + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + cookie: cookies, + }; } // ----------------------------- Parsing ----------------------------- @@ -1157,6 +1054,8 @@ export default async function fetchFacebookItems( try { const response = await fetchHtml(searchUrl, DELAY_MS, { maxRetries: 3, + includeResponseUrl: true, + headers: createFacebookHeaders(cookiesHeader), onRateInfo: (remaining, reset) => { if (remaining && reset) { logger.log( @@ -1164,22 +1063,27 @@ export default async function fetchFacebookItems( ); } }, - cookies: cookiesHeader, }); searchHtml = response.html; searchResponseUrl = response.responseUrl; } catch (err) { if (err instanceof HttpError) { logger.warn( - `\nFacebook marketplace access failed (${err.status}): ${err.message}`, + `\nFacebook marketplace access failed (${err.statusCode}): ${err.message}`, ); - if (err.status === 400 || err.status === 401 || err.status === 403) { + if (err.statusCode === 400 || err.statusCode === 401 || err.statusCode === 403) { logger.warn( "This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.", ); } return finalizeResults([]); } + if (err instanceof RateLimitError) { + logger.warn( + `\nFacebook marketplace access rate limited: ${err.message}`, + ); + return finalizeResults([]); + } throw err; } @@ -1261,6 +1165,8 @@ export async function fetchFacebookItem( let itemResponseUrl = itemUrl; try { const response = await fetchHtml(itemUrl, 1000, { + includeResponseUrl: true, + headers: createFacebookHeaders(cookiesHeader), onRateInfo: (remaining, reset) => { if (remaining && reset) { logger.log( @@ -1268,18 +1174,17 @@ export async function fetchFacebookItem( ); } }, - cookies: cookiesHeader, }); itemHtml = response.html; itemResponseUrl = response.responseUrl; } catch (err) { if (err instanceof HttpError) { logger.warn( - `\nFacebook marketplace item access failed (${err.status}): ${err.message}`, + `\nFacebook marketplace item access failed (${err.statusCode}): ${err.message}`, ); // Enhanced error handling based on status codes - switch (err.status) { + switch (err.statusCode) { case 400: case 401: case 403: @@ -1305,10 +1210,19 @@ export async function fetchFacebookItem( ); break; default: - logger.warn(`Unexpected error status: ${err.status}`); + logger.warn(`Unexpected error status: ${err.statusCode}`); } return null; } + if (err instanceof RateLimitError) { + logger.warn( + `\nFacebook marketplace item rate limited for item ${itemId}: ${err.message}`, + ); + logger.warn( + "Rate limited: Too many requests. Facebook is blocking access temporarily.", + ); + return null; + } throw err; } diff --git a/packages/core/src/utils/http.ts b/packages/core/src/utils/http.ts index c624c97..e8ab2f2 100644 --- a/packages/core/src/utils/http.ts +++ b/packages/core/src/utils/http.ts @@ -1,3 +1,4 @@ +import type { HTMLString } from "../types/common"; import { delay } from "./delay"; /** Custom error class for HTTP-related failures */ @@ -60,10 +61,20 @@ export function isRecord(value: unknown): value is Record { /** * Calculate exponential backoff delay with jitter */ -function calculateBackoffDelay(attempt: number, baseMs: number): number { +function calculateBackoffDelay( + attempt: number, + baseMs: number, + jitter: () => number = Math.random, +): number { const exponentialDelay = baseMs * 2 ** attempt; - const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter - return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds + const jitterDelay = jitter() * 0.1 * exponentialDelay; // 10% jitter + return Math.min(exponentialDelay + jitterDelay, 30000); // Cap at 30 seconds +} + +/** Result type when includeResponseUrl is true */ +export interface FetchHtmlResult { + html: HTMLString; + responseUrl: string; } /** Options for fetchHtml */ @@ -73,6 +84,8 @@ export interface FetchHtmlOptions { timeoutMs?: number; onRateInfo?: (remaining: string | null, reset: string | null) => void; headers?: Record; + includeResponseUrl?: boolean; + jitter?: () => number; } /** @@ -80,14 +93,24 @@ export interface FetchHtmlOptions { * @param url - The URL to fetch * @param delayMs - Delay in milliseconds between requests (rate limiting) * @param opts - Optional fetch options - * @returns The HTML content as a string + * @returns The HTML content as a string, or an object with html and responseUrl * @throws HttpError, NetworkError, or RateLimitError on failure */ +export async function fetchHtml( + url: string, + delayMs: number, + opts: FetchHtmlOptions & { includeResponseUrl: true }, +): Promise; export async function fetchHtml( url: string, delayMs: number, opts?: FetchHtmlOptions, -): Promise { +): Promise; +export async function fetchHtml( + url: string, + delayMs: number, + opts?: FetchHtmlOptions, +): Promise { const maxRetries = opts?.maxRetries ?? 3; const retryBaseMs = opts?.retryBaseMs ?? 1000; const timeoutMs = opts?.timeoutMs ?? 30000; @@ -138,10 +161,10 @@ export async function fetchHtml( : Number.NaN; const waitMs = Number.isFinite(resetSeconds) ? Math.max(0, resetSeconds * 1000) - : calculateBackoffDelay(attempt, retryBaseMs); + : calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random); if (attempt < maxRetries) { - await new Promise((resolve) => setTimeout(resolve, waitMs)); + await delay(waitMs); continue; } throw new RateLimitError( @@ -153,9 +176,7 @@ export async function fetchHtml( // Retry on server errors if (res.status >= 500 && res.status < 600 && attempt < maxRetries) { - await new Promise((resolve) => - setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)), - ); + await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random)); continue; } @@ -170,7 +191,9 @@ export async function fetchHtml( // Respect per-request delay to maintain rate limiting await delay(delayMs); - return html; + return opts?.includeResponseUrl + ? { html, responseUrl: res.url || url } + : html; } catch (err) { // Re-throw known errors if ( @@ -183,9 +206,7 @@ export async function fetchHtml( if (err instanceof Error && err.name === "AbortError") { if (attempt < maxRetries) { - await new Promise((resolve) => - setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)), - ); + await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random)); continue; } throw new NetworkError(`Request timeout for ${url}`, url, err); @@ -193,9 +214,7 @@ export async function fetchHtml( // Network or other errors if (attempt < maxRetries) { - await new Promise((resolve) => - setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)), - ); + await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random)); continue; } throw new NetworkError( diff --git a/packages/core/src/utils/logger.ts b/packages/core/src/utils/logger.ts index 30cd024..26c5c79 100644 --- a/packages/core/src/utils/logger.ts +++ b/packages/core/src/utils/logger.ts @@ -7,4 +7,7 @@ export const logger = { warn: (...args: Parameters) => { if (!isTest()) console.warn(...args); }, + error: (...args: Parameters) => { + if (!isTest()) console.error(...args); + }, }; diff --git a/packages/core/test/ebay-core.test.ts b/packages/core/test/ebay-core.test.ts index 2016fdb..69c8de1 100644 --- a/packages/core/test/ebay-core.test.ts +++ b/packages/core/test/ebay-core.test.ts @@ -32,6 +32,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(""), }), ) as unknown as typeof fetch; @@ -64,6 +65,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -88,6 +90,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -114,6 +117,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -146,6 +150,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -188,6 +193,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -214,6 +220,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -243,6 +250,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -272,6 +280,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -301,6 +310,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -343,6 +353,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -375,6 +386,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -407,6 +419,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -440,6 +453,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -467,6 +481,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -499,6 +514,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -529,6 +545,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -574,6 +591,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` @@ -612,6 +630,7 @@ describe("eBay Scraper Cookie Handling", () => { global.fetch = mock(() => Promise.resolve({ ok: true, + headers: { get: () => null }, text: () => Promise.resolve(` diff --git a/packages/core/test/http.test.ts b/packages/core/test/http.test.ts index 056b171..828c01b 100644 --- a/packages/core/test/http.test.ts +++ b/packages/core/test/http.test.ts @@ -38,4 +38,23 @@ describe("fetchHtml", () => { expect(scheduledDelays).not.toContain(1000); }); + + test("fetchHtml returns responseUrl when includeResponseUrl is true", async () => { + process.env.NODE_ENV = "test"; + global.fetch = mock(() => + Promise.resolve({ + ok: true, + status: 200, + url: "https://example.test/final", + headers: { get: () => null }, + text: () => Promise.resolve(""), + }), + ) as unknown as typeof fetch; + + const result = await fetchHtml("https://example.test", 0, { + includeResponseUrl: true, + }); + expect(result.html).toBe(""); + expect(result.responseUrl).toBe("https://example.test/final"); + }); }); From 82e7abc05709929f6fe0e5436e18c51f78634f3a Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 14:48:47 -0400 Subject: [PATCH 05/14] fix: keep shared http refactor in scope --- packages/core/src/scrapers/ebay.ts | 2 +- packages/core/src/utils/logger.ts | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/src/scrapers/ebay.ts b/packages/core/src/scrapers/ebay.ts index a0d0864..ca8b5af 100644 --- a/packages/core/src/scrapers/ebay.ts +++ b/packages/core/src/scrapers/ebay.ts @@ -512,7 +512,7 @@ export default async function fetchEbayItems( return finalizeResults(filteredListings); } catch (err) { if (err instanceof HttpError) { - logger.error( + logger.warn( `Failed to fetch eBay search (${err.statusCode}): ${err.message}`, ); return finalizeResults([]); diff --git a/packages/core/src/utils/logger.ts b/packages/core/src/utils/logger.ts index 26c5c79..30cd024 100644 --- a/packages/core/src/utils/logger.ts +++ b/packages/core/src/utils/logger.ts @@ -7,7 +7,4 @@ export const logger = { warn: (...args: Parameters) => { if (!isTest()) console.warn(...args); }, - error: (...args: Parameters) => { - if (!isTest()) console.error(...args); - }, }; From 5d86a4e54db98b2ec411205595f2e65ad142c4c9 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 14:52:08 -0400 Subject: [PATCH 06/14] fix: preserve ebay rate-limit fallback --- packages/core/src/scrapers/ebay.ts | 6 +++--- packages/core/test/ebay-core.test.ts | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/core/src/scrapers/ebay.ts b/packages/core/src/scrapers/ebay.ts index ca8b5af..06e365b 100644 --- a/packages/core/src/scrapers/ebay.ts +++ b/packages/core/src/scrapers/ebay.ts @@ -9,7 +9,7 @@ import { ensureCookies, formatCookiesForHeader, } from "../utils/cookies"; -import { fetchHtml, HttpError } from "../utils/http"; +import { fetchHtml, HttpError, RateLimitError } from "../utils/http"; import { logger } from "../utils/logger"; import { classifyUnstableListings } from "../utils/unstable"; @@ -511,9 +511,9 @@ export default async function fetchEbayItems( logger.log(`Parsed ${filteredListings.length} eBay listings.`); return finalizeResults(filteredListings); } catch (err) { - if (err instanceof HttpError) { + if (err instanceof HttpError || err instanceof RateLimitError) { logger.warn( - `Failed to fetch eBay search (${err.statusCode}): ${err.message}`, + `Failed to fetch eBay search (${err instanceof HttpError ? err.statusCode : 429}): ${err.message}`, ); return finalizeResults([]); } diff --git a/packages/core/test/ebay-core.test.ts b/packages/core/test/ebay-core.test.ts index 69c8de1..f82f56f 100644 --- a/packages/core/test/ebay-core.test.ts +++ b/packages/core/test/ebay-core.test.ts @@ -86,6 +86,21 @@ describe("eBay Scraper Cookie Handling", () => { ]); }); + test("returns empty results when eBay rate-limits the request", async () => { + global.fetch = mock(() => + Promise.resolve({ + ok: false, + status: 429, + headers: { get: () => "0" }, + text: () => Promise.resolve(""), + }), + ) as unknown as typeof fetch; + + const results = await fetchEbayItems("laptop", 1000); + + expect(results).toEqual([]); + }); + test("deduplicates repeated item links from the same card", async () => { global.fetch = mock(() => Promise.resolve({ From f5339cadf1b72b04cd453c0e5a41c35284788beb Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 21:05:36 -0400 Subject: [PATCH 07/14] style: format shared http refactor --- packages/core/src/scrapers/facebook.ts | 10 +++++---- packages/core/src/utils/http.ts | 30 ++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/packages/core/src/scrapers/facebook.ts b/packages/core/src/scrapers/facebook.ts index 12fb129..f5bc728 100644 --- a/packages/core/src/scrapers/facebook.ts +++ b/packages/core/src/scrapers/facebook.ts @@ -1071,7 +1071,11 @@ export default async function fetchFacebookItems( logger.warn( `\nFacebook marketplace access failed (${err.statusCode}): ${err.message}`, ); - if (err.statusCode === 400 || err.statusCode === 401 || err.statusCode === 403) { + if ( + err.statusCode === 400 || + err.statusCode === 401 || + err.statusCode === 403 + ) { logger.warn( "This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.", ); @@ -1079,9 +1083,7 @@ export default async function fetchFacebookItems( return finalizeResults([]); } if (err instanceof RateLimitError) { - logger.warn( - `\nFacebook marketplace access rate limited: ${err.message}`, - ); + logger.warn(`\nFacebook marketplace access rate limited: ${err.message}`); return finalizeResults([]); } throw err; diff --git a/packages/core/src/utils/http.ts b/packages/core/src/utils/http.ts index e8ab2f2..54d4f24 100644 --- a/packages/core/src/utils/http.ts +++ b/packages/core/src/utils/http.ts @@ -161,7 +161,11 @@ export async function fetchHtml( : Number.NaN; const waitMs = Number.isFinite(resetSeconds) ? Math.max(0, resetSeconds * 1000) - : calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random); + : calculateBackoffDelay( + attempt, + retryBaseMs, + opts?.jitter ?? Math.random, + ); if (attempt < maxRetries) { await delay(waitMs); @@ -176,7 +180,13 @@ export async function fetchHtml( // Retry on server errors if (res.status >= 500 && res.status < 600 && attempt < maxRetries) { - await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random)); + await delay( + calculateBackoffDelay( + attempt, + retryBaseMs, + opts?.jitter ?? Math.random, + ), + ); continue; } @@ -206,7 +216,13 @@ export async function fetchHtml( if (err instanceof Error && err.name === "AbortError") { if (attempt < maxRetries) { - await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random)); + await delay( + calculateBackoffDelay( + attempt, + retryBaseMs, + opts?.jitter ?? Math.random, + ), + ); continue; } throw new NetworkError(`Request timeout for ${url}`, url, err); @@ -214,7 +230,13 @@ export async function fetchHtml( // Network or other errors if (attempt < maxRetries) { - await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random)); + await delay( + calculateBackoffDelay( + attempt, + retryBaseMs, + opts?.jitter ?? Math.random, + ), + ); continue; } throw new NetworkError( From f95b974c7ec948c0288897f59205ed0e39817e73 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 29 Apr 2026 21:09:10 -0400 Subject: [PATCH 08/14] fix: harden shared http helper --- packages/core/src/utils/http.ts | 70 ++++++++++++++++++++++++++------- packages/core/test/http.test.ts | 64 ++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 14 deletions(-) diff --git a/packages/core/src/utils/http.ts b/packages/core/src/utils/http.ts index 54d4f24..ad4d600 100644 --- a/packages/core/src/utils/http.ts +++ b/packages/core/src/utils/http.ts @@ -71,6 +71,43 @@ function calculateBackoffDelay( return Math.min(exponentialDelay + jitterDelay, 30000); // Cap at 30 seconds } +const MAX_RATE_LIMIT_WAIT_MS = 30_000; +const MAX_DELTA_RESET_SECONDS = 86_400; + +function mergeHeaders( + defaultHeaders: Record, + customHeaders?: Record, +): Record { + const merged: Record = {}; + + for (const [key, value] of Object.entries(defaultHeaders)) { + merged[key.toLowerCase()] = value; + } + + for (const [key, value] of Object.entries(customHeaders ?? {})) { + merged[key.toLowerCase()] = value; + } + + return merged; +} + +function calculateRateLimitWaitMs( + resetHeader: string | null, + fallbackWaitMs: number, +): number { + if (!resetHeader) return fallbackWaitMs; + + const resetValue = Number(resetHeader); + if (!Number.isFinite(resetValue)) return fallbackWaitMs; + + const waitMs = + resetValue <= MAX_DELTA_RESET_SECONDS + ? resetValue * 1000 + : resetValue * 1000 - Date.now(); + + return Math.min(Math.max(0, waitMs), MAX_RATE_LIMIT_WAIT_MS); +} + /** Result type when includeResponseUrl is true */ export interface FetchHtmlResult { html: HTMLString; @@ -141,13 +178,17 @@ export async function fetchHtml( const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMs); - const res = await fetch(url, { - method: "GET", - headers: { ...defaultHeaders, ...opts?.headers }, - signal: controller.signal, - }); - - clearTimeout(timeoutId); + const res = await (async () => { + try { + return await fetch(url, { + method: "GET", + headers: mergeHeaders(defaultHeaders, opts?.headers), + signal: controller.signal, + }); + } finally { + clearTimeout(timeoutId); + } + })(); const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining"); const rateLimitReset = res.headers.get("X-RateLimit-Reset"); @@ -159,13 +200,14 @@ export async function fetchHtml( const resetSeconds = rateLimitReset ? Number(rateLimitReset) : Number.NaN; - const waitMs = Number.isFinite(resetSeconds) - ? Math.max(0, resetSeconds * 1000) - : calculateBackoffDelay( - attempt, - retryBaseMs, - opts?.jitter ?? Math.random, - ); + const waitMs = calculateRateLimitWaitMs( + rateLimitReset, + calculateBackoffDelay( + attempt, + retryBaseMs, + opts?.jitter ?? Math.random, + ), + ); if (attempt < maxRetries) { await delay(waitMs); diff --git a/packages/core/test/http.test.ts b/packages/core/test/http.test.ts index 828c01b..f385ebc 100644 --- a/packages/core/test/http.test.ts +++ b/packages/core/test/http.test.ts @@ -57,4 +57,68 @@ describe("fetchHtml", () => { expect(result.html).toBe(""); expect(result.responseUrl).toBe("https://example.test/final"); }); + + test("rate limit epoch reset uses bounded wait", async () => { + process.env.NODE_ENV = "production"; + const scheduledDelays: number[] = []; + const farFutureEpochSeconds = Math.floor(Date.now() / 1000) + 315_360_000; + let calls = 0; + + global.fetch = mock(() => { + calls += 1; + return Promise.resolve({ + ok: calls > 1, + status: calls > 1 ? 200 : 429, + url: "https://example.test", + headers: { + get: (name: string) => + name === "X-RateLimit-Reset" ? String(farFutureEpochSeconds) : null, + }, + text: () => Promise.resolve(""), + }); + }) as unknown as typeof fetch; + globalThis.setTimeout = mock((handler: TimerHandler, timeout?: number) => { + scheduledDelays.push(Number(timeout)); + if (timeout !== 1_234_567 && typeof handler === "function") { + handler(); + } + return 0 as unknown as ReturnType; + }) as unknown as typeof setTimeout; + globalThis.clearTimeout = mock(() => {}) as unknown as typeof clearTimeout; + + await fetchHtml("https://example.test", 0, { + maxRetries: 1, + timeoutMs: 1_234_567, + }); + + expect(scheduledDelays).toContain(30_000); + expect(scheduledDelays).not.toContain(farFutureEpochSeconds * 1000); + }); + + test("custom Accept header overrides default accept without duplicate casing", async () => { + process.env.NODE_ENV = "test"; + const customAccept = "text/plain"; + let requestHeaders: HeadersInit | undefined; + + global.fetch = mock((_url: string | URL | Request, init?: RequestInit) => { + requestHeaders = init?.headers; + return Promise.resolve({ + ok: true, + status: 200, + url: "https://example.test", + headers: { get: () => null }, + text: () => Promise.resolve(""), + }); + }) as unknown as typeof fetch; + + await fetchHtml("https://example.test", 0, { + headers: { Accept: customAccept }, + }); + + expect(requestHeaders).toBeDefined(); + expect((requestHeaders as Record).accept).toBe( + customAccept, + ); + expect((requestHeaders as Record).Accept).toBeUndefined(); + }); }); From 31866de787193577db423240fa1946d454a00613 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:48:15 -0400 Subject: [PATCH 09/14] refactor: clean kijiji scraper internals --- packages/core/src/scrapers/kijiji.ts | 83 ++-------------------------- 1 file changed, 5 insertions(+), 78 deletions(-) diff --git a/packages/core/src/scrapers/kijiji.ts b/packages/core/src/scrapers/kijiji.ts index 54015db..724f816 100644 --- a/packages/core/src/scrapers/kijiji.ts +++ b/packages/core/src/scrapers/kijiji.ts @@ -11,6 +11,7 @@ import { formatCookiesForHeader, loadCookiesOptional, } from "../utils/cookies"; +import { delay } from "../utils/delay"; import { formatCentsToCurrency } from "../utils/format"; import { fetchHtml, @@ -568,78 +569,6 @@ export function parseSearch( return results; } -/** - Parse a listing page into a typed object (backward compatible). -*/ -function _parseListing( - htmlString: HTMLString, - BASE_URL: string, -): KijijiListingDetails | null { - const apolloState = extractApolloState(htmlString); - if (!apolloState) return null; - - const listingKey = findApolloListingKey( - apolloState, - (value) => typeof value.url === "string" && typeof value.title === "string", - ); - if (!listingKey) return null; - - const root = apolloState[listingKey]; - if (!isRecord(root)) return null; - - const { - url, - title, - description, - price, - type, - status, - activationDate, - endDate, - metrics, - location, - } = root as ApolloListingRoot; - - const cents = price?.amount != null ? Number(price.amount) : undefined; - const amountFormatted = - cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined; - - const numberOfViews = - metrics?.views != null ? Number(metrics.views) : undefined; - - const listingUrl = - typeof url === "string" - ? url.startsWith("http") - ? url - : `${BASE_URL}${url}` - : ""; - - if (!listingUrl || !title) return null; - - return { - url: listingUrl, - title, - description, - listingPrice: amountFormatted - ? { - amountFormatted, - cents: - cents !== undefined && Number.isFinite(cents) ? cents : undefined, - currency: price?.currency, - } - : undefined, - listingType: type, - listingStatus: status, - creationDate: activationDate, - endDate, - numberOfViews: - numberOfViews !== undefined && Number.isFinite(numberOfViews) - ? numberOfViews - : undefined, - address: location?.address ?? null, - }; -} - /** * Parse a listing page into a detailed object with all available fields */ @@ -928,9 +857,7 @@ export default async function fetchKijijiItems( const batchPromises = batch.map(async (link, batchIndex) => { try { if (batchIndex > 0) { - await new Promise((resolve) => - setTimeout(resolve, DELAY_MS * batchIndex), - ); + await delay(DELAY_MS * batchIndex); } const html = await fetchHtml(link, 0, { @@ -952,11 +879,11 @@ export default async function fetchKijijiItems( return parsed; } catch (err) { if (err instanceof HttpError) { - console.error( + logger.warn( `\nFailed to fetch ${link}\n - ${err.statusCode} ${err.message}`, ); } else { - console.error( + logger.warn( `\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`, ); } @@ -974,7 +901,7 @@ export default async function fetchKijijiItems( results.push(...batchResults); if (i + CONCURRENT_REQUESTS < newListingLinks.length) { - await new Promise((resolve) => setTimeout(resolve, DELAY_MS)); + await delay(DELAY_MS); } } From c0dda57f6461cde4b9312992f7a60d8b4f29e605 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:51:13 -0400 Subject: [PATCH 10/14] test: require explicit fetch mocks --- packages/core/test/setup.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/core/test/setup.ts b/packages/core/test/setup.ts index 412f8b8..8a5b105 100644 --- a/packages/core/test/setup.ts +++ b/packages/core/test/setup.ts @@ -1,11 +1,6 @@ -// Test setup for Bun test runner -// This file is loaded before any tests run due to bunfig.toml preload - -// Mock fetch globally for tests -global.fetch = - global.fetch || - (() => { - throw new Error("fetch is not available in test environment"); - }); - -// Add any global test utilities here +global.fetch = Object.assign( + () => { + throw new Error("Tests must mock fetch explicitly"); + }, + { preconnect: fetch.preconnect }, +) as typeof fetch; From 28b3267b7dd577715730f71f3739ed9396f0229b Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:53:31 -0400 Subject: [PATCH 11/14] test: preload core fetch guard --- bunfig.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/bunfig.toml b/bunfig.toml index 11fff1e..997bd34 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -3,3 +3,4 @@ exact = true [test] root = "./do-not-run-tests-from-root" +preload = ["./packages/core/test/setup.ts"] From d1cd028f340b76924dde5bbe9e33496ca9dacad7 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:56:14 -0400 Subject: [PATCH 12/14] chore: add sentinel file for bun test root --- do-not-run-tests-from-root/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 do-not-run-tests-from-root/.gitkeep diff --git a/do-not-run-tests-from-root/.gitkeep b/do-not-run-tests-from-root/.gitkeep new file mode 100644 index 0000000..e69de29 From db173aef1b1f757e24267d4578e10399648d4828 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:58:06 -0400 Subject: [PATCH 13/14] Revert "chore: add sentinel file for bun test root" This reverts commit d1cd028f340b76924dde5bbe9e33496ca9dacad7. --- do-not-run-tests-from-root/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 do-not-run-tests-from-root/.gitkeep diff --git a/do-not-run-tests-from-root/.gitkeep b/do-not-run-tests-from-root/.gitkeep deleted file mode 100644 index e69de29..0000000 From 24e0a8266e02e99514c1d5a24b8cb83405f39721 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:58:06 -0400 Subject: [PATCH 14/14] Revert "test: preload core fetch guard" This reverts commit 28b3267b7dd577715730f71f3739ed9396f0229b. --- bunfig.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/bunfig.toml b/bunfig.toml index 997bd34..11fff1e 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -3,4 +3,3 @@ exact = true [test] root = "./do-not-run-tests-from-root" -preload = ["./packages/core/test/setup.ts"]