old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: increase kijiji scraping request rate to 4 rps

Browse Source 
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
This commit is contained in:
Dmytro Stanchiev
2026-01-23 14:41:56 -05:00
parent f3839aba54
commit 65eb8d1724
3 changed files with 165 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
packages/mcp-server/src/protocol/handler.ts
View File

@@ -1,10 +1,8 @@
import {
fetchEbayItems,
fetchFacebookItems,
fetchKijijiItems,
} from "@marketplace-scrapers/core";
import { tools } from "./tools";
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; // 3 minutes default
/**
* Handle MCP JSON-RPC 2.0 protocol requests
*/
@@ -105,32 +103,44 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
error: { code: -32602, message: "query parameter is required" },
});
}
const searchOptions = {
location: args.location,
category: args.category,
keywords: args.keywords,
sortBy: args.sortBy,
sortOrder: args.sortOrder,
maxPages: args.maxPages || 5,
priceMin: args.priceMin,
priceMax: args.priceMax,
};
const items = await Promise.race([
fetchKijijiItems(
query,
4,
"https://www.kijiji.ca",
searchOptions,
{},
),
new Promise((_, reject) =>
const params = new URLSearchParams({ q: query });
if (args.location) params.append("location", args.location);
if (args.category) params.append("category", args.category);
if (args.keywords) params.append("keywords", args.keywords);
if (args.sortBy) params.append("sortBy", args.sortBy);
if (args.sortOrder) params.append("sortOrder", args.sortOrder);
if (args.maxPages)
params.append("maxPages", args.maxPages.toString());
if (args.priceMin)
params.append("priceMin", args.priceMin.toString());
if (args.priceMax)
params.append("priceMax", args.priceMax.toString());
console.log(
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
);
const response = await Promise.race([
fetch(`${API_BASE_URL}/kijiji?${params.toString()}`),
new Promise<Response>((_, reject) =>
setTimeout(
() => reject(new Error("Request timed out after 60 seconds")),
60000,
() =>
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
API_TIMEOUT,
),
),
]);
result = items || [];
if (!response.ok) {
const errorText = await response.text();
console.error(
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else if (name === "search_facebook") {
const query = args.query;
if (!query) {
@@ -140,23 +150,37 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
error: { code: -32602, message: "query parameter is required" },
});
}
const items = await Promise.race([
fetchFacebookItems(
query,
1,
args.location || "toronto",
args.maxItems || 25,
args.cookiesSource,
undefined,
),
new Promise((_, reject) =>
const params = new URLSearchParams({ q: query });
if (args.location) params.append("location", args.location);
if (args.maxItems)
params.append("maxItems", args.maxItems.toString());
if (args.cookiesSource) params.append("cookies", args.cookiesSource);
console.log(
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
);
const response = await Promise.race([
fetch(`${API_BASE_URL}/facebook?${params.toString()}`),
new Promise<Response>((_, reject) =>
setTimeout(
() => reject(new Error("Request timed out after 60 seconds")),
60000,
() =>
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
API_TIMEOUT,
),
),
]);
result = items || [];
if (!response.ok) {
const errorText = await response.text();
console.error(
`[MCP] Facebook API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else if (name === "search_ebay") {
const query = args.query;
if (!query) {
@@ -166,26 +190,49 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
error: { code: -32602, message: "query parameter is required" },
});
}
const items = await Promise.race([
fetchEbayItems(query, 1, {
minPrice: args.minPrice,
maxPrice: args.maxPrice,
strictMode: args.strictMode || false,
exclusions: args.exclusions || [],
keywords: args.keywords || [query],
buyItNowOnly: args.buyItNowOnly !== false,
canadaOnly: args.canadaOnly !== false,
}),
new Promise((_, reject) =>
const params = new URLSearchParams({ q: query });
if (args.minPrice)
params.append("minPrice", args.minPrice.toString());
if (args.maxPrice)
params.append("maxPrice", args.maxPrice.toString());
if (args.strictMode !== undefined)
params.append("strictMode", args.strictMode.toString());
if (args.exclusions?.length)
params.append("exclusions", args.exclusions.join(","));
if (args.keywords?.length)
params.append("keywords", args.keywords.join(","));
if (args.buyItNowOnly !== undefined)
params.append("buyItNowOnly", args.buyItNowOnly.toString());
if (args.canadaOnly !== undefined)
params.append("canadaOnly", args.canadaOnly.toString());
if (args.maxItems)
params.append("maxItems", args.maxItems.toString());
console.log(
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
);
const response = await Promise.race([
fetch(`${API_BASE_URL}/ebay?${params.toString()}`),
new Promise<Response>((_, reject) =>
setTimeout(
() => reject(new Error("Request timed out after 60 seconds")),
60000,
() =>
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
API_TIMEOUT,
),
),
]);
const results = args.maxItems ? items.slice(0, args.maxItems) : items;
result = results || [];
if (!response.ok) {
const errorText = await response.text();
console.error(
`[MCP] eBay API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else {
return Response.json({
jsonrpc: "2.0",