old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: expose unstable mode in api routes

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-27 02:49:35 -04:00
parent 224e83ac4c
commit 3c38232cd5
4 changed files with 154 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
packages/api-server/src/routes/ebay.ts
View File

@@ -37,7 +37,24 @@ export async function ebayRoute(req: Request): Promise<Response> {
const maxItemsParam = reqUrl.searchParams.get("maxItems");
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
const items = await fetchEbayItems(SEARCH_QUERY, 1, {
const hideUnstableResults =
reqUrl.searchParams.get("unstableFilter") === "true";
const items = hideUnstableResults
? await fetchEbayItems(
SEARCH_QUERY,
1,
{
minPrice,
maxPrice,
strictMode,
exclusions,
keywords,
buyItNowOnly,
canadaOnly,
},
{ hideUnstableResults: true },
)
: await fetchEbayItems(SEARCH_QUERY, 1, {
minPrice,
maxPrice,
strictMode,
@@ -47,7 +64,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
canadaOnly,
});
const results = maxItems ? items.slice(0, maxItems) : items;
const results = hideUnstableResults || !maxItems ? items : items.slice(0, maxItems);
if (!results || results.length === 0)
return Response.json(

8
packages/api-server/src/routes/facebook.ts
View File

@@ -20,9 +20,15 @@ export async function facebookRoute(req: Request): Promise<Response> {
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
const maxItemsParam = reqUrl.searchParams.get("maxItems");
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : 25;
const hideUnstableResults =
reqUrl.searchParams.get("unstableFilter") === "true";
try {
const items = await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
const items = hideUnstableResults
? await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems, {
hideUnstableResults: true,
})
: await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },

13
packages/api-server/src/routes/kijiji.ts
View File

@@ -23,6 +23,8 @@ export async function kijijiRoute(req: Request): Promise<Response> {
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
const priceMaxParam = reqUrl.searchParams.get("priceMax");
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
const hideUnstableResults =
reqUrl.searchParams.get("unstableFilter") === "true";
const searchOptions = {
location: reqUrl.searchParams.get("location") || undefined,
@@ -45,7 +47,16 @@ export async function kijijiRoute(req: Request): Promise<Response> {
};
try {
const items = await fetchKijijiItems(
const items = hideUnstableResults
? await fetchKijijiItems(
SEARCH_QUERY,
4, // 4 requests per second for faster scraping
"https://www.kijiji.ca",
searchOptions,
{},
{ hideUnstableResults: true },
)
: await fetchKijijiItems(
SEARCH_QUERY,
4, // 4 requests per second for faster scraping
"https://www.kijiji.ca",

102
packages/api-server/test/routes.test.ts
View File

@@ -2,10 +2,12 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchKijijiItems = mock(() => Promise.resolve([{ title: "item" }]));
mock.module("@marketplace-scrapers/core", () => ({
fetchFacebookItems,
fetchEbayItems,
fetchKijijiItems,
}));
describe("API routes", () => {
@@ -18,11 +20,16 @@ describe("API routes", () => {
fetchEbayItems.mockImplementation(() =>
Promise.resolve([{ title: "item" }]),
);
fetchKijijiItems.mockReset();
fetchKijijiItems.mockImplementation(() =>
Promise.resolve([{ title: "item" }]),
);
});
afterEach(() => {
fetchFacebookItems.mockClear();
fetchEbayItems.mockClear();
fetchKijijiItems.mockClear();
});
test("facebookRoute ignores cookies query parameter", async () => {
@@ -56,4 +63,99 @@ describe("API routes", () => {
canadaOnly: true,
});
});
test("kijijiRoute ignores cookies query parameter", async () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&cookies=s%3D1&maxPages=3",
),
);
expect(fetchKijijiItems).toHaveBeenCalledWith(
"laptop",
4,
"https://www.kijiji.ca",
{
location: undefined,
category: undefined,
keywords: undefined,
sortBy: null,
sortOrder: null,
maxPages: 3,
priceMin: undefined,
priceMax: undefined,
cookies: "s=1",
},
{},
);
});
test("facebookRoute forwards unstableFilter=true to core", async () => {
const { facebookRoute } = await import("../src/routes/facebook");
await facebookRoute(
new Request(
"http://localhost/api/facebook?q=laptop&location=toronto&maxItems=3&unstableFilter=true",
),
);
expect(fetchFacebookItems).toHaveBeenCalledWith("laptop", 1, "toronto", 3, {
hideUnstableResults: true,
});
});
test("ebayRoute forwards unstableFilter=true to core", async () => {
const { ebayRoute } = await import("../src/routes/ebay");
await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&buyItNowOnly=true&unstableFilter=true",
),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
minPrice: undefined,
maxPrice: undefined,
strictMode: false,
exclusions: [],
keywords: ["laptop"],
buyItNowOnly: true,
canadaOnly: true,
}, {
hideUnstableResults: true,
});
});
test("kijijiRoute forwards unstableFilter=true to core", async () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=5&unstableFilter=true",
),
);
expect(fetchKijijiItems).toHaveBeenCalledWith(
"laptop",
4,
"https://www.kijiji.ca",
{
location: undefined,
category: undefined,
keywords: undefined,
sortBy: null,
sortOrder: null,
maxPages: 5,
priceMin: undefined,
priceMax: undefined,
cookies: undefined,
},
{},
{
hideUnstableResults: true,
},
);
});
});