old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(api): preserve unstable buckets

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-28 21:34:47 -04:00
parent 3fe5fdb63f
commit c6c44a0914
4 changed files with 89 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
packages/api-server/src/routes/ebay.ts
View File

@@ -21,7 +21,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
const minPriceParam = reqUrl.searchParams.get("minPrice"); const minPriceParam = reqUrl.searchParams.get("minPrice");
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined; const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
if (minPriceParam && (Number.isNaN(minPrice) || minPrice < 0)) { if (minPriceParam && (Number.isNaN(minPrice) || (minPrice ?? 0) < 0)) {
return Response.json( return Response.json(
{ message: "Invalid minPrice parameter" }, { message: "Invalid minPrice parameter" },
{ status: 400 }, { status: 400 },
@@ -29,7 +29,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
} }
const maxPriceParam = reqUrl.searchParams.get("maxPrice"); const maxPriceParam = reqUrl.searchParams.get("maxPrice");
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined; const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
if (maxPriceParam && (Number.isNaN(maxPrice) || maxPrice < 0)) { if (maxPriceParam && (Number.isNaN(maxPrice) || (maxPrice ?? 0) < 0)) {
return Response.json( return Response.json(
{ message: "Invalid maxPrice parameter" }, { message: "Invalid maxPrice parameter" },
{ status: 400 }, { status: 400 },
@@ -49,7 +49,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
const maxItemsParam = reqUrl.searchParams.get("maxItems"); const maxItemsParam = reqUrl.searchParams.get("maxItems");
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined; const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
if (maxItemsParam && (Number.isNaN(maxItems) || maxItems < 0)) { if (maxItemsParam && (Number.isNaN(maxItems) || (maxItems ?? 0) < 0)) {
return Response.json( return Response.json(
{ message: "Invalid maxItems parameter" }, { message: "Invalid maxItems parameter" },
{ status: 400 }, { status: 400 },
@@ -67,22 +67,27 @@ export async function ebayRoute(req: Request): Promise<Response> {
canadaOnly, canadaOnly,
maxItems, maxItems,
}; };
const items = hideUnstableResults if (hideUnstableResults) {
? await fetchEbayItems(SEARCH_QUERY, 1, opts, { const items = await fetchEbayItems(SEARCH_QUERY, 1, opts, {
hideUnstableResults: true, hideUnstableResults: true,
}) });
: await fetchEbayItems(SEARCH_QUERY, 1, opts); if (items.results.length === 0 && items.unstableResults.length === 0) {
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
}
return Response.json(items, { status: 200 });
}
const isEmpty = hideUnstableResults const items = await fetchEbayItems(SEARCH_QUERY, 1, opts);
? items.results.length === 0 && items.unstableResults.length === 0 const isEmpty = !items || items.length === 0;
: !items || items.length === 0;
if (isEmpty) if (isEmpty)
return Response.json( return Response.json(
{ message: "Search didn't return any results!" }, { message: "Search didn't return any results!" },
{ status: 404 }, { status: 404 },
); );
return Response.json(items, { status: 200 }); return Response.json(items, { status: 200 });
} catch (error) { } catch (error) {
console.error("eBay scraping error:", error); console.error("eBay scraping error:", error);

28
packages/api-server/src/routes/facebook.ts
View File

@@ -30,17 +30,27 @@ export async function facebookRoute(req: Request): Promise<Response> {
reqUrl.searchParams.get("unstableFilter") === "true"; reqUrl.searchParams.get("unstableFilter") === "true";
try { try {
const items = hideUnstableResults if (hideUnstableResults) {
? await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems, { const items = await fetchFacebookItems(
SEARCH_QUERY,
1,
LOCATION,
maxItems,
{
hideUnstableResults: true, hideUnstableResults: true,
}) },
: await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems); );
if (items.results.length === 0 && items.unstableResults.length === 0) {
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
}
return Response.json(items, { status: 200 });
}
const isEmpty = hideUnstableResults const items = await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
? items.results.length === 0 && items.unstableResults.length === 0 if (!items || items.length === 0)
: !items || items.length === 0;
if (isEmpty)
return Response.json( return Response.json(
{ message: "Search didn't return any results!" }, { message: "Search didn't return any results!" },
{ status: 404 }, { status: 404 },

48
packages/api-server/src/routes/kijiji.ts
View File

@@ -27,7 +27,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
} }
const priceMinParam = reqUrl.searchParams.get("priceMin"); const priceMinParam = reqUrl.searchParams.get("priceMin");
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined; const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
if (priceMinParam && (Number.isNaN(priceMin) || priceMin < 0)) { if (priceMinParam && (Number.isNaN(priceMin) || (priceMin ?? 0) < 0)) {
return Response.json( return Response.json(
{ message: "Invalid priceMin parameter" }, { message: "Invalid priceMin parameter" },
{ status: 400 }, { status: 400 },
@@ -35,7 +35,7 @@ export async function kijijiRoute(req: Request): Promise<Response> {
} }
const priceMaxParam = reqUrl.searchParams.get("priceMax"); const priceMaxParam = reqUrl.searchParams.get("priceMax");
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined; const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
if (priceMaxParam && (Number.isNaN(priceMax) || priceMax < 0)) { if (priceMaxParam && (Number.isNaN(priceMax) || (priceMax ?? 0) < 0)) {
return Response.json( return Response.json(
{ message: "Invalid priceMax parameter" }, { message: "Invalid priceMax parameter" },
{ status: 400 }, { status: 400 },
@@ -65,28 +65,32 @@ export async function kijijiRoute(req: Request): Promise<Response> {
}; };
try { try {
const items = hideUnstableResults if (hideUnstableResults) {
? await fetchKijijiItems( const items = await fetchKijijiItems(
SEARCH_QUERY, SEARCH_QUERY,
4, // 4 requests per second for faster scraping 4, // 4 requests per second for faster scraping
"https://www.kijiji.ca", "https://www.kijiji.ca",
searchOptions, searchOptions,
{}, {},
{ hideUnstableResults: true }, { hideUnstableResults: true },
) );
: await fetchKijijiItems( if (items.results.length === 0 && items.unstableResults.length === 0) {
SEARCH_QUERY, return Response.json(
4, // 4 requests per second for faster scraping { message: "Search didn't return any results!" },
"https://www.kijiji.ca", { status: 404 },
searchOptions,
{},
); );
}
return Response.json(items, { status: 200 });
}
const isEmpty = hideUnstableResults const items = await fetchKijijiItems(
? items.results.length === 0 && items.unstableResults.length === 0 SEARCH_QUERY,
: !items || items.length === 0; 4, // 4 requests per second for faster scraping
"https://www.kijiji.ca",
if (isEmpty) searchOptions,
{},
);
if (!items || items.length === 0)
return Response.json( return Response.json(
{ message: "Search didn't return any results!" }, { message: "Search didn't return any results!" },
{ status: 404 }, { status: 404 },

36
packages/api-server/test/routes.test.ts
View File

@@ -1,8 +1,23 @@
import { beforeEach, describe, expect, mock, test } from "bun:test"; import { beforeEach, describe, expect, mock, test } from "bun:test";
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }])); const fetchFacebookItems = mock(
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }])); (): Promise<
const fetchKijijiItems = mock(() => Promise.resolve([{ title: "item" }])); | { title: string }[]
| { results: { title: string }[]; unstableResults: { title: string }[] }
> => Promise.resolve([{ title: "item" }]),
);
const fetchEbayItems = mock(
(): Promise<
| { title: string }[]
| { results: { title: string }[]; unstableResults: { title: string }[] }
> => Promise.resolve([{ title: "item" }]),
);
const fetchKijijiItems = mock(
(): Promise<
| { title: string }[]
| { results: { title: string }[]; unstableResults: { title: string }[] }
> => Promise.resolve([{ title: "item" }]),
);
mock.module("@marketplace-scrapers/core", () => ({ mock.module("@marketplace-scrapers/core", () => ({
fetchFacebookItems, fetchFacebookItems,
@@ -13,16 +28,19 @@ mock.module("@marketplace-scrapers/core", () => ({
describe("API routes", () => { describe("API routes", () => {
beforeEach(() => { beforeEach(() => {
fetchFacebookItems.mockReset(); fetchFacebookItems.mockReset();
fetchFacebookItems.mockImplementation(() => fetchFacebookItems.mockImplementation(
Promise.resolve([{ title: "item" }]), () =>
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
); );
fetchEbayItems.mockReset(); fetchEbayItems.mockReset();
fetchEbayItems.mockImplementation(() => fetchEbayItems.mockImplementation(
Promise.resolve([{ title: "item" }]), () =>
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
); );
fetchKijijiItems.mockReset(); fetchKijijiItems.mockReset();
fetchKijijiItems.mockImplementation(() => fetchKijijiItems.mockImplementation(
Promise.resolve([{ title: "item" }]), () =>
Promise.resolve([{ title: "item" }]) as Promise<{ title: string }[]>,
); );
}); });