chore: merge code-smell-cleanup
This commit is contained in:
@@ -11,7 +11,6 @@ import {
|
|||||||
* Search eBay for listings (default: Buy It Now only, Canada only)
|
* Search eBay for listings (default: Buy It Now only, Canada only)
|
||||||
*/
|
*/
|
||||||
export async function ebayRoute(req: Request): Promise<Response> {
|
export async function ebayRoute(req: Request): Promise<Response> {
|
||||||
try {
|
|
||||||
const reqUrl = new URL(req.url);
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
const SEARCH_QUERY = getRequiredSearchQuery(req);
|
const SEARCH_QUERY = getRequiredSearchQuery(req);
|
||||||
@@ -64,6 +63,8 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
canadaOnly,
|
canadaOnly,
|
||||||
maxItems,
|
maxItems,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
if (hideUnstableResults) {
|
if (hideUnstableResults) {
|
||||||
const items = await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
const items = await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
||||||
hideUnstableResults: true,
|
hideUnstableResults: true,
|
||||||
|
|||||||
@@ -12,6 +12,15 @@ export function getRequiredSearchQuery(req: Request): string | Response {
|
|||||||
return query;
|
return query;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function parseNonNegativeIntegerParam(
|
||||||
|
searchParams: URLSearchParams,
|
||||||
|
name: string,
|
||||||
|
defaultValue: number,
|
||||||
|
): number | Response;
|
||||||
|
export function parseNonNegativeIntegerParam(
|
||||||
|
searchParams: URLSearchParams,
|
||||||
|
name: string,
|
||||||
|
): number | undefined | Response;
|
||||||
export function parseNonNegativeIntegerParam(
|
export function parseNonNegativeIntegerParam(
|
||||||
searchParams: URLSearchParams,
|
searchParams: URLSearchParams,
|
||||||
name: string,
|
name: string,
|
||||||
@@ -21,14 +30,13 @@ export function parseNonNegativeIntegerParam(
|
|||||||
if (rawValue === null) {
|
if (rawValue === null) {
|
||||||
return defaultValue;
|
return defaultValue;
|
||||||
}
|
}
|
||||||
const value = Number(rawValue);
|
if (!/^\d+$/.test(rawValue)) {
|
||||||
if (!Number.isInteger(value) || value < 0) {
|
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: `Invalid ${name} parameter` },
|
{ message: `Invalid ${name} parameter` },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return value;
|
return Number(rawValue);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function emptySearchResponse(hint?: string): Response {
|
export function emptySearchResponse(hint?: string): Response {
|
||||||
|
|||||||
@@ -502,6 +502,66 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid maxItems parameter");
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for non-integer maxItems", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=10abc"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for decimal maxItems", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=1.5"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for empty maxItems", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxItems="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for whitespace maxItems", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for hex maxItems", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
test("facebookRoute returns 400 for invalid maxItems", async () => {
|
test("facebookRoute returns 400 for invalid maxItems", async () => {
|
||||||
const { facebookRoute } = await import("../src/routes/facebook");
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
@@ -514,7 +574,139 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid maxItems parameter");
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute rejects non-integer minPrice", async () => {
|
test("facebookRoute returns 400 for non-integer maxItems", async () => {
|
||||||
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
|
const response = await facebookRoute(
|
||||||
|
new Request("http://localhost/api/facebook?q=laptop&maxItems=10abc"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("facebookRoute returns 400 for decimal maxItems", async () => {
|
||||||
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
|
const response = await facebookRoute(
|
||||||
|
new Request("http://localhost/api/facebook?q=laptop&maxItems=1.5"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("facebookRoute returns 400 for empty maxItems", async () => {
|
||||||
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
|
const response = await facebookRoute(
|
||||||
|
new Request("http://localhost/api/facebook?q=laptop&maxItems="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("facebookRoute returns 400 for whitespace maxItems", async () => {
|
||||||
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
|
const response = await facebookRoute(
|
||||||
|
new Request("http://localhost/api/facebook?q=laptop&maxItems=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("facebookRoute returns 400 for hex maxItems", async () => {
|
||||||
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
|
const response = await facebookRoute(
|
||||||
|
new Request("http://localhost/api/facebook?q=laptop&maxItems=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxItems parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for empty minPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&minPrice="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid minPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for whitespace minPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&minPrice=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid minPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for hex minPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&minPrice=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid minPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for empty maxPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxPrice="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for whitespace maxPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxPrice=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for hex maxPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxPrice=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for non-integer minPrice", async () => {
|
||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
@@ -538,7 +730,19 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid minPrice parameter");
|
expect(body.message).toBe("Invalid minPrice parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute rejects non-integer maxPrice", async () => {
|
test("ebayRoute returns 400 for decimal minPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&minPrice=1.5"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid minPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute returns 400 for non-integer maxPrice", async () => {
|
||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
@@ -562,7 +766,19 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid maxPrice parameter");
|
expect(body.message).toBe("Invalid maxPrice parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("kijijiRoute rejects decimal maxPages", async () => {
|
test("ebayRoute returns 400 for decimal maxPrice", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
const response = await ebayRoute(
|
||||||
|
new Request("http://localhost/api/ebay?q=laptop&maxPrice=1.5"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPrice parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for decimal maxPages", async () => {
|
||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
@@ -586,6 +802,54 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid maxPages parameter");
|
expect(body.message).toBe("Invalid maxPages parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for non-integer maxPages", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages=10abc"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPages parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for empty maxPages", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPages parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for whitespace maxPages", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPages parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for hex maxPages", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid maxPages parameter");
|
||||||
|
});
|
||||||
|
|
||||||
test("kijijiRoute returns 400 for invalid priceMin", async () => {
|
test("kijijiRoute returns 400 for invalid priceMin", async () => {
|
||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
@@ -598,6 +862,66 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid priceMin parameter");
|
expect(body.message).toBe("Invalid priceMin parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for decimal priceMin", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin=1.5"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMin parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for non-integer priceMin", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin=10abc"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMin parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for empty priceMin", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMin parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for whitespace priceMin", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMin parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for hex priceMin", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMin parameter");
|
||||||
|
});
|
||||||
|
|
||||||
test("kijijiRoute returns 400 for invalid priceMax", async () => {
|
test("kijijiRoute returns 400 for invalid priceMax", async () => {
|
||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
@@ -610,16 +934,64 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Invalid priceMax parameter");
|
expect(body.message).toBe("Invalid priceMax parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("facebookRoute rejects non-integer maxItems", async () => {
|
test("kijijiRoute returns 400 for decimal priceMax", async () => {
|
||||||
const { facebookRoute } = await import("../src/routes/facebook");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await facebookRoute(
|
const response = await kijijiRoute(
|
||||||
new Request("http://localhost/api/facebook?q=laptop&maxItems=10abc"),
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax=1.5"),
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
const body = await response.json();
|
const body = await response.json();
|
||||||
expect(body.message).toBe("Invalid maxItems parameter");
|
expect(body.message).toBe("Invalid priceMax parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for non-integer priceMax", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax=10abc"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMax parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for empty priceMax", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax="),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMax parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for whitespace priceMax", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax=%20%20"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMax parameter");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("kijijiRoute returns 400 for hex priceMax", async () => {
|
||||||
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
|
const response = await kijijiRoute(
|
||||||
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax=0x10"),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(response.status).toBe(400);
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.message).toBe("Invalid priceMax parameter");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("facebookRoute returns 400 for negative maxItems", async () => {
|
test("facebookRoute returns 400 for negative maxItems", async () => {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
} from "../utils/cookies";
|
} from "../utils/cookies";
|
||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
import { solveEbayChallenge } from "../utils/ebay-challenge";
|
import { solveEbayChallenge } from "../utils/ebay-challenge";
|
||||||
|
import { fetchHtml, HttpError, RateLimitError } from "../utils/http";
|
||||||
import { logger } from "../utils/logger";
|
import { logger } from "../utils/logger";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
@@ -326,17 +327,6 @@ function parseEbayPrice(
|
|||||||
return { cents, currency };
|
return { cents, currency };
|
||||||
}
|
}
|
||||||
|
|
||||||
class HttpError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly status: number,
|
|
||||||
public readonly url: string,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "HttpError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- Parsing -----------------------------
|
// ----------------------------- Parsing -----------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -953,9 +943,9 @@ export default async function fetchEbayItems(
|
|||||||
logger.log(`Parsed ${filteredListings.length} eBay listings.`);
|
logger.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||||
return finalizeResults(filteredListings);
|
return finalizeResults(filteredListings);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError || err instanceof RateLimitError) {
|
||||||
console.error(
|
logger.warn(
|
||||||
`Failed to fetch eBay search (${err.status}): ${err.message}`,
|
`Failed to fetch eBay search (${err instanceof HttpError ? err.statusCode : 429}): ${err.message}`,
|
||||||
);
|
);
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,9 +12,8 @@ import {
|
|||||||
formatCookiesForHeader,
|
formatCookiesForHeader,
|
||||||
parseCookieString,
|
parseCookieString,
|
||||||
} from "../utils/cookies";
|
} from "../utils/cookies";
|
||||||
import { delay } from "../utils/delay";
|
|
||||||
import { formatCentsToCurrency } from "../utils/format";
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
import { isRecord } from "../utils/http";
|
import { fetchHtml, HttpError, isRecord, RateLimitError } from "../utils/http";
|
||||||
import { logger } from "../utils/logger";
|
import { logger } from "../utils/logger";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
@@ -219,17 +218,6 @@ export async function ensureFacebookCookies(): Promise<Cookie[]> {
|
|||||||
return ensureCookies(FACEBOOK_COOKIE_CONFIG);
|
return ensureCookies(FACEBOOK_COOKIE_CONFIG);
|
||||||
}
|
}
|
||||||
|
|
||||||
class HttpError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly status: number,
|
|
||||||
public readonly url: string,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "HttpError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- Extraction Metrics -----------------------------
|
// ----------------------------- Extraction Metrics -----------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -274,33 +262,11 @@ function logExtractionMetrics(success: boolean, itemId?: string) {
|
|||||||
|
|
||||||
// ----------------------------- HTTP Client -----------------------------
|
// ----------------------------- HTTP Client -----------------------------
|
||||||
|
|
||||||
/**
|
function createFacebookHeaders(cookies: string): Record<string, string> {
|
||||||
Fetch HTML with a basic retry strategy and simple rate-limit delay between calls.
|
return {
|
||||||
- Retries on 429 and 5xx
|
|
||||||
- Respects X-RateLimit-Reset when present (seconds)
|
|
||||||
- Supports custom cookies for Facebook authentication
|
|
||||||
*/
|
|
||||||
async function fetchHtml(
|
|
||||||
url: string,
|
|
||||||
DELAY_MS: number,
|
|
||||||
opts?: {
|
|
||||||
maxRetries?: number;
|
|
||||||
retryBaseMs?: number;
|
|
||||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
|
||||||
cookies?: string;
|
|
||||||
},
|
|
||||||
): Promise<{ html: HTMLString; responseUrl: string }> {
|
|
||||||
const maxRetries = opts?.maxRetries ?? 3;
|
|
||||||
const retryBaseMs = opts?.retryBaseMs ?? 500;
|
|
||||||
let lastRateLimitError: HttpError | null = null;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
||||||
try {
|
|
||||||
const headers: Record<string, string> = {
|
|
||||||
accept:
|
accept:
|
||||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||||
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
||||||
"accept-encoding": "gzip, deflate, br",
|
|
||||||
"cache-control": "no-cache",
|
"cache-control": "no-cache",
|
||||||
"upgrade-insecure-requests": "1",
|
"upgrade-insecure-requests": "1",
|
||||||
"sec-fetch-dest": "document",
|
"sec-fetch-dest": "document",
|
||||||
@@ -309,77 +275,8 @@ async function fetchHtml(
|
|||||||
"sec-fetch-user": "?1",
|
"sec-fetch-user": "?1",
|
||||||
"user-agent":
|
"user-agent":
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
cookie: cookies,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Add cookies if provided
|
|
||||||
if (opts?.cookies) {
|
|
||||||
headers.cookie = opts.cookies;
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = await fetch(url, {
|
|
||||||
method: "GET",
|
|
||||||
headers,
|
|
||||||
});
|
|
||||||
|
|
||||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
|
||||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
|
||||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
// Respect 429 reset if provided
|
|
||||||
if (res.status === 429) {
|
|
||||||
lastRateLimitError = new HttpError(
|
|
||||||
`Request failed with status ${res.status}`,
|
|
||||||
res.status,
|
|
||||||
url,
|
|
||||||
);
|
|
||||||
const resetSeconds = rateLimitReset
|
|
||||||
? Number(rateLimitReset)
|
|
||||||
: Number.NaN;
|
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
|
||||||
? Math.max(0, resetSeconds * 1000)
|
|
||||||
: (attempt + 1) * retryBaseMs;
|
|
||||||
if (attempt >= maxRetries) {
|
|
||||||
throw lastRateLimitError;
|
|
||||||
}
|
|
||||||
await delay(waitMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// For Facebook, 400 often means authentication required
|
|
||||||
// Don't retry 4xx client errors except 429
|
|
||||||
if (res.status >= 400 && res.status < 500 && res.status !== 429) {
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${res.status} (Facebook may require authentication cookies for access)`,
|
|
||||||
res.status,
|
|
||||||
url,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
// Retry on 5xx
|
|
||||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
|
||||||
await delay((attempt + 1) * retryBaseMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${res.status}`,
|
|
||||||
res.status,
|
|
||||||
url,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await res.text();
|
|
||||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
|
||||||
await delay(DELAY_MS);
|
|
||||||
return { html, responseUrl: res.url || url };
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof HttpError) {
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
if (attempt >= maxRetries) throw err;
|
|
||||||
await delay((attempt + 1) * retryBaseMs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw lastRateLimitError ?? new Error("Exhausted retries without response");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----------------------------- Parsing -----------------------------
|
// ----------------------------- Parsing -----------------------------
|
||||||
@@ -1157,6 +1054,8 @@ export default async function fetchFacebookItems(
|
|||||||
try {
|
try {
|
||||||
const response = await fetchHtml(searchUrl, DELAY_MS, {
|
const response = await fetchHtml(searchUrl, DELAY_MS, {
|
||||||
maxRetries: 3,
|
maxRetries: 3,
|
||||||
|
includeResponseUrl: true,
|
||||||
|
headers: createFacebookHeaders(cookiesHeader),
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
logger.log(
|
logger.log(
|
||||||
@@ -1164,22 +1063,29 @@ export default async function fetchFacebookItems(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cookies: cookiesHeader,
|
|
||||||
});
|
});
|
||||||
searchHtml = response.html;
|
searchHtml = response.html;
|
||||||
searchResponseUrl = response.responseUrl;
|
searchResponseUrl = response.responseUrl;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`\nFacebook marketplace access failed (${err.status}): ${err.message}`,
|
`\nFacebook marketplace access failed (${err.statusCode}): ${err.message}`,
|
||||||
);
|
);
|
||||||
if (err.status === 400 || err.status === 401 || err.status === 403) {
|
if (
|
||||||
|
err.statusCode === 400 ||
|
||||||
|
err.statusCode === 401 ||
|
||||||
|
err.statusCode === 403
|
||||||
|
) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
}
|
}
|
||||||
|
if (err instanceof RateLimitError) {
|
||||||
|
logger.warn(`\nFacebook marketplace access rate limited: ${err.message}`);
|
||||||
|
return finalizeResults([]);
|
||||||
|
}
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1261,6 +1167,8 @@ export async function fetchFacebookItem(
|
|||||||
let itemResponseUrl = itemUrl;
|
let itemResponseUrl = itemUrl;
|
||||||
try {
|
try {
|
||||||
const response = await fetchHtml(itemUrl, 1000, {
|
const response = await fetchHtml(itemUrl, 1000, {
|
||||||
|
includeResponseUrl: true,
|
||||||
|
headers: createFacebookHeaders(cookiesHeader),
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
logger.log(
|
logger.log(
|
||||||
@@ -1268,18 +1176,17 @@ export async function fetchFacebookItem(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
cookies: cookiesHeader,
|
|
||||||
});
|
});
|
||||||
itemHtml = response.html;
|
itemHtml = response.html;
|
||||||
itemResponseUrl = response.responseUrl;
|
itemResponseUrl = response.responseUrl;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
|
`\nFacebook marketplace item access failed (${err.statusCode}): ${err.message}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Enhanced error handling based on status codes
|
// Enhanced error handling based on status codes
|
||||||
switch (err.status) {
|
switch (err.statusCode) {
|
||||||
case 400:
|
case 400:
|
||||||
case 401:
|
case 401:
|
||||||
case 403:
|
case 403:
|
||||||
@@ -1305,10 +1212,19 @@ export async function fetchFacebookItem(
|
|||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
logger.warn(`Unexpected error status: ${err.status}`);
|
logger.warn(`Unexpected error status: ${err.statusCode}`);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
if (err instanceof RateLimitError) {
|
||||||
|
logger.warn(
|
||||||
|
`\nFacebook marketplace item rate limited for item ${itemId}: ${err.message}`,
|
||||||
|
);
|
||||||
|
logger.warn(
|
||||||
|
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
formatCookiesForHeader,
|
formatCookiesForHeader,
|
||||||
loadCookiesOptional,
|
loadCookiesOptional,
|
||||||
} from "../utils/cookies";
|
} from "../utils/cookies";
|
||||||
|
import { delay } from "../utils/delay";
|
||||||
import { formatCentsToCurrency } from "../utils/format";
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
import {
|
import {
|
||||||
fetchHtml,
|
fetchHtml,
|
||||||
@@ -568,78 +569,6 @@ export function parseSearch(
|
|||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
Parse a listing page into a typed object (backward compatible).
|
|
||||||
*/
|
|
||||||
function _parseListing(
|
|
||||||
htmlString: HTMLString,
|
|
||||||
BASE_URL: string,
|
|
||||||
): KijijiListingDetails | null {
|
|
||||||
const apolloState = extractApolloState(htmlString);
|
|
||||||
if (!apolloState) return null;
|
|
||||||
|
|
||||||
const listingKey = findApolloListingKey(
|
|
||||||
apolloState,
|
|
||||||
(value) => typeof value.url === "string" && typeof value.title === "string",
|
|
||||||
);
|
|
||||||
if (!listingKey) return null;
|
|
||||||
|
|
||||||
const root = apolloState[listingKey];
|
|
||||||
if (!isRecord(root)) return null;
|
|
||||||
|
|
||||||
const {
|
|
||||||
url,
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
price,
|
|
||||||
type,
|
|
||||||
status,
|
|
||||||
activationDate,
|
|
||||||
endDate,
|
|
||||||
metrics,
|
|
||||||
location,
|
|
||||||
} = root as ApolloListingRoot;
|
|
||||||
|
|
||||||
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
|
||||||
const amountFormatted =
|
|
||||||
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
|
|
||||||
|
|
||||||
const numberOfViews =
|
|
||||||
metrics?.views != null ? Number(metrics.views) : undefined;
|
|
||||||
|
|
||||||
const listingUrl =
|
|
||||||
typeof url === "string"
|
|
||||||
? url.startsWith("http")
|
|
||||||
? url
|
|
||||||
: `${BASE_URL}${url}`
|
|
||||||
: "";
|
|
||||||
|
|
||||||
if (!listingUrl || !title) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
url: listingUrl,
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
listingPrice: amountFormatted
|
|
||||||
? {
|
|
||||||
amountFormatted,
|
|
||||||
cents:
|
|
||||||
cents !== undefined && Number.isFinite(cents) ? cents : undefined,
|
|
||||||
currency: price?.currency,
|
|
||||||
}
|
|
||||||
: undefined,
|
|
||||||
listingType: type,
|
|
||||||
listingStatus: status,
|
|
||||||
creationDate: activationDate,
|
|
||||||
endDate,
|
|
||||||
numberOfViews:
|
|
||||||
numberOfViews !== undefined && Number.isFinite(numberOfViews)
|
|
||||||
? numberOfViews
|
|
||||||
: undefined,
|
|
||||||
address: location?.address ?? null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse a listing page into a detailed object with all available fields
|
* Parse a listing page into a detailed object with all available fields
|
||||||
*/
|
*/
|
||||||
@@ -938,9 +867,7 @@ export default async function fetchKijijiItems(
|
|||||||
const batchPromises = batch.map(async (link, batchIndex) => {
|
const batchPromises = batch.map(async (link, batchIndex) => {
|
||||||
try {
|
try {
|
||||||
if (batchIndex > 0) {
|
if (batchIndex > 0) {
|
||||||
await new Promise((resolve) =>
|
await delay(DELAY_MS * batchIndex);
|
||||||
setTimeout(resolve, DELAY_MS * batchIndex),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const html = await fetchHtml(link, 0, {
|
const html = await fetchHtml(link, 0, {
|
||||||
@@ -962,11 +889,11 @@ export default async function fetchKijijiItems(
|
|||||||
return parsed;
|
return parsed;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
console.error(
|
logger.warn(
|
||||||
`\nFailed to fetch ${link}\n - ${err.statusCode} ${err.message}`,
|
`\nFailed to fetch ${link}\n - ${err.statusCode} ${err.message}`,
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
console.error(
|
logger.warn(
|
||||||
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
|
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -984,7 +911,7 @@ export default async function fetchKijijiItems(
|
|||||||
results.push(...batchResults);
|
results.push(...batchResults);
|
||||||
|
|
||||||
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
|
await delay(DELAY_MS);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import type { HTMLString } from "../types/common";
|
||||||
import { delay } from "./delay";
|
import { delay } from "./delay";
|
||||||
|
|
||||||
/** Custom error class for HTTP-related failures */
|
/** Custom error class for HTTP-related failures */
|
||||||
@@ -60,10 +61,57 @@ export function isRecord(value: unknown): value is Record<string, unknown> {
|
|||||||
/**
|
/**
|
||||||
* Calculate exponential backoff delay with jitter
|
* Calculate exponential backoff delay with jitter
|
||||||
*/
|
*/
|
||||||
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
function calculateBackoffDelay(
|
||||||
|
attempt: number,
|
||||||
|
baseMs: number,
|
||||||
|
jitter: () => number = Math.random,
|
||||||
|
): number {
|
||||||
const exponentialDelay = baseMs * 2 ** attempt;
|
const exponentialDelay = baseMs * 2 ** attempt;
|
||||||
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
const jitterDelay = jitter() * 0.1 * exponentialDelay; // 10% jitter
|
||||||
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
return Math.min(exponentialDelay + jitterDelay, 30000); // Cap at 30 seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAX_RATE_LIMIT_WAIT_MS = 30_000;
|
||||||
|
const MAX_DELTA_RESET_SECONDS = 86_400;
|
||||||
|
|
||||||
|
function mergeHeaders(
|
||||||
|
defaultHeaders: Record<string, string>,
|
||||||
|
customHeaders?: Record<string, string>,
|
||||||
|
): Record<string, string> {
|
||||||
|
const merged: Record<string, string> = {};
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(defaultHeaders)) {
|
||||||
|
merged[key.toLowerCase()] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(customHeaders ?? {})) {
|
||||||
|
merged[key.toLowerCase()] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
|
function calculateRateLimitWaitMs(
|
||||||
|
resetHeader: string | null,
|
||||||
|
fallbackWaitMs: number,
|
||||||
|
): number {
|
||||||
|
if (!resetHeader) return fallbackWaitMs;
|
||||||
|
|
||||||
|
const resetValue = Number(resetHeader);
|
||||||
|
if (!Number.isFinite(resetValue)) return fallbackWaitMs;
|
||||||
|
|
||||||
|
const waitMs =
|
||||||
|
resetValue <= MAX_DELTA_RESET_SECONDS
|
||||||
|
? resetValue * 1000
|
||||||
|
: resetValue * 1000 - Date.now();
|
||||||
|
|
||||||
|
return Math.min(Math.max(0, waitMs), MAX_RATE_LIMIT_WAIT_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Result type when includeResponseUrl is true */
|
||||||
|
export interface FetchHtmlResult {
|
||||||
|
html: HTMLString;
|
||||||
|
responseUrl: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Options for fetchHtml */
|
/** Options for fetchHtml */
|
||||||
@@ -73,6 +121,8 @@ export interface FetchHtmlOptions {
|
|||||||
timeoutMs?: number;
|
timeoutMs?: number;
|
||||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
||||||
headers?: Record<string, string>;
|
headers?: Record<string, string>;
|
||||||
|
includeResponseUrl?: boolean;
|
||||||
|
jitter?: () => number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -80,14 +130,24 @@ export interface FetchHtmlOptions {
|
|||||||
* @param url - The URL to fetch
|
* @param url - The URL to fetch
|
||||||
* @param delayMs - Delay in milliseconds between requests (rate limiting)
|
* @param delayMs - Delay in milliseconds between requests (rate limiting)
|
||||||
* @param opts - Optional fetch options
|
* @param opts - Optional fetch options
|
||||||
* @returns The HTML content as a string
|
* @returns The HTML content as a string, or an object with html and responseUrl
|
||||||
* @throws HttpError, NetworkError, or RateLimitError on failure
|
* @throws HttpError, NetworkError, or RateLimitError on failure
|
||||||
*/
|
*/
|
||||||
|
export async function fetchHtml(
|
||||||
|
url: string,
|
||||||
|
delayMs: number,
|
||||||
|
opts: FetchHtmlOptions & { includeResponseUrl: true },
|
||||||
|
): Promise<FetchHtmlResult>;
|
||||||
export async function fetchHtml(
|
export async function fetchHtml(
|
||||||
url: string,
|
url: string,
|
||||||
delayMs: number,
|
delayMs: number,
|
||||||
opts?: FetchHtmlOptions,
|
opts?: FetchHtmlOptions,
|
||||||
): Promise<string> {
|
): Promise<HTMLString>;
|
||||||
|
export async function fetchHtml(
|
||||||
|
url: string,
|
||||||
|
delayMs: number,
|
||||||
|
opts?: FetchHtmlOptions,
|
||||||
|
): Promise<HTMLString | FetchHtmlResult> {
|
||||||
const maxRetries = opts?.maxRetries ?? 3;
|
const maxRetries = opts?.maxRetries ?? 3;
|
||||||
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
||||||
const timeoutMs = opts?.timeoutMs ?? 30000;
|
const timeoutMs = opts?.timeoutMs ?? 30000;
|
||||||
@@ -118,13 +178,17 @@ export async function fetchHtml(
|
|||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
|
||||||
const res = await fetch(url, {
|
const res = await (async () => {
|
||||||
|
try {
|
||||||
|
return await fetch(url, {
|
||||||
method: "GET",
|
method: "GET",
|
||||||
headers: { ...defaultHeaders, ...opts?.headers },
|
headers: mergeHeaders(defaultHeaders, opts?.headers),
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
|
} finally {
|
||||||
clearTimeout(timeoutId);
|
clearTimeout(timeoutId);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
||||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
||||||
@@ -136,12 +200,17 @@ export async function fetchHtml(
|
|||||||
const resetSeconds = rateLimitReset
|
const resetSeconds = rateLimitReset
|
||||||
? Number(rateLimitReset)
|
? Number(rateLimitReset)
|
||||||
: Number.NaN;
|
: Number.NaN;
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
const waitMs = calculateRateLimitWaitMs(
|
||||||
? Math.max(0, resetSeconds * 1000)
|
rateLimitReset,
|
||||||
: calculateBackoffDelay(attempt, retryBaseMs);
|
calculateBackoffDelay(
|
||||||
|
attempt,
|
||||||
|
retryBaseMs,
|
||||||
|
opts?.jitter ?? Math.random,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
if (attempt < maxRetries) {
|
if (attempt < maxRetries) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
await delay(waitMs);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
throw new RateLimitError(
|
throw new RateLimitError(
|
||||||
@@ -153,8 +222,12 @@ export async function fetchHtml(
|
|||||||
|
|
||||||
// Retry on server errors
|
// Retry on server errors
|
||||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
||||||
await new Promise((resolve) =>
|
await delay(
|
||||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
calculateBackoffDelay(
|
||||||
|
attempt,
|
||||||
|
retryBaseMs,
|
||||||
|
opts?.jitter ?? Math.random,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -170,7 +243,9 @@ export async function fetchHtml(
|
|||||||
|
|
||||||
// Respect per-request delay to maintain rate limiting
|
// Respect per-request delay to maintain rate limiting
|
||||||
await delay(delayMs);
|
await delay(delayMs);
|
||||||
return html;
|
return opts?.includeResponseUrl
|
||||||
|
? { html, responseUrl: res.url || url }
|
||||||
|
: html;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Re-throw known errors
|
// Re-throw known errors
|
||||||
if (
|
if (
|
||||||
@@ -183,8 +258,12 @@ export async function fetchHtml(
|
|||||||
|
|
||||||
if (err instanceof Error && err.name === "AbortError") {
|
if (err instanceof Error && err.name === "AbortError") {
|
||||||
if (attempt < maxRetries) {
|
if (attempt < maxRetries) {
|
||||||
await new Promise((resolve) =>
|
await delay(
|
||||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
calculateBackoffDelay(
|
||||||
|
attempt,
|
||||||
|
retryBaseMs,
|
||||||
|
opts?.jitter ?? Math.random,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -193,8 +272,12 @@ export async function fetchHtml(
|
|||||||
|
|
||||||
// Network or other errors
|
// Network or other errors
|
||||||
if (attempt < maxRetries) {
|
if (attempt < maxRetries) {
|
||||||
await new Promise((resolve) =>
|
await delay(
|
||||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
calculateBackoffDelay(
|
||||||
|
attempt,
|
||||||
|
retryBaseMs,
|
||||||
|
opts?.jitter ?? Math.random,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () => Promise.resolve("<html><body></body></html>"),
|
text: () => Promise.resolve("<html><body></body></html>"),
|
||||||
}),
|
}),
|
||||||
) as unknown as typeof fetch;
|
) as unknown as typeof fetch;
|
||||||
@@ -70,6 +71,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -90,10 +92,26 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("returns empty results when eBay rate-limits the request", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 429,
|
||||||
|
headers: { get: () => "0" },
|
||||||
|
text: () => Promise.resolve(""),
|
||||||
|
}),
|
||||||
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
const results = await fetchEbayItems("laptop", 1000);
|
||||||
|
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
test("deduplicates repeated item links from the same card", async () => {
|
test("deduplicates repeated item links from the same card", async () => {
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -120,6 +138,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -152,6 +171,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -194,6 +214,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -295,6 +316,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -324,6 +346,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -353,6 +376,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -382,6 +406,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -424,6 +449,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -456,6 +482,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -488,6 +515,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -521,6 +549,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -548,6 +577,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -580,6 +610,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -610,6 +641,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -655,6 +687,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
@@ -693,6 +726,7 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
headers: { get: () => null },
|
||||||
text: () =>
|
text: () =>
|
||||||
Promise.resolve(`
|
Promise.resolve(`
|
||||||
<html><body>
|
<html><body>
|
||||||
|
|||||||
@@ -38,4 +38,87 @@ describe("fetchHtml", () => {
|
|||||||
|
|
||||||
expect(scheduledDelays).not.toContain(1000);
|
expect(scheduledDelays).not.toContain(1000);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("fetchHtml returns responseUrl when includeResponseUrl is true", async () => {
|
||||||
|
process.env.NODE_ENV = "test";
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
url: "https://example.test/final",
|
||||||
|
headers: { get: () => null },
|
||||||
|
text: () => Promise.resolve("<html></html>"),
|
||||||
|
}),
|
||||||
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
const result = await fetchHtml("https://example.test", 0, {
|
||||||
|
includeResponseUrl: true,
|
||||||
|
});
|
||||||
|
expect(result.html).toBe("<html></html>");
|
||||||
|
expect(result.responseUrl).toBe("https://example.test/final");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rate limit epoch reset uses bounded wait", async () => {
|
||||||
|
process.env.NODE_ENV = "production";
|
||||||
|
const scheduledDelays: number[] = [];
|
||||||
|
const farFutureEpochSeconds = Math.floor(Date.now() / 1000) + 315_360_000;
|
||||||
|
let calls = 0;
|
||||||
|
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
calls += 1;
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: calls > 1,
|
||||||
|
status: calls > 1 ? 200 : 429,
|
||||||
|
url: "https://example.test",
|
||||||
|
headers: {
|
||||||
|
get: (name: string) =>
|
||||||
|
name === "X-RateLimit-Reset" ? String(farFutureEpochSeconds) : null,
|
||||||
|
},
|
||||||
|
text: () => Promise.resolve("<html></html>"),
|
||||||
|
});
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
globalThis.setTimeout = mock((handler: TimerHandler, timeout?: number) => {
|
||||||
|
scheduledDelays.push(Number(timeout));
|
||||||
|
if (timeout !== 1_234_567 && typeof handler === "function") {
|
||||||
|
handler();
|
||||||
|
}
|
||||||
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||||
|
}) as unknown as typeof setTimeout;
|
||||||
|
globalThis.clearTimeout = mock(() => {}) as unknown as typeof clearTimeout;
|
||||||
|
|
||||||
|
await fetchHtml("https://example.test", 0, {
|
||||||
|
maxRetries: 1,
|
||||||
|
timeoutMs: 1_234_567,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(scheduledDelays).toContain(30_000);
|
||||||
|
expect(scheduledDelays).not.toContain(farFutureEpochSeconds * 1000);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("custom Accept header overrides default accept without duplicate casing", async () => {
|
||||||
|
process.env.NODE_ENV = "test";
|
||||||
|
const customAccept = "text/plain";
|
||||||
|
let requestHeaders: HeadersInit | undefined;
|
||||||
|
|
||||||
|
global.fetch = mock((_url: string | URL | Request, init?: RequestInit) => {
|
||||||
|
requestHeaders = init?.headers;
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
url: "https://example.test",
|
||||||
|
headers: { get: () => null },
|
||||||
|
text: () => Promise.resolve("<html></html>"),
|
||||||
|
});
|
||||||
|
}) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
await fetchHtml("https://example.test", 0, {
|
||||||
|
headers: { Accept: customAccept },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(requestHeaders).toBeDefined();
|
||||||
|
expect((requestHeaders as Record<string, string>).accept).toBe(
|
||||||
|
customAccept,
|
||||||
|
);
|
||||||
|
expect((requestHeaders as Record<string, string>).Accept).toBeUndefined();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,11 +1,6 @@
|
|||||||
// Test setup for Bun test runner
|
global.fetch = Object.assign(
|
||||||
// This file is loaded before any tests run due to bunfig.toml preload
|
() => {
|
||||||
|
throw new Error("Tests must mock fetch explicitly");
|
||||||
// Mock fetch globally for tests
|
},
|
||||||
global.fetch =
|
{ preconnect: fetch.preconnect },
|
||||||
global.fetch ||
|
) as typeof fetch;
|
||||||
(() => {
|
|
||||||
throw new Error("fetch is not available in test environment");
|
|
||||||
});
|
|
||||||
|
|
||||||
// Add any global test utilities here
|
|
||||||
|
|||||||
@@ -2,7 +2,32 @@ import { logger } from "../logger";
|
|||||||
import { tools } from "./tools";
|
import { tools } from "./tools";
|
||||||
|
|
||||||
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
|
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
|
||||||
const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; // 3 minutes default
|
const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000;
|
||||||
|
|
||||||
|
async function callMarketplaceApi(
|
||||||
|
marketplace: string,
|
||||||
|
params: URLSearchParams,
|
||||||
|
): Promise<unknown> {
|
||||||
|
const url = `${API_BASE_URL}/${marketplace}?${params.toString()}`;
|
||||||
|
logger.log(`[MCP] Calling ${marketplace} API`);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(url),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() => reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
logger.error(
|
||||||
|
`[MCP] ${marketplace} API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Handle MCP JSON-RPC 2.0 protocol requests
|
* Handle MCP JSON-RPC 2.0 protocol requests
|
||||||
@@ -167,31 +192,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
if (args.unstableFilter !== undefined)
|
if (args.unstableFilter !== undefined)
|
||||||
params.append("unstableFilter", args.unstableFilter.toString());
|
params.append("unstableFilter", args.unstableFilter.toString());
|
||||||
|
|
||||||
logger.log(
|
result = await callMarketplaceApi("facebook", params);
|
||||||
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
|
|
||||||
);
|
|
||||||
const response = await Promise.race([
|
|
||||||
fetch(`${API_BASE_URL}/facebook?${params.toString()}`),
|
|
||||||
new Promise<Response>((_, reject) =>
|
|
||||||
setTimeout(
|
|
||||||
() =>
|
|
||||||
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
|
||||||
API_TIMEOUT,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text();
|
|
||||||
logger.error(
|
|
||||||
`[MCP] Facebook API error ${response.status}: ${errorText}`,
|
|
||||||
);
|
|
||||||
throw new Error(`API returned ${response.status}: ${errorText}`);
|
|
||||||
}
|
|
||||||
result = await response.json();
|
|
||||||
logger.log(
|
|
||||||
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
|
|
||||||
);
|
|
||||||
} else if (name === "search_ebay") {
|
} else if (name === "search_ebay") {
|
||||||
const query = args.query;
|
const query = args.query;
|
||||||
if (!query) {
|
if (!query) {
|
||||||
@@ -221,31 +222,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
if (args.unstableFilter !== undefined)
|
if (args.unstableFilter !== undefined)
|
||||||
params.append("unstableFilter", args.unstableFilter.toString());
|
params.append("unstableFilter", args.unstableFilter.toString());
|
||||||
|
|
||||||
logger.log(
|
result = await callMarketplaceApi("ebay", params);
|
||||||
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
|
|
||||||
);
|
|
||||||
const response = await Promise.race([
|
|
||||||
fetch(`${API_BASE_URL}/ebay?${params.toString()}`),
|
|
||||||
new Promise<Response>((_, reject) =>
|
|
||||||
setTimeout(
|
|
||||||
() =>
|
|
||||||
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
|
||||||
API_TIMEOUT,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
]);
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text();
|
|
||||||
logger.error(
|
|
||||||
`[MCP] eBay API error ${response.status}: ${errorText}`,
|
|
||||||
);
|
|
||||||
throw new Error(`API returned ${response.status}: ${errorText}`);
|
|
||||||
}
|
|
||||||
result = await response.json();
|
|
||||||
logger.log(
|
|
||||||
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
return Response.json({
|
return Response.json({
|
||||||
jsonrpc: "2.0",
|
jsonrpc: "2.0",
|
||||||
|
|||||||
@@ -152,6 +152,33 @@ describe("MCP protocol unstableFilter", () => {
|
|||||||
expect(String(calledUrl)).toContain("unstableFilter=true");
|
expect(String(calledUrl)).toContain("unstableFilter=true");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("tools/call returns API JSON as text content", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve(
|
||||||
|
new Response(JSON.stringify([{ title: "item" }]), { status: 200 }),
|
||||||
|
),
|
||||||
|
) as unknown as typeof fetch;
|
||||||
|
|
||||||
|
const response = await handleMcpRequest(
|
||||||
|
new Request("http://localhost", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id: 1,
|
||||||
|
method: "tools/call",
|
||||||
|
params: {
|
||||||
|
name: "search_facebook",
|
||||||
|
arguments: { query: "laptop" },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const body = await response.json();
|
||||||
|
expect(body.result.content[0].type).toBe("text");
|
||||||
|
expect(JSON.parse(body.result.content[0].text)).toEqual([{ title: "item" }]);
|
||||||
|
});
|
||||||
|
|
||||||
test("handler should forward unstableFilter=true for search_ebay", async () => {
|
test("handler should forward unstableFilter=true for search_ebay", async () => {
|
||||||
await handleMcpRequest(
|
await handleMcpRequest(
|
||||||
new Request("http://localhost", {
|
new Request("http://localhost", {
|
||||||
|
|||||||
Reference in New Issue
Block a user