feat: add maxItems support to ebay scraper
This commit is contained in:
@@ -65,6 +65,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
||||
keywords,
|
||||
buyItNowOnly,
|
||||
canadaOnly,
|
||||
maxItems,
|
||||
};
|
||||
const items = hideUnstableResults
|
||||
? await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
||||
@@ -82,13 +83,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
||||
{ status: 404 },
|
||||
);
|
||||
|
||||
const results = hideUnstableResults
|
||||
? items
|
||||
: maxItems !== undefined
|
||||
? items.slice(0, maxItems)
|
||||
: items;
|
||||
|
||||
return Response.json(results, { status: 200 });
|
||||
return Response.json(items, { status: 200 });
|
||||
} catch (error) {
|
||||
console.error("eBay scraping error:", error);
|
||||
const errorMessage =
|
||||
|
||||
@@ -382,21 +382,20 @@ describe("API routes", () => {
|
||||
expect(body.message).toBe("Search didn't return any results!");
|
||||
});
|
||||
|
||||
test("ebayRoute respects maxItems=0 in default mode", async () => {
|
||||
test("ebayRoute forwards maxItems to core in default mode", async () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
fetchEbayItems.mockImplementation(() =>
|
||||
Promise.resolve([{ title: "a" }, { title: "b" }]),
|
||||
Promise.resolve([{ title: "a" }]),
|
||||
);
|
||||
|
||||
const response = await ebayRoute(
|
||||
await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&maxItems=0",
|
||||
"http://localhost/api/ebay?q=laptop&maxItems=2",
|
||||
),
|
||||
);
|
||||
|
||||
const body = await response.json();
|
||||
expect(body).toHaveLength(0);
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }));
|
||||
});
|
||||
|
||||
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
|
||||
@@ -420,6 +419,30 @@ describe("API routes", () => {
|
||||
expect(body.unstableResults).toHaveLength(2);
|
||||
expect(body.results[0].title).toBe("a");
|
||||
expect(body.unstableResults[0].title).toBe("d");
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 4 }), {
|
||||
hideUnstableResults: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
fetchEbayItems.mockImplementation(() =>
|
||||
Promise.resolve({
|
||||
results: [{ title: "a" }],
|
||||
unstableResults: [{ title: "b" }],
|
||||
}),
|
||||
);
|
||||
|
||||
await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&unstableFilter=true&maxItems=2",
|
||||
),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }), {
|
||||
hideUnstableResults: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("ebayRoute returns 404 when unstable results are empty", async () => {
|
||||
|
||||
@@ -394,6 +394,7 @@ export default async function fetchEbayItems(
|
||||
keywords?: string[];
|
||||
buyItNowOnly?: boolean;
|
||||
canadaOnly?: boolean;
|
||||
maxItems?: number;
|
||||
} | undefined,
|
||||
unstableMode: { hideUnstableResults: true },
|
||||
): Promise<UnstableListingBuckets<EbayListingDetails>>;
|
||||
@@ -408,6 +409,7 @@ export default async function fetchEbayItems(
|
||||
keywords?: string[];
|
||||
buyItNowOnly?: boolean;
|
||||
canadaOnly?: boolean;
|
||||
maxItems?: number;
|
||||
},
|
||||
unstableMode?: UnstableListingModeOptions,
|
||||
): Promise<EbayListingDetails[]>;
|
||||
@@ -422,21 +424,12 @@ export default async function fetchEbayItems(
|
||||
keywords?: string[];
|
||||
buyItNowOnly?: boolean;
|
||||
canadaOnly?: boolean;
|
||||
maxItems?: number;
|
||||
} = {},
|
||||
unstableMode: UnstableListingModeOptions = {},
|
||||
) {
|
||||
const requestsPerSecond = REQUESTS_PER_SECOND > 0 ? REQUESTS_PER_SECOND : 1;
|
||||
|
||||
const finalizeResults = (
|
||||
listings: EbayListingDetails[],
|
||||
): EbayListingDetails[] | UnstableListingBuckets<EbayListingDetails> => {
|
||||
if (!unstableMode.hideUnstableResults) {
|
||||
return listings;
|
||||
}
|
||||
|
||||
return classifyUnstableListings(listings);
|
||||
};
|
||||
|
||||
const {
|
||||
minPrice = 0,
|
||||
maxPrice = Number.MAX_SAFE_INTEGER,
|
||||
@@ -445,8 +438,22 @@ export default async function fetchEbayItems(
|
||||
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
||||
buyItNowOnly = true,
|
||||
canadaOnly = true,
|
||||
maxItems,
|
||||
} = opts;
|
||||
|
||||
const finalizeResults = (
|
||||
listings: EbayListingDetails[],
|
||||
): EbayListingDetails[] | UnstableListingBuckets<EbayListingDetails> => {
|
||||
const limitedListings =
|
||||
maxItems !== undefined ? listings.slice(0, maxItems) : listings;
|
||||
|
||||
if (!unstableMode.hideUnstableResults) {
|
||||
return limitedListings;
|
||||
}
|
||||
|
||||
return classifyUnstableListings(limitedListings);
|
||||
};
|
||||
|
||||
const cookies = await loadEbayCookies();
|
||||
|
||||
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
||||
|
||||
@@ -552,4 +552,86 @@ describe("eBay Scraper Cookie Handling", () => {
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("respects maxItems in default mode", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(`
|
||||
<html><body>
|
||||
<li class="s-item">
|
||||
<a href="https://www.ebay.ca/itm/1"></a>
|
||||
<h3>First Bundle</h3>
|
||||
<span class="s-item__price">CA $100.00</span>
|
||||
</li>
|
||||
<li class="s-item">
|
||||
<a href="https://www.ebay.ca/itm/2"></a>
|
||||
<h3>Second Bundle</h3>
|
||||
<span class="s-item__price">CA $110.00</span>
|
||||
</li>
|
||||
<li class="s-item">
|
||||
<a href="https://www.ebay.ca/itm/3"></a>
|
||||
<h3>Third Bundle</h3>
|
||||
<span class="s-item__price">CA $70.00</span>
|
||||
</li>
|
||||
</body></html>
|
||||
`),
|
||||
}),
|
||||
) as typeof fetch;
|
||||
|
||||
const results = await fetchEbayItems("laptop", 1000, { maxItems: 2 });
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0]).toEqual(
|
||||
expect.objectContaining({ title: "First Bundle" }),
|
||||
);
|
||||
expect(results[1]).toEqual(
|
||||
expect.objectContaining({ title: "Second Bundle" }),
|
||||
);
|
||||
});
|
||||
|
||||
test("respects maxItems in unstable mode", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(`
|
||||
<html><body>
|
||||
<li class="s-item">
|
||||
<a href="https://www.ebay.ca/itm/1"></a>
|
||||
<h3>First Bundle</h3>
|
||||
<span class="s-item__price">CA $100.00</span>
|
||||
</li>
|
||||
<li class="s-item">
|
||||
<a href="https://www.ebay.ca/itm/2"></a>
|
||||
<h3>Second Bundle</h3>
|
||||
<span class="s-item__price">CA $110.00</span>
|
||||
</li>
|
||||
<li class="s-item">
|
||||
<a href="https://www.ebay.ca/itm/3"></a>
|
||||
<h3>Third Bundle</h3>
|
||||
<span class="s-item__price">CA $70.00</span>
|
||||
</li>
|
||||
</body></html>
|
||||
`),
|
||||
}),
|
||||
) as typeof fetch;
|
||||
|
||||
const results = await fetchEbayItems(
|
||||
"laptop",
|
||||
1000,
|
||||
{ maxItems: 2 },
|
||||
{ hideUnstableResults: true },
|
||||
);
|
||||
|
||||
expect(results.results).toHaveLength(2);
|
||||
expect(results.unstableResults).toHaveLength(0);
|
||||
expect(results.results[0]).toEqual(
|
||||
expect.objectContaining({ title: "First Bundle" }),
|
||||
);
|
||||
expect(results.results[1]).toEqual(
|
||||
expect.objectContaining({ title: "Second Bundle" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user