feat: add maxItems support to ebay scraper
This commit is contained in:
@@ -65,6 +65,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
keywords,
|
keywords,
|
||||||
buyItNowOnly,
|
buyItNowOnly,
|
||||||
canadaOnly,
|
canadaOnly,
|
||||||
|
maxItems,
|
||||||
};
|
};
|
||||||
const items = hideUnstableResults
|
const items = hideUnstableResults
|
||||||
? await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
? await fetchEbayItems(SEARCH_QUERY, 1, opts, {
|
||||||
@@ -82,13 +83,7 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
|
|
||||||
const results = hideUnstableResults
|
return Response.json(items, { status: 200 });
|
||||||
? items
|
|
||||||
: maxItems !== undefined
|
|
||||||
? items.slice(0, maxItems)
|
|
||||||
: items;
|
|
||||||
|
|
||||||
return Response.json(results, { status: 200 });
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("eBay scraping error:", error);
|
console.error("eBay scraping error:", error);
|
||||||
const errorMessage =
|
const errorMessage =
|
||||||
|
|||||||
@@ -382,21 +382,20 @@ describe("API routes", () => {
|
|||||||
expect(body.message).toBe("Search didn't return any results!");
|
expect(body.message).toBe("Search didn't return any results!");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute respects maxItems=0 in default mode", async () => {
|
test("ebayRoute forwards maxItems to core in default mode", async () => {
|
||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
fetchEbayItems.mockImplementation(() =>
|
fetchEbayItems.mockImplementation(() =>
|
||||||
Promise.resolve([{ title: "a" }, { title: "b" }]),
|
Promise.resolve([{ title: "a" }]),
|
||||||
);
|
);
|
||||||
|
|
||||||
const response = await ebayRoute(
|
await ebayRoute(
|
||||||
new Request(
|
new Request(
|
||||||
"http://localhost/api/ebay?q=laptop&maxItems=0",
|
"http://localhost/api/ebay?q=laptop&maxItems=2",
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
const body = await response.json();
|
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }));
|
||||||
expect(body).toHaveLength(0);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
|
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
|
||||||
@@ -420,6 +419,30 @@ describe("API routes", () => {
|
|||||||
expect(body.unstableResults).toHaveLength(2);
|
expect(body.unstableResults).toHaveLength(2);
|
||||||
expect(body.results[0].title).toBe("a");
|
expect(body.results[0].title).toBe("a");
|
||||||
expect(body.unstableResults[0].title).toBe("d");
|
expect(body.unstableResults[0].title).toBe("d");
|
||||||
|
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 4 }), {
|
||||||
|
hideUnstableResults: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
|
||||||
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
|
fetchEbayItems.mockImplementation(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
results: [{ title: "a" }],
|
||||||
|
unstableResults: [{ title: "b" }],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await ebayRoute(
|
||||||
|
new Request(
|
||||||
|
"http://localhost/api/ebay?q=laptop&unstableFilter=true&maxItems=2",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }), {
|
||||||
|
hideUnstableResults: true,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute returns 404 when unstable results are empty", async () => {
|
test("ebayRoute returns 404 when unstable results are empty", async () => {
|
||||||
|
|||||||
@@ -394,6 +394,7 @@ export default async function fetchEbayItems(
|
|||||||
keywords?: string[];
|
keywords?: string[];
|
||||||
buyItNowOnly?: boolean;
|
buyItNowOnly?: boolean;
|
||||||
canadaOnly?: boolean;
|
canadaOnly?: boolean;
|
||||||
|
maxItems?: number;
|
||||||
} | undefined,
|
} | undefined,
|
||||||
unstableMode: { hideUnstableResults: true },
|
unstableMode: { hideUnstableResults: true },
|
||||||
): Promise<UnstableListingBuckets<EbayListingDetails>>;
|
): Promise<UnstableListingBuckets<EbayListingDetails>>;
|
||||||
@@ -408,6 +409,7 @@ export default async function fetchEbayItems(
|
|||||||
keywords?: string[];
|
keywords?: string[];
|
||||||
buyItNowOnly?: boolean;
|
buyItNowOnly?: boolean;
|
||||||
canadaOnly?: boolean;
|
canadaOnly?: boolean;
|
||||||
|
maxItems?: number;
|
||||||
},
|
},
|
||||||
unstableMode?: UnstableListingModeOptions,
|
unstableMode?: UnstableListingModeOptions,
|
||||||
): Promise<EbayListingDetails[]>;
|
): Promise<EbayListingDetails[]>;
|
||||||
@@ -422,21 +424,12 @@ export default async function fetchEbayItems(
|
|||||||
keywords?: string[];
|
keywords?: string[];
|
||||||
buyItNowOnly?: boolean;
|
buyItNowOnly?: boolean;
|
||||||
canadaOnly?: boolean;
|
canadaOnly?: boolean;
|
||||||
|
maxItems?: number;
|
||||||
} = {},
|
} = {},
|
||||||
unstableMode: UnstableListingModeOptions = {},
|
unstableMode: UnstableListingModeOptions = {},
|
||||||
) {
|
) {
|
||||||
const requestsPerSecond = REQUESTS_PER_SECOND > 0 ? REQUESTS_PER_SECOND : 1;
|
const requestsPerSecond = REQUESTS_PER_SECOND > 0 ? REQUESTS_PER_SECOND : 1;
|
||||||
|
|
||||||
const finalizeResults = (
|
|
||||||
listings: EbayListingDetails[],
|
|
||||||
): EbayListingDetails[] | UnstableListingBuckets<EbayListingDetails> => {
|
|
||||||
if (!unstableMode.hideUnstableResults) {
|
|
||||||
return listings;
|
|
||||||
}
|
|
||||||
|
|
||||||
return classifyUnstableListings(listings);
|
|
||||||
};
|
|
||||||
|
|
||||||
const {
|
const {
|
||||||
minPrice = 0,
|
minPrice = 0,
|
||||||
maxPrice = Number.MAX_SAFE_INTEGER,
|
maxPrice = Number.MAX_SAFE_INTEGER,
|
||||||
@@ -445,8 +438,22 @@ export default async function fetchEbayItems(
|
|||||||
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
||||||
buyItNowOnly = true,
|
buyItNowOnly = true,
|
||||||
canadaOnly = true,
|
canadaOnly = true,
|
||||||
|
maxItems,
|
||||||
} = opts;
|
} = opts;
|
||||||
|
|
||||||
|
const finalizeResults = (
|
||||||
|
listings: EbayListingDetails[],
|
||||||
|
): EbayListingDetails[] | UnstableListingBuckets<EbayListingDetails> => {
|
||||||
|
const limitedListings =
|
||||||
|
maxItems !== undefined ? listings.slice(0, maxItems) : listings;
|
||||||
|
|
||||||
|
if (!unstableMode.hideUnstableResults) {
|
||||||
|
return limitedListings;
|
||||||
|
}
|
||||||
|
|
||||||
|
return classifyUnstableListings(limitedListings);
|
||||||
|
};
|
||||||
|
|
||||||
const cookies = await loadEbayCookies();
|
const cookies = await loadEbayCookies();
|
||||||
|
|
||||||
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
||||||
|
|||||||
@@ -552,4 +552,86 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
],
|
],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("respects maxItems in default mode", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(`
|
||||||
|
<html><body>
|
||||||
|
<li class="s-item">
|
||||||
|
<a href="https://www.ebay.ca/itm/1"></a>
|
||||||
|
<h3>First Bundle</h3>
|
||||||
|
<span class="s-item__price">CA $100.00</span>
|
||||||
|
</li>
|
||||||
|
<li class="s-item">
|
||||||
|
<a href="https://www.ebay.ca/itm/2"></a>
|
||||||
|
<h3>Second Bundle</h3>
|
||||||
|
<span class="s-item__price">CA $110.00</span>
|
||||||
|
</li>
|
||||||
|
<li class="s-item">
|
||||||
|
<a href="https://www.ebay.ca/itm/3"></a>
|
||||||
|
<h3>Third Bundle</h3>
|
||||||
|
<span class="s-item__price">CA $70.00</span>
|
||||||
|
</li>
|
||||||
|
</body></html>
|
||||||
|
`),
|
||||||
|
}),
|
||||||
|
) as typeof fetch;
|
||||||
|
|
||||||
|
const results = await fetchEbayItems("laptop", 1000, { maxItems: 2 });
|
||||||
|
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
expect(results[0]).toEqual(
|
||||||
|
expect.objectContaining({ title: "First Bundle" }),
|
||||||
|
);
|
||||||
|
expect(results[1]).toEqual(
|
||||||
|
expect.objectContaining({ title: "Second Bundle" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("respects maxItems in unstable mode", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(`
|
||||||
|
<html><body>
|
||||||
|
<li class="s-item">
|
||||||
|
<a href="https://www.ebay.ca/itm/1"></a>
|
||||||
|
<h3>First Bundle</h3>
|
||||||
|
<span class="s-item__price">CA $100.00</span>
|
||||||
|
</li>
|
||||||
|
<li class="s-item">
|
||||||
|
<a href="https://www.ebay.ca/itm/2"></a>
|
||||||
|
<h3>Second Bundle</h3>
|
||||||
|
<span class="s-item__price">CA $110.00</span>
|
||||||
|
</li>
|
||||||
|
<li class="s-item">
|
||||||
|
<a href="https://www.ebay.ca/itm/3"></a>
|
||||||
|
<h3>Third Bundle</h3>
|
||||||
|
<span class="s-item__price">CA $70.00</span>
|
||||||
|
</li>
|
||||||
|
</body></html>
|
||||||
|
`),
|
||||||
|
}),
|
||||||
|
) as typeof fetch;
|
||||||
|
|
||||||
|
const results = await fetchEbayItems(
|
||||||
|
"laptop",
|
||||||
|
1000,
|
||||||
|
{ maxItems: 2 },
|
||||||
|
{ hideUnstableResults: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(results.results).toHaveLength(2);
|
||||||
|
expect(results.unstableResults).toHaveLength(0);
|
||||||
|
expect(results.results[0]).toEqual(
|
||||||
|
expect.objectContaining({ title: "First Bundle" }),
|
||||||
|
);
|
||||||
|
expect(results.results[1]).toEqual(
|
||||||
|
expect.objectContaining({ title: "Second Bundle" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user