old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make ebay auth env-only

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-21 21:46:40 -04:00
parent a7a5eca7ad
commit 918ee92441
2 changed files with 50 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
packages/core/src/scrapers/ebay.ts
View File

@@ -1,8 +1,8 @@
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import { import {
type CookieConfig, type CookieConfig,
ensureCookies,
formatCookiesForHeader, formatCookiesForHeader,
loadCookiesOptional,
} from "../utils/cookies"; } from "../utils/cookies";
import { delay } from "../utils/delay"; import { delay } from "../utils/delay";
@@ -11,7 +11,6 @@ const EBAY_COOKIE_CONFIG: CookieConfig = {
name: "eBay", name: "eBay",
domain: ".ebay.ca", domain: ".ebay.ca",
envVar: "EBAY_COOKIE", envVar: "EBAY_COOKIE",
filePath: "./cookies/ebay.json",
}; };
// ----------------------------- Types ----------------------------- // ----------------------------- Types -----------------------------
@@ -335,27 +334,18 @@ function parseEbayListings(
// ----------------------------- Cookie Loading ----------------------------- // ----------------------------- Cookie Loading -----------------------------
/** /**
* Load eBay cookies with priority: URL param > ENV var > file * Load eBay cookies from EBAY_COOKIE
* Uses shared cookie utility for consistent handling across all scrapers
*/ */
async function loadEbayCookies( async function loadEbayCookies(): Promise<string | undefined> {
cookiesSource?: string, try {
): Promise<string | undefined> { const cookies = await ensureCookies(EBAY_COOKIE_CONFIG);
const cookies = await loadCookiesOptional(EBAY_COOKIE_CONFIG, cookiesSource); return formatCookiesForHeader(cookies, "www.ebay.ca");
} catch {
if (cookies.length === 0) {
console.warn( console.warn(
"No eBay cookies found. eBay may block requests without valid session cookies.\n" + "No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
"Provide cookies via (in priority order):\n" +
" 1. 'cookies' URL parameter (highest priority), or\n" +
" 2. EBAY_COOKIE environment variable, or\n" +
" 3. ./cookies/ebay.json file (lowest priority)\n" +
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
); );
return undefined; return undefined;
} }
return formatCookiesForHeader(cookies, "www.ebay.ca");
} }
// ----------------------------- Main ----------------------------- // ----------------------------- Main -----------------------------
@@ -371,7 +361,6 @@ export default async function fetchEbayItems(
keywords?: string[]; keywords?: string[];
buyItNowOnly?: boolean; buyItNowOnly?: boolean;
canadaOnly?: boolean; canadaOnly?: boolean;
cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection)
} = {}, } = {},
) { ) {
const { const {
@@ -382,11 +371,9 @@ export default async function fetchEbayItems(
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
buyItNowOnly = true, buyItNowOnly = true,
canadaOnly = true, canadaOnly = true,
cookies: cookiesSource,
} = opts; } = opts;
// Load eBay cookies with priority: URL param > ENV var > file const cookies = await loadEbayCookies();
const cookies = await loadEbayCookies(cookiesSource);
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference // Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
const urlParams = new URLSearchParams({ const urlParams = new URLSearchParams({

41
packages/core/test/ebay-core.test.ts Normal file
View File

@@ -0,0 +1,41 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import fetchEbayItems from "../src/scrapers/ebay";
const originalFetch = global.fetch;
const originalWarn = console.warn;
describe("eBay Scraper Cookie Handling", () => {
beforeEach(() => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () => Promise.resolve("<html><body></body></html>"),
}),
) as typeof fetch;
});
afterEach(() => {
global.fetch = originalFetch;
console.warn = originalWarn;
delete process.env.EBAY_COOKIE;
});
test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => {
const warnMock = mock(() => {});
console.warn = warnMock;
await fetchEbayItems("laptop", 1000, {
cookies: "s=from-request",
});
expect(global.fetch).toHaveBeenCalledTimes(1);
const [, init] = (global.fetch as ReturnType<typeof mock>).mock.calls[0];
const headers = (init as RequestInit).headers as Record<string, string>;
expect(headers.Cookie).toBeUndefined();
expect(warnMock).toHaveBeenCalledWith(
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
);
});
});