refactor: use shared cookie utility in ebay scraper
Replace inline cookie loading with shared utility functions. Now supports both JSON array and cookie string formats. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,19 @@
|
|||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
|
import {
|
||||||
|
type CookieConfig,
|
||||||
|
formatCookiesForHeader,
|
||||||
|
loadCookiesOptional,
|
||||||
|
} from "../utils/cookies";
|
||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
|
|
||||||
|
// eBay cookie configuration
|
||||||
|
const EBAY_COOKIE_CONFIG: CookieConfig = {
|
||||||
|
name: "eBay",
|
||||||
|
domain: ".ebay.ca",
|
||||||
|
envVar: "EBAY_COOKIE",
|
||||||
|
filePath: "./cookies/ebay.json",
|
||||||
|
};
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
export interface EbayListingDetails {
|
export interface EbayListingDetails {
|
||||||
@@ -323,54 +336,28 @@ function parseEbayListings(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Load eBay cookies with priority: URL param > ENV var > file
|
* Load eBay cookies with priority: URL param > ENV var > file
|
||||||
* @param cookiesSource - Optional cookie string from URL parameter (highest priority)
|
* Uses shared cookie utility for consistent handling across all scrapers
|
||||||
* @param cookiePath - Path to cookie file (default: ./cookies/ebay.json) (lowest priority)
|
|
||||||
* @returns Cookie string for HTTP header or undefined if no cookies found
|
|
||||||
*/
|
*/
|
||||||
async function loadEbayCookies(
|
async function loadEbayCookies(
|
||||||
cookiesSource?: string,
|
cookiesSource?: string,
|
||||||
cookiePath = "./cookies/ebay.json",
|
|
||||||
): Promise<string | undefined> {
|
): Promise<string | undefined> {
|
||||||
// Priority 1: URL parameter (if provided)
|
const cookies = await loadCookiesOptional(EBAY_COOKIE_CONFIG, cookiesSource);
|
||||||
if (cookiesSource?.trim()) {
|
|
||||||
console.log("Loaded eBay cookies from URL parameter");
|
|
||||||
return cookiesSource.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Priority 2: Environment variable
|
if (cookies.length === 0) {
|
||||||
const envCookies = process.env.EBAY_COOKIE;
|
|
||||||
if (envCookies?.trim()) {
|
|
||||||
console.log("Loaded eBay cookies from EBAY_COOKIE env var");
|
|
||||||
return envCookies.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Priority 3: Cookie file (fallback)
|
|
||||||
try {
|
|
||||||
const file = Bun.file(cookiePath);
|
|
||||||
if (await file.exists()) {
|
|
||||||
const content = await file.text();
|
|
||||||
const trimmed = content.trim();
|
|
||||||
if (trimmed) {
|
|
||||||
console.log(`Loaded eBay cookies from ${cookiePath}`);
|
|
||||||
return trimmed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
console.warn(`Could not load cookies from ${cookiePath}: ${e}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// No cookies found (eBay cookies are optional, just warn)
|
|
||||||
console.warn(
|
console.warn(
|
||||||
"No eBay cookies found. eBay may block requests without valid session cookies.\n" +
|
"No eBay cookies found. eBay may block requests without valid session cookies.\n" +
|
||||||
"Provide cookies via (in priority order):\n" +
|
"Provide cookies via (in priority order):\n" +
|
||||||
" 1. 'cookies' URL parameter (highest priority), or\n" +
|
" 1. 'cookies' URL parameter (highest priority), or\n" +
|
||||||
" 2. EBAY_COOKIE environment variable, or\n" +
|
" 2. EBAY_COOKIE environment variable, or\n" +
|
||||||
" 3. ./cookies/ebay.json file (lowest priority)\n" +
|
" 3. ./cookies/ebay.json file (lowest priority)\n" +
|
||||||
'Format: Cookie string like "name1=value1; name2=value2"',
|
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
|
||||||
);
|
);
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return formatCookiesForHeader(cookies, "www.ebay.ca");
|
||||||
|
}
|
||||||
|
|
||||||
// ----------------------------- Main -----------------------------
|
// ----------------------------- Main -----------------------------
|
||||||
|
|
||||||
export default async function fetchEbayItems(
|
export default async function fetchEbayItems(
|
||||||
@@ -384,8 +371,7 @@ export default async function fetchEbayItems(
|
|||||||
keywords?: string[];
|
keywords?: string[];
|
||||||
buyItNowOnly?: boolean;
|
buyItNowOnly?: boolean;
|
||||||
canadaOnly?: boolean;
|
canadaOnly?: boolean;
|
||||||
cookies?: string; // Optional: Cookie string from URL parameter (highest priority)
|
cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection)
|
||||||
cookiePath?: string; // Optional: Path to cookie file (default: ./cookies/ebay.json)
|
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
const {
|
const {
|
||||||
@@ -397,11 +383,10 @@ export default async function fetchEbayItems(
|
|||||||
buyItNowOnly = true,
|
buyItNowOnly = true,
|
||||||
canadaOnly = true,
|
canadaOnly = true,
|
||||||
cookies: cookiesSource,
|
cookies: cookiesSource,
|
||||||
cookiePath,
|
|
||||||
} = opts;
|
} = opts;
|
||||||
|
|
||||||
// Load eBay cookies with priority: URL param > ENV var > file
|
// Load eBay cookies with priority: URL param > ENV var > file
|
||||||
const cookies = await loadEbayCookies(cookiesSource, cookiePath);
|
const cookies = await loadEbayCookies(cookiesSource);
|
||||||
|
|
||||||
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
||||||
const urlParams = new URLSearchParams({
|
const urlParams = new URLSearchParams({
|
||||||
|
|||||||
Reference in New Issue
Block a user