From 9c4c347933aedc7cae6b5e31a42a17ed04552896 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 20:44:37 -0400 Subject: [PATCH] feat: ebay splashui challenge solver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit argon2id pow → /challengesvc/answer → chlgref cookie warm homepage for akamai cookies, detect 307 redirect, solve + retry transparently in fetchEbayItems flow --- bun.lock | 3 + packages/core/package.json | 1 + packages/core/src/index.ts | 1 + packages/core/src/scrapers/ebay.ts | 219 ++++++++++++++--- packages/core/src/types/argon2-wasm-pro.d.ts | 25 ++ packages/core/src/utils/ebay-challenge.ts | 239 +++++++++++++++++++ packages/core/test/ebay-core.test.ts | 17 +- 7 files changed, 472 insertions(+), 33 deletions(-) create mode 100644 packages/core/src/types/argon2-wasm-pro.d.ts create mode 100644 packages/core/src/utils/ebay-challenge.ts diff --git a/bun.lock b/bun.lock index 712c3a1..58be89a 100644 --- a/bun.lock +++ b/bun.lock @@ -32,6 +32,7 @@ "version": "1.0.0", "dependencies": { "@typescript/native-preview": "catalog:", + "argon2-wasm-pro": "1.1.0", "cli-progress": "^3.12.0", "linkedom": "^0.18.12", "unidecode": "^1.1.0", @@ -120,6 +121,8 @@ "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "argon2-wasm-pro": ["argon2-wasm-pro@1.1.0", "", {}, "sha512-ApZAKEgbWQILckY+IdjrETB0oTC8L9YHT3JVQhdun77tilExkXNyM/T/qbkvX+Uv68+IQmVwewQwg6yJnSwVxQ=="], + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], diff --git a/packages/core/package.json b/packages/core/package.json index 886bcc4..4d07672 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@typescript/native-preview": "catalog:", + "argon2-wasm-pro": "1.1.0", "cli-progress": "^3.12.0", "linkedom": "^0.18.12", "unidecode": "^1.1.0" diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9c918fd..8f34561 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -39,6 +39,7 @@ export * from "./types/common"; // Export shared utilities export * from "./utils/cookies"; export * from "./utils/delay"; +export * from "./utils/ebay-challenge"; export * from "./utils/format"; export * from "./utils/http"; export * from "./utils/unstable"; diff --git a/packages/core/src/scrapers/ebay.ts b/packages/core/src/scrapers/ebay.ts index c479f7e..709412f 100644 --- a/packages/core/src/scrapers/ebay.ts +++ b/packages/core/src/scrapers/ebay.ts @@ -10,6 +10,7 @@ import { formatCookiesForHeader, } from "../utils/cookies"; import { delay } from "../utils/delay"; +import { solveEbayChallenge } from "../utils/ebay-challenge"; import { logger } from "../utils/logger"; import { classifyUnstableListings } from "../utils/unstable"; @@ -611,10 +612,10 @@ function parseEbayListings( ); } -// ----------------------------- Cookie Loading ----------------------------- +// ----------------------------- Session & Challenge ----------------------------- /** - * Load eBay cookies from EBAY_COOKIE + * Load eBay cookies from EBAY_COOKIE env var */ async function loadEbayCookies(): Promise { try { @@ -628,6 +629,92 @@ async function loadEbayCookies(): Promise { } } +const EBAY_UA = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + +/** + * Visit eBay homepage to collect Akamai fingerprinting cookies. + * These are required to pass the edge layer before any search request. + */ +async function warmEbaySession(): Promise { + try { + const res = await fetch("https://www.ebay.ca", { + headers: { + "User-Agent": EBAY_UA, + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8", + }, + redirect: "manual", + }); + + if (!res.ok) return undefined; + + const setCookies = res.headers.getSetCookie?.() ?? []; + const jar: Record = {}; + for (const header of setCookies) { + const match = header.match(/^([^=]+)=([^;]+)/); + if (match?.[1] && match[2]) jar[match[1]] = match[2]; + } + + const cookieKeys = Object.keys(jar); + if (cookieKeys.length === 0) return undefined; + + return cookieKeys.map((k) => `${k}=${jar[k] ?? ""}`).join("; "); + } catch { + return undefined; + } +} + +function mergeCookies( + base: string, + ...additions: (string | undefined)[] +): string { + const jar: Record = {}; + const all = [base, ...additions.filter(Boolean)] as string[]; + for (const str of all) { + for (const pair of str.split(";")) { + const eq = pair.indexOf("="); + if (eq > 0) { + jar[pair.substring(0, eq).trim()] = pair.substring(eq + 1).trim(); + } + } + } + return Object.entries(jar) + .map(([k, v]) => `${k}=${v}`) + .join("; "); +} + +function collectResponseCookies(res: Response, jar: Record) { + for (const header of res.headers.getSetCookie?.() ?? []) { + const match = header.match(/^([^=]+)=([^;]+)/); + if (match?.[1] && match[2]) jar[match[1]] = match[2]; + } +} + +function cookiesToString(jar: Record): string { + return Object.entries(jar) + .map(([k, v]) => `${k}=${v}`) + .join("; "); +} + +const CHALLENGE_REDIRECT = 307; +const CHALLENGE_MARKER = "splashui/challenge"; + +function isChallengeRedirect(res: Response): boolean { + return ( + res.status === CHALLENGE_REDIRECT && + (res.headers.get("location") ?? "").includes(CHALLENGE_MARKER) + ); +} + +function isChallengeHtml(html: string): boolean { + return ( + html.length < 50000 && + (html.includes("_crefId") || html.includes("_cdetail")) + ); +} + // ----------------------------- Main ----------------------------- export default async function fetchEbayItems( @@ -703,7 +790,10 @@ export default async function fetchEbayItems( return classifyUnstableListings(limitedListings); }; - const cookies = await loadEbayCookies(); + // Collect cookies from env var + warm-up session + const envCookies = await loadEbayCookies(); + const warmCookies = await warmEbaySession(); + const baseCookies = mergeCookies(envCookies ?? "", warmCookies); // Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference const urlParams = new URLSearchParams({ @@ -727,35 +817,113 @@ export default async function fetchEbayItems( logger.log(`Fetching eBay search: ${searchUrl}`); try { - // Use custom headers modeled after real browser requests to bypass bot detection - const headers: Record = { - "User-Agent": - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + const searchHeaders: Record = { + "User-Agent": EBAY_UA, + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8", - "Accept-Encoding": "gzip, deflate, br, zstd", Referer: "https://www.ebay.ca/", - Connection: "keep-alive", - "Cache-Control": "no-cache", - Pragma: "no-cache", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "same-origin", - "Sec-Fetch-User": "?1", - Priority: "u=0, i", }; - // Add cookies if available (helps bypass bot detection) - if (cookies) { - headers.Cookie = cookies; + if (baseCookies) { + searchHeaders.Cookie = baseCookies; } - const res = await fetch(searchUrl, { + // Step 1: Make search request (follow redirects for challenge flow) + let res = await fetch(searchUrl, { method: "GET", - headers, + headers: searchHeaders, + redirect: "manual", }); + const cookieJar: Record = {}; + + // Collect cookies from homepage warm-up + if (baseCookies) { + for (const pair of baseCookies.split(";")) { + const eq = pair.indexOf("="); + if (eq > 0) { + cookieJar[pair.substring(0, eq).trim()] = pair + .substring(eq + 1) + .trim(); + } + } + } + + // Step 2: Follow challenge redirect if present + if (isChallengeRedirect(res)) { + const chalUrl = res.headers.get("location") ?? ""; + collectResponseCookies(res, cookieJar); + + logger.log("Challenge detected, fetching challenge page..."); + res = await fetch(chalUrl, { + headers: { ...searchHeaders, Cookie: cookiesToString(cookieJar) }, + redirect: "manual", + }); + collectResponseCookies(res, cookieJar); + } + + // Step 3: If response is challenge HTML, solve and submit + const responseHtml = await res.text(); + + if (isChallengeHtml(responseHtml)) { + logger.log("Solving challenge..."); + const result = await solveEbayChallenge( + responseHtml, + cookiesToString(cookieJar), + ); + + if (result) { + // Merge answer cookies into jar + if (baseCookies) { + searchHeaders.Cookie = mergeCookies(baseCookies, result.cookies); + } else { + searchHeaders.Cookie = result.cookies; + } + + logger.log("Challenge solved, retrying search..."); + + // Delay briefly before retry + await delay(DELAY_MS); + + res = await fetch(searchUrl, { + method: "GET", + headers: searchHeaders, + }); + + if (!res.ok && res.status !== 200) { + logger.warn(`Retry after challenge returned ${res.status}`); + return finalizeResults([]); + } + + const retryHtml = await res.text(); + await delay(DELAY_MS); + + const listings = parseEbayListings( + retryHtml, + keywords, + exclusions, + strictMode, + ); + + const filteredListings = listings.filter((listing) => { + const cents = listing.listingPrice?.cents; + return ( + typeof cents === "number" && cents >= minPrice && cents <= maxPrice + ); + }); + + logger.log( + `Parsed ${filteredListings.length} eBay listings (after challenge).`, + ); + return finalizeResults(filteredListings); + } + + logger.warn("Challenge solve failed, returning empty results."); + return finalizeResults([]); + } + + // Step 4: Normal flow — no challenge if (!res.ok) { throw new HttpError( `Request failed with status ${res.status}`, @@ -764,20 +932,17 @@ export default async function fetchEbayItems( ); } - const searchHtml = await res.text(); - // Respect per-request delay to keep at or under REQUESTS_PER_SECOND await delay(DELAY_MS); logger.log(`\nParsing eBay listings...`); const listings = parseEbayListings( - searchHtml, + responseHtml, keywords, exclusions, strictMode, ); - // Filter by price range (additional safety check) const filteredListings = listings.filter((listing) => { const cents = listing.listingPrice?.cents; return ( diff --git a/packages/core/src/types/argon2-wasm-pro.d.ts b/packages/core/src/types/argon2-wasm-pro.d.ts new file mode 100644 index 0000000..1956f4e --- /dev/null +++ b/packages/core/src/types/argon2-wasm-pro.d.ts @@ -0,0 +1,25 @@ +declare module "argon2-wasm-pro" { + interface Argon2Options { + pass: string | Uint8Array; + salt: Uint8Array; + time: number; + mem: number; + hashLen: number; + parallelism: number; + type: number; + } + + interface Argon2Result { + hash: Uint8Array; + hashHex: string; + encoded: string; + } + + function hash(options: Argon2Options): Promise; + + const argon2: { + hash: typeof hash; + }; + + export default argon2; +} diff --git a/packages/core/src/utils/ebay-challenge.ts b/packages/core/src/utils/ebay-challenge.ts new file mode 100644 index 0000000..b58bc17 --- /dev/null +++ b/packages/core/src/utils/ebay-challenge.ts @@ -0,0 +1,239 @@ +import argon2 from "argon2-wasm-pro"; + +// ------------------ Types ------------------ + +interface ChallengeDetails { + p2: number; + p6: number; + p7: number; + p9: string; + p11: string; + p12: number; + p13: number; + p15: number; +} + +interface ChallengeParams { + crefId: string; + cdetail: ChallengeDetails; + iid: string; + chlghost: string; + appName: string; + p: string; + destUrl: string; +} + +interface ChallengeResult { + cookies: string; +} + +// ------------------ Helpers ------------------ + +function memcmp(a: Uint8Array, b: number[], len: number): number { + for (let i = 0; i < len; i++) { + const va = a[i] ?? 0; + const vb = b[i] ?? 0; + if (va !== vb) return (va & 0xff) - (vb & 0xff); + } + return 0; +} + +function intToBytes(val: number, arr: Uint8Array, offset: number) { + arr[offset] = val >>> 24; + arr[offset + 1] = val >>> 16; + arr[offset + 2] = val >>> 8; + arr[offset + 3] = val; +} + +function string2Bin(str: string): number[] { + const result: number[] = []; + for (let i = 0; i < str.length; i++) { + result.push(str.charCodeAt(i)); + } + return result; +} + +function bufferToBase64(buf: Uint8Array): string { + return btoa(String.fromCharCode(...buf)); +} + +function parseCookiesFromSetCookie(cookies: string[]): Record { + const result: Record = {}; + for (const header of cookies) { + const match = header.match(/^([^=]+)=([^;]+)/); + if (match?.[1] && match[2]) { + result[match[1]] = match[2]; + } + } + return result; +} + +// ------------------ Default headers ------------------ + +const BROWSER_UA = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"; + +const _EBAY_HEADERS: Record = { + "User-Agent": BROWSER_UA, + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8", +}; + +// ------------------ Parser ------------------ + +export function parseChallengePage(html: string): ChallengeParams | null { + const getHidden = (id: string): string => { + const re = new RegExp( + `id=${id}\\s+value='([^']*)'` + + `|id=${id}\\s+value="([^"]*)"` + + `|id=${id}\\s+value=([^\\s>]+)`, + "i", + ); + const m = html.match(re); + if (!m) return ""; + return m[1] ?? m[2] ?? m[3] ?? ""; + }; + + const crefId = getHidden("_crefId"); + const cdetailRaw = getHidden("_cdetail"); + const iid = getHidden("_iid"); + const chlghost = getHidden("_chlghost"); + const appName = getHidden("_appName"); + const p = getHidden("_p"); + + const formActionMatch = html.match( + /]*action=([^\s>]+)/i, + ); + const destUrl = formActionMatch?.[1]?.trim() ?? ""; + + if (!crefId || !cdetailRaw) return null; + + let cdetail: ChallengeDetails; + try { + const parsed = JSON.parse(cdetailRaw); + const d = parsed.details; + cdetail = { + p2: Number(d.p2), + p6: Number(d.p6), + p7: Number(d.p7), + p9: d.p9, + p11: d.p11, + p12: Number(d.p12), + p13: Number(d.p13), + p15: Number(d.p15), + }; + } catch { + return null; + } + + return { + crefId, + cdetail, + iid, + chlghost: chlghost || "https://www.ebay.ca", + appName: appName || "orch", + p, + destUrl, + }; +} + +// ------------------ Solver ------------------ + +async function solveArgon2Challenge( + cdetail: ChallengeDetails, +): Promise { + const targetBytes = string2Bin(atob(cdetail.p11)); + const targetLen = targetBytes.length; + const nonceLen = cdetail.p6; + const answerCount = cdetail.p15; + const salt = new Uint8Array( + Uint8Array.from(atob(cdetail.p9), (c) => c.charCodeAt(0)), + ); + + const answers: string[] = []; + let nonce = new Uint8Array(nonceLen); + crypto.getRandomValues(nonce); + intToBytes(0, nonce, nonce.length - 4); + let counter = 0; + + while (answers.length < answerCount) { + const result = await argon2.hash({ + pass: nonce, + salt, + time: cdetail.p2, + mem: cdetail.p13, + hashLen: cdetail.p7, + parallelism: cdetail.p12, + type: 2, + }); + + const hashBytes = result.hash as Uint8Array; + + if (memcmp(hashBytes, targetBytes, targetLen) <= 0) { + answers.push(bufferToBase64(nonce)); + nonce = new Uint8Array(nonceLen); + crypto.getRandomValues(nonce); + intToBytes(0, nonce, nonce.length - 4); + counter = 0; + } else { + counter++; + intToBytes(counter, nonce, nonce.length - 4); + } + } + + return answers; +} + +// ------------------ Public API ------------------ + +export async function solveEbayChallenge( + html: string, + cookieHeader?: string, +): Promise { + const params = parseChallengePage(html); + if (!params) return null; + + const answers = await solveArgon2Challenge(params.cdetail); + const encodedAnswers = encodeURIComponent(answers.join(",")); + + const body = JSON.stringify({ + iid: params.iid, + appName: params.appName, + referenceId: params.crefId, + pvt: Date.now().toString(), + crt: Date.now().toString(), + encodedAnswers, + p: params.p, + ru: params.destUrl, + }); + + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/plain, */*", + "user-agent": BROWSER_UA, + }; + + if (cookieHeader) { + headers.cookie = cookieHeader; + } + + const res = await fetch(`${params.chlghost}/splashui/challengesvc/answer`, { + method: "POST", + headers, + body, + }); + + if (!res.ok) return null; + + // Collect cookies from answer response + const setCookies = res.headers.getSetCookie?.() ?? []; + const answerCookies = parseCookiesFromSetCookie(setCookies); + + const cookieEntries = Object.entries(answerCookies); + if (cookieEntries.length === 0) return null; + + const cookies = cookieEntries.map(([k, v]) => `${k}=${v}`).join("; "); + + return { cookies }; +} diff --git a/packages/core/test/ebay-core.test.ts b/packages/core/test/ebay-core.test.ts index bcfa0d7..42baf8c 100644 --- a/packages/core/test/ebay-core.test.ts +++ b/packages/core/test/ebay-core.test.ts @@ -47,17 +47,22 @@ describe("eBay Scraper Cookie Handling", () => { test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => { await fetchEbayItems("laptop", 1000); - expect(global.fetch).toHaveBeenCalledTimes(1); + // First call is homepage warm-up, second is search + expect(global.fetch).toHaveBeenCalledTimes(2); - const firstFetchCall = (global.fetch as unknown as ReturnType) - .mock.calls[0]; - if (!firstFetchCall) { - throw new Error("Expected fetch to be called"); + // The search request is the second call + const secondFetchCall = (global.fetch as unknown as ReturnType) + .mock.calls[1]; + if (!secondFetchCall) { + throw new Error("Expected search fetch to be called"); } - const [, init] = firstFetchCall; + const [searchUrl, init] = secondFetchCall; const headers = (init as RequestInit).headers as Record; + expect(searchUrl).toBe( + "https://www.ebay.ca/sch/i.html?_nkw=laptop&_sacat=0&_from=R40&LH_BIN=1&LH_PrefLoc=1", + ); expect(headers.Cookie).toBeUndefined(); });