feat: ebay splashui challenge solver

argon2id pow → /challengesvc/answer → chlgref cookie
warm homepage for akamai cookies, detect 307 redirect,
solve + retry transparently in fetchEbayItems flow

bun.lock

@@ -32,6 +32,7 @@
       "version": "1.0.0",
       "dependencies": {
         "@typescript/native-preview": "catalog:",
+        "argon2-wasm-pro": "1.1.0",
         "cli-progress": "^3.12.0",
         "linkedom": "^0.18.12",
         "unidecode": "^1.1.0",
@@ -120,6 +121,8 @@
 
     "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
 
+    "argon2-wasm-pro": ["argon2-wasm-pro@1.1.0", "", {}, "sha512-ApZAKEgbWQILckY+IdjrETB0oTC8L9YHT3JVQhdun77tilExkXNyM/T/qbkvX+Uv68+IQmVwewQwg6yJnSwVxQ=="],
+
     "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
 
     "bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],

packages/core/package.json

@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "@typescript/native-preview": "catalog:",
+    "argon2-wasm-pro": "1.1.0",
     "cli-progress": "^3.12.0",
     "linkedom": "^0.18.12",
     "unidecode": "^1.1.0"

packages/core/src/index.ts

@@ -39,6 +39,7 @@ export * from "./types/common";
 // Export shared utilities
 export * from "./utils/cookies";
 export * from "./utils/delay";
+export * from "./utils/ebay-challenge";
 export * from "./utils/format";
 export * from "./utils/http";
 export * from "./utils/unstable";

packages/core/src/scrapers/ebay.ts

@@ -10,6 +10,7 @@ import {
   formatCookiesForHeader,
 } from "../utils/cookies";
 import { delay } from "../utils/delay";
+import { solveEbayChallenge } from "../utils/ebay-challenge";
 import { logger } from "../utils/logger";
 import { classifyUnstableListings } from "../utils/unstable";
 
@@ -611,10 +612,10 @@ function parseEbayListings(
   );
 }
 
-// ----------------------------- Cookie Loading -----------------------------
+// ----------------------------- Session & Challenge -----------------------------
 
 /**
- * Load eBay cookies from EBAY_COOKIE
+ * Load eBay cookies from EBAY_COOKIE env var
  */
 async function loadEbayCookies(): Promise<string | undefined> {
   try {
@@ -628,6 +629,92 @@ async function loadEbayCookies(): Promise<string | undefined> {
   }
 }
 
+const EBAY_UA =
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+
+/**
+ * Visit eBay homepage to collect Akamai fingerprinting cookies.
+ * These are required to pass the edge layer before any search request.
+ */
+async function warmEbaySession(): Promise<string | undefined> {
+  try {
+    const res = await fetch("https://www.ebay.ca", {
+      headers: {
+        "User-Agent": EBAY_UA,
+        Accept:
+          "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        "Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8",
+      },
+      redirect: "manual",
+    });
+
+    if (!res.ok) return undefined;
+
+    const setCookies = res.headers.getSetCookie?.() ?? [];
+    const jar: Record<string, string> = {};
+    for (const header of setCookies) {
+      const match = header.match(/^([^=]+)=([^;]+)/);
+      if (match?.[1] && match[2]) jar[match[1]] = match[2];
+    }
+
+    const cookieKeys = Object.keys(jar);
+    if (cookieKeys.length === 0) return undefined;
+
+    return cookieKeys.map((k) => `${k}=${jar[k] ?? ""}`).join("; ");
+  } catch {
+    return undefined;
+  }
+}
+
+function mergeCookies(
+  base: string,
+  ...additions: (string | undefined)[]
+): string {
+  const jar: Record<string, string> = {};
+  const all = [base, ...additions.filter(Boolean)] as string[];
+  for (const str of all) {
+    for (const pair of str.split(";")) {
+      const eq = pair.indexOf("=");
+      if (eq > 0) {
+        jar[pair.substring(0, eq).trim()] = pair.substring(eq + 1).trim();
+      }
+    }
+  }
+  return Object.entries(jar)
+    .map(([k, v]) => `${k}=${v}`)
+    .join("; ");
+}
+
+function collectResponseCookies(res: Response, jar: Record<string, string>) {
+  for (const header of res.headers.getSetCookie?.() ?? []) {
+    const match = header.match(/^([^=]+)=([^;]+)/);
+    if (match?.[1] && match[2]) jar[match[1]] = match[2];
+  }
+}
+
+function cookiesToString(jar: Record<string, string>): string {
+  return Object.entries(jar)
+    .map(([k, v]) => `${k}=${v}`)
+    .join("; ");
+}
+
+const CHALLENGE_REDIRECT = 307;
+const CHALLENGE_MARKER = "splashui/challenge";
+
+function isChallengeRedirect(res: Response): boolean {
+  return (
+    res.status === CHALLENGE_REDIRECT &&
+    (res.headers.get("location") ?? "").includes(CHALLENGE_MARKER)
+  );
+}
+
+function isChallengeHtml(html: string): boolean {
+  return (
+    html.length < 50000 &&
+    (html.includes("_crefId") || html.includes("_cdetail"))
+  );
+}
+
 // ----------------------------- Main -----------------------------
 
 export default async function fetchEbayItems(
@@ -703,7 +790,10 @@ export default async function fetchEbayItems(
     return classifyUnstableListings(limitedListings);
   };
 
-  const cookies = await loadEbayCookies();
+  // Collect cookies from env var + warm-up session
+  const envCookies = await loadEbayCookies();
+  const warmCookies = await warmEbaySession();
+  const baseCookies = mergeCookies(envCookies ?? "", warmCookies);
 
   // Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
   const urlParams = new URLSearchParams({
@@ -727,35 +817,113 @@ export default async function fetchEbayItems(
   logger.log(`Fetching eBay search: ${searchUrl}`);
 
   try {
-    // Use custom headers modeled after real browser requests to bypass bot detection
-    const headers: Record<string, string> = {
-      "User-Agent":
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    const searchHeaders: Record<string, string> = {
+      "User-Agent": EBAY_UA,
+      Accept:
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
       "Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8",
-      "Accept-Encoding": "gzip, deflate, br, zstd",
       Referer: "https://www.ebay.ca/",
-      Connection: "keep-alive",
-      "Cache-Control": "no-cache",
-      Pragma: "no-cache",
-      "Upgrade-Insecure-Requests": "1",
-      "Sec-Fetch-Dest": "document",
-      "Sec-Fetch-Mode": "navigate",
-      "Sec-Fetch-Site": "same-origin",
-      "Sec-Fetch-User": "?1",
-      Priority: "u=0, i",
     };
 
-    // Add cookies if available (helps bypass bot detection)
-    if (cookies) {
-      headers.Cookie = cookies;
+    if (baseCookies) {
+      searchHeaders.Cookie = baseCookies;
     }
 
-    const res = await fetch(searchUrl, {
+    // Step 1: Make search request (follow redirects for challenge flow)
+    let res = await fetch(searchUrl, {
       method: "GET",
-      headers,
+      headers: searchHeaders,
+      redirect: "manual",
     });
 
+    const cookieJar: Record<string, string> = {};
+
+    // Collect cookies from homepage warm-up
+    if (baseCookies) {
+      for (const pair of baseCookies.split(";")) {
+        const eq = pair.indexOf("=");
+        if (eq > 0) {
+          cookieJar[pair.substring(0, eq).trim()] = pair
+            .substring(eq + 1)
+            .trim();
+        }
+      }
+    }
+
+    // Step 2: Follow challenge redirect if present
+    if (isChallengeRedirect(res)) {
+      const chalUrl = res.headers.get("location") ?? "";
+      collectResponseCookies(res, cookieJar);
+
+      logger.log("Challenge detected, fetching challenge page...");
+      res = await fetch(chalUrl, {
+        headers: { ...searchHeaders, Cookie: cookiesToString(cookieJar) },
+        redirect: "manual",
+      });
+      collectResponseCookies(res, cookieJar);
+    }
+
+    // Step 3: If response is challenge HTML, solve and submit
+    const responseHtml = await res.text();
+
+    if (isChallengeHtml(responseHtml)) {
+      logger.log("Solving challenge...");
+      const result = await solveEbayChallenge(
+        responseHtml,
+        cookiesToString(cookieJar),
+      );
+
+      if (result) {
+        // Merge answer cookies into jar
+        if (baseCookies) {
+          searchHeaders.Cookie = mergeCookies(baseCookies, result.cookies);
+        } else {
+          searchHeaders.Cookie = result.cookies;
+        }
+
+        logger.log("Challenge solved, retrying search...");
+
+        // Delay briefly before retry
+        await delay(DELAY_MS);
+
+        res = await fetch(searchUrl, {
+          method: "GET",
+          headers: searchHeaders,
+        });
+
+        if (!res.ok && res.status !== 200) {
+          logger.warn(`Retry after challenge returned ${res.status}`);
+          return finalizeResults([]);
+        }
+
+        const retryHtml = await res.text();
+        await delay(DELAY_MS);
+
+        const listings = parseEbayListings(
+          retryHtml,
+          keywords,
+          exclusions,
+          strictMode,
+        );
+
+        const filteredListings = listings.filter((listing) => {
+          const cents = listing.listingPrice?.cents;
+          return (
+            typeof cents === "number" && cents >= minPrice && cents <= maxPrice
+          );
+        });
+
+        logger.log(
+          `Parsed ${filteredListings.length} eBay listings (after challenge).`,
+        );
+        return finalizeResults(filteredListings);
+      }
+
+      logger.warn("Challenge solve failed, returning empty results.");
+      return finalizeResults([]);
+    }
+
+    // Step 4: Normal flow — no challenge
     if (!res.ok) {
       throw new HttpError(
         `Request failed with status ${res.status}`,
@@ -764,20 +932,17 @@ export default async function fetchEbayItems(
       );
     }
 
-    const searchHtml = await res.text();
-    // Respect per-request delay to keep at or under REQUESTS_PER_SECOND
     await delay(DELAY_MS);
 
     logger.log(`\nParsing eBay listings...`);
 
     const listings = parseEbayListings(
-      searchHtml,
+      responseHtml,
       keywords,
       exclusions,
       strictMode,
     );
 
-    // Filter by price range (additional safety check)
     const filteredListings = listings.filter((listing) => {
       const cents = listing.listingPrice?.cents;
       return (

packages/core/src/types/argon2-wasm-pro.d.ts

@@ -0,0 +1,25 @@
+declare module "argon2-wasm-pro" {
+  interface Argon2Options {
+    pass: string | Uint8Array;
+    salt: Uint8Array;
+    time: number;
+    mem: number;
+    hashLen: number;
+    parallelism: number;
+    type: number;
+  }
+
+  interface Argon2Result {
+    hash: Uint8Array;
+    hashHex: string;
+    encoded: string;
+  }
+
+  function hash(options: Argon2Options): Promise<Argon2Result>;
+
+  const argon2: {
+    hash: typeof hash;
+  };
+
+  export default argon2;
+}

packages/core/src/utils/ebay-challenge.ts

@@ -0,0 +1,239 @@
+import argon2 from "argon2-wasm-pro";
+
+// ------------------ Types ------------------
+
+interface ChallengeDetails {
+  p2: number;
+  p6: number;
+  p7: number;
+  p9: string;
+  p11: string;
+  p12: number;
+  p13: number;
+  p15: number;
+}
+
+interface ChallengeParams {
+  crefId: string;
+  cdetail: ChallengeDetails;
+  iid: string;
+  chlghost: string;
+  appName: string;
+  p: string;
+  destUrl: string;
+}
+
+interface ChallengeResult {
+  cookies: string;
+}
+
+// ------------------ Helpers ------------------
+
+function memcmp(a: Uint8Array, b: number[], len: number): number {
+  for (let i = 0; i < len; i++) {
+    const va = a[i] ?? 0;
+    const vb = b[i] ?? 0;
+    if (va !== vb) return (va & 0xff) - (vb & 0xff);
+  }
+  return 0;
+}
+
+function intToBytes(val: number, arr: Uint8Array, offset: number) {
+  arr[offset] = val >>> 24;
+  arr[offset + 1] = val >>> 16;
+  arr[offset + 2] = val >>> 8;
+  arr[offset + 3] = val;
+}
+
+function string2Bin(str: string): number[] {
+  const result: number[] = [];
+  for (let i = 0; i < str.length; i++) {
+    result.push(str.charCodeAt(i));
+  }
+  return result;
+}
+
+function bufferToBase64(buf: Uint8Array): string {
+  return btoa(String.fromCharCode(...buf));
+}
+
+function parseCookiesFromSetCookie(cookies: string[]): Record<string, string> {
+  const result: Record<string, string> = {};
+  for (const header of cookies) {
+    const match = header.match(/^([^=]+)=([^;]+)/);
+    if (match?.[1] && match[2]) {
+      result[match[1]] = match[2];
+    }
+  }
+  return result;
+}
+
+// ------------------ Default headers ------------------
+
+const BROWSER_UA =
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+
+const _EBAY_HEADERS: Record<string, string> = {
+  "User-Agent": BROWSER_UA,
+  Accept:
+    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+  "Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8",
+};
+
+// ------------------ Parser ------------------
+
+export function parseChallengePage(html: string): ChallengeParams | null {
+  const getHidden = (id: string): string => {
+    const re = new RegExp(
+      `id=${id}\\s+value='([^']*)'` +
+        `|id=${id}\\s+value="([^"]*)"` +
+        `|id=${id}\\s+value=([^\\s>]+)`,
+      "i",
+    );
+    const m = html.match(re);
+    if (!m) return "";
+    return m[1] ?? m[2] ?? m[3] ?? "";
+  };
+
+  const crefId = getHidden("_crefId");
+  const cdetailRaw = getHidden("_cdetail");
+  const iid = getHidden("_iid");
+  const chlghost = getHidden("_chlghost");
+  const appName = getHidden("_appName");
+  const p = getHidden("_p");
+
+  const formActionMatch = html.match(
+    /<form\s+id=destForm\s+[^>]*action=([^\s>]+)/i,
+  );
+  const destUrl = formActionMatch?.[1]?.trim() ?? "";
+
+  if (!crefId || !cdetailRaw) return null;
+
+  let cdetail: ChallengeDetails;
+  try {
+    const parsed = JSON.parse(cdetailRaw);
+    const d = parsed.details;
+    cdetail = {
+      p2: Number(d.p2),
+      p6: Number(d.p6),
+      p7: Number(d.p7),
+      p9: d.p9,
+      p11: d.p11,
+      p12: Number(d.p12),
+      p13: Number(d.p13),
+      p15: Number(d.p15),
+    };
+  } catch {
+    return null;
+  }
+
+  return {
+    crefId,
+    cdetail,
+    iid,
+    chlghost: chlghost || "https://www.ebay.ca",
+    appName: appName || "orch",
+    p,
+    destUrl,
+  };
+}
+
+// ------------------ Solver ------------------
+
+async function solveArgon2Challenge(
+  cdetail: ChallengeDetails,
+): Promise<string[]> {
+  const targetBytes = string2Bin(atob(cdetail.p11));
+  const targetLen = targetBytes.length;
+  const nonceLen = cdetail.p6;
+  const answerCount = cdetail.p15;
+  const salt = new Uint8Array(
+    Uint8Array.from(atob(cdetail.p9), (c) => c.charCodeAt(0)),
+  );
+
+  const answers: string[] = [];
+  let nonce = new Uint8Array(nonceLen);
+  crypto.getRandomValues(nonce);
+  intToBytes(0, nonce, nonce.length - 4);
+  let counter = 0;
+
+  while (answers.length < answerCount) {
+    const result = await argon2.hash({
+      pass: nonce,
+      salt,
+      time: cdetail.p2,
+      mem: cdetail.p13,
+      hashLen: cdetail.p7,
+      parallelism: cdetail.p12,
+      type: 2,
+    });
+
+    const hashBytes = result.hash as Uint8Array;
+
+    if (memcmp(hashBytes, targetBytes, targetLen) <= 0) {
+      answers.push(bufferToBase64(nonce));
+      nonce = new Uint8Array(nonceLen);
+      crypto.getRandomValues(nonce);
+      intToBytes(0, nonce, nonce.length - 4);
+      counter = 0;
+    } else {
+      counter++;
+      intToBytes(counter, nonce, nonce.length - 4);
+    }
+  }
+
+  return answers;
+}
+
+// ------------------ Public API ------------------
+
+export async function solveEbayChallenge(
+  html: string,
+  cookieHeader?: string,
+): Promise<ChallengeResult | null> {
+  const params = parseChallengePage(html);
+  if (!params) return null;
+
+  const answers = await solveArgon2Challenge(params.cdetail);
+  const encodedAnswers = encodeURIComponent(answers.join(","));
+
+  const body = JSON.stringify({
+    iid: params.iid,
+    appName: params.appName,
+    referenceId: params.crefId,
+    pvt: Date.now().toString(),
+    crt: Date.now().toString(),
+    encodedAnswers,
+    p: params.p,
+    ru: params.destUrl,
+  });
+
+  const headers: Record<string, string> = {
+    "content-type": "application/json",
+    accept: "application/json, text/plain, */*",
+    "user-agent": BROWSER_UA,
+  };
+
+  if (cookieHeader) {
+    headers.cookie = cookieHeader;
+  }
+
+  const res = await fetch(`${params.chlghost}/splashui/challengesvc/answer`, {
+    method: "POST",
+    headers,
+    body,
+  });
+
+  if (!res.ok) return null;
+
+  // Collect cookies from answer response
+  const setCookies = res.headers.getSetCookie?.() ?? [];
+  const answerCookies = parseCookiesFromSetCookie(setCookies);
+
+  const cookieEntries = Object.entries(answerCookies);
+  if (cookieEntries.length === 0) return null;
+
+  const cookies = cookieEntries.map(([k, v]) => `${k}=${v}`).join("; ");
+
+  return { cookies };
+}

packages/core/test/ebay-core.test.ts

@@ -47,17 +47,22 @@ describe("eBay Scraper Cookie Handling", () => {
   test("should ignore request cookie overrides and rely on EBAY_COOKIE", async () => {
     await fetchEbayItems("laptop", 1000);
 
-    expect(global.fetch).toHaveBeenCalledTimes(1);
+    // First call is homepage warm-up, second is search
+    expect(global.fetch).toHaveBeenCalledTimes(2);
 
-    const firstFetchCall = (global.fetch as unknown as ReturnType<typeof mock>)
-      .mock.calls[0];
-    if (!firstFetchCall) {
-      throw new Error("Expected fetch to be called");
+    // The search request is the second call
+    const secondFetchCall = (global.fetch as unknown as ReturnType<typeof mock>)
+      .mock.calls[1];
+    if (!secondFetchCall) {
+      throw new Error("Expected search fetch to be called");
     }
 
-    const [, init] = firstFetchCall;
+    const [searchUrl, init] = secondFetchCall;
     const headers = (init as RequestInit).headers as Record<string, string>;
 
+    expect(searchUrl).toBe(
+      "https://www.ebay.ca/sch/i.html?_nkw=laptop&_sacat=0&_from=R40&LH_BIN=1&LH_PrefLoc=1",
+    );
     expect(headers.Cookie).toBeUndefined();
   });