old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ebay splashui challenge solver

Browse Source 
argon2id pow → /challengesvc/answer → chlgref cookie
warm homepage for akamai cookies, detect 307 redirect,
solve + retry transparently in fetchEbayItems flow
This commit is contained in:
Dmytro Stanchiev
2026-04-30 20:44:37 -04:00
parent 53eafe6d4c
commit 9c4c347933
7 changed files with 472 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
packages/core/src/scrapers/ebay.ts
View File

@@ -10,6 +10,7 @@ import {
formatCookiesForHeader,
} from "../utils/cookies";
import { delay } from "../utils/delay";
import { solveEbayChallenge } from "../utils/ebay-challenge";
import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable";
@@ -611,10 +612,10 @@ function parseEbayListings(
);
}
// ----------------------------- Cookie Loading -----------------------------
// ----------------------------- Session & Challenge -----------------------------
/**
* Load eBay cookies from EBAY_COOKIE
* Load eBay cookies from EBAY_COOKIE env var
*/
async function loadEbayCookies(): Promise<string | undefined> {
try {
@@ -628,6 +629,92 @@ async function loadEbayCookies(): Promise<string | undefined> {
}
}
const EBAY_UA =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
/**
* Visit eBay homepage to collect Akamai fingerprinting cookies.
* These are required to pass the edge layer before any search request.
*/
async function warmEbaySession(): Promise<string | undefined> {
try {
const res = await fetch("https://www.ebay.ca", {
headers: {
"User-Agent": EBAY_UA,
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8",
},
redirect: "manual",
});
if (!res.ok) return undefined;
const setCookies = res.headers.getSetCookie?.() ?? [];
const jar: Record<string, string> = {};
for (const header of setCookies) {
const match = header.match(/^([^=]+)=([^;]+)/);
if (match?.[1] && match[2]) jar[match[1]] = match[2];
}
const cookieKeys = Object.keys(jar);
if (cookieKeys.length === 0) return undefined;
return cookieKeys.map((k) => `${k}=${jar[k] ?? ""}`).join("; ");
} catch {
return undefined;
}
}
function mergeCookies(
base: string,
...additions: (string | undefined)[]
): string {
const jar: Record<string, string> = {};
const all = [base, ...additions.filter(Boolean)] as string[];
for (const str of all) {
for (const pair of str.split(";")) {
const eq = pair.indexOf("=");
if (eq > 0) {
jar[pair.substring(0, eq).trim()] = pair.substring(eq + 1).trim();
}
}
}
return Object.entries(jar)
.map(([k, v]) => `${k}=${v}`)
.join("; ");
}
function collectResponseCookies(res: Response, jar: Record<string, string>) {
for (const header of res.headers.getSetCookie?.() ?? []) {
const match = header.match(/^([^=]+)=([^;]+)/);
if (match?.[1] && match[2]) jar[match[1]] = match[2];
}
}
function cookiesToString(jar: Record<string, string>): string {
return Object.entries(jar)
.map(([k, v]) => `${k}=${v}`)
.join("; ");
}
const CHALLENGE_REDIRECT = 307;
const CHALLENGE_MARKER = "splashui/challenge";
function isChallengeRedirect(res: Response): boolean {
return (
res.status === CHALLENGE_REDIRECT &&
(res.headers.get("location") ?? "").includes(CHALLENGE_MARKER)
);
}
function isChallengeHtml(html: string): boolean {
return (
html.length < 50000 &&
(html.includes("_crefId") || html.includes("_cdetail"))
);
}
// ----------------------------- Main -----------------------------
export default async function fetchEbayItems(
@@ -703,7 +790,10 @@ export default async function fetchEbayItems(
return classifyUnstableListings(limitedListings);
};
const cookies = await loadEbayCookies();
// Collect cookies from env var + warm-up session
const envCookies = await loadEbayCookies();
const warmCookies = await warmEbaySession();
const baseCookies = mergeCookies(envCookies ?? "", warmCookies);
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
const urlParams = new URLSearchParams({
@@ -727,35 +817,113 @@ export default async function fetchEbayItems(
logger.log(`Fetching eBay search: ${searchUrl}`);
try {
// Use custom headers modeled after real browser requests to bypass bot detection
const headers: Record<string, string> = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
const searchHeaders: Record<string, string> = {
"User-Agent": EBAY_UA,
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-CA,en-US;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate, br, zstd",
Referer: "https://www.ebay.ca/",
Connection: "keep-alive",
"Cache-Control": "no-cache",
Pragma: "no-cache",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-User": "?1",
Priority: "u=0, i",
};
// Add cookies if available (helps bypass bot detection)
if (cookies) {
headers.Cookie = cookies;
if (baseCookies) {
searchHeaders.Cookie = baseCookies;
}
const res = await fetch(searchUrl, {
// Step 1: Make search request (follow redirects for challenge flow)
let res = await fetch(searchUrl, {
method: "GET",
headers,
headers: searchHeaders,
redirect: "manual",
});
const cookieJar: Record<string, string> = {};
// Collect cookies from homepage warm-up
if (baseCookies) {
for (const pair of baseCookies.split(";")) {
const eq = pair.indexOf("=");
if (eq > 0) {
cookieJar[pair.substring(0, eq).trim()] = pair
.substring(eq + 1)
.trim();
}
}
}
// Step 2: Follow challenge redirect if present
if (isChallengeRedirect(res)) {
const chalUrl = res.headers.get("location") ?? "";
collectResponseCookies(res, cookieJar);
logger.log("Challenge detected, fetching challenge page...");
res = await fetch(chalUrl, {
headers: { ...searchHeaders, Cookie: cookiesToString(cookieJar) },
redirect: "manual",
});
collectResponseCookies(res, cookieJar);
}
// Step 3: If response is challenge HTML, solve and submit
const responseHtml = await res.text();
if (isChallengeHtml(responseHtml)) {
logger.log("Solving challenge...");
const result = await solveEbayChallenge(
responseHtml,
cookiesToString(cookieJar),
);
if (result) {
// Merge answer cookies into jar
if (baseCookies) {
searchHeaders.Cookie = mergeCookies(baseCookies, result.cookies);
} else {
searchHeaders.Cookie = result.cookies;
}
logger.log("Challenge solved, retrying search...");
// Delay briefly before retry
await delay(DELAY_MS);
res = await fetch(searchUrl, {
method: "GET",
headers: searchHeaders,
});
if (!res.ok && res.status !== 200) {
logger.warn(`Retry after challenge returned ${res.status}`);
return finalizeResults([]);
}
const retryHtml = await res.text();
await delay(DELAY_MS);
const listings = parseEbayListings(
retryHtml,
keywords,
exclusions,
strictMode,
);
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return (
typeof cents === "number" && cents >= minPrice && cents <= maxPrice
);
});
logger.log(
`Parsed ${filteredListings.length} eBay listings (after challenge).`,
);
return finalizeResults(filteredListings);
}
logger.warn("Challenge solve failed, returning empty results.");
return finalizeResults([]);
}
// Step 4: Normal flow — no challenge
if (!res.ok) {
throw new HttpError(
`Request failed with status ${res.status}`,
@@ -764,20 +932,17 @@ export default async function fetchEbayItems(
);
}
const searchHtml = await res.text();
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS);
logger.log(`\nParsing eBay listings...`);
const listings = parseEbayListings(
searchHtml,
responseHtml,
keywords,
exclusions,
strictMode,
);
// Filter by price range (additional safety check)
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return (