old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: share scraper http fetching

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-29 13:14:20 -04:00
parent 22eb65d4a2
commit 6e50ebf901
6 changed files with 121 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/core/src/scrapers/ebay.ts
View File

@@ -9,7 +9,7 @@ import {
ensureCookies, ensureCookies,
formatCookiesForHeader, formatCookiesForHeader,
} from "../utils/cookies"; } from "../utils/cookies";
import { delay } from "../utils/delay"; import { fetchHtml, HttpError } from "../utils/http";
import { logger } from "../utils/logger"; import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable"; import { classifyUnstableListings } from "../utils/unstable";
@@ -102,17 +102,6 @@ function parseEbayPrice(
return { cents, currency }; return { cents, currency };
} }
class HttpError extends Error {
constructor(
message: string,
public readonly status: number,
public readonly url: string,
) {
super(message);
this.name = "HttpError";
}
}
// ----------------------------- Parsing ----------------------------- // ----------------------------- Parsing -----------------------------
/** /**
@@ -500,22 +489,7 @@ export default async function fetchEbayItems(
headers.Cookie = cookies; headers.Cookie = cookies;
} }
const res = await fetch(searchUrl, { const searchHtml = await fetchHtml(searchUrl, DELAY_MS, { headers });
method: "GET",
headers,
});
if (!res.ok) {
throw new HttpError(
`Request failed with status ${res.status}`,
res.status,
searchUrl,
);
}
const searchHtml = await res.text();
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS);
logger.log(`\nParsing eBay listings...`); logger.log(`\nParsing eBay listings...`);
@@ -538,8 +512,8 @@ export default async function fetchEbayItems(
return finalizeResults(filteredListings); return finalizeResults(filteredListings);
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
console.error( logger.error(
`Failed to fetch eBay search (${err.status}): ${err.message}`, `Failed to fetch eBay search (${err.statusCode}): ${err.message}`,
); );
return finalizeResults([]); return finalizeResults([]);
} }

166
packages/core/src/scrapers/facebook.ts
View File

@@ -12,9 +12,8 @@ import {
formatCookiesForHeader, formatCookiesForHeader,
parseCookieString, parseCookieString,
} from "../utils/cookies"; } from "../utils/cookies";
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format"; import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http"; import { fetchHtml, HttpError, isRecord, RateLimitError } from "../utils/http";
import { logger } from "../utils/logger"; import { logger } from "../utils/logger";
import { classifyUnstableListings } from "../utils/unstable"; import { classifyUnstableListings } from "../utils/unstable";
@@ -219,17 +218,6 @@ export async function ensureFacebookCookies(): Promise<Cookie[]> {
return ensureCookies(FACEBOOK_COOKIE_CONFIG); return ensureCookies(FACEBOOK_COOKIE_CONFIG);
} }
class HttpError extends Error {
constructor(
message: string,
public readonly status: number,
public readonly url: string,
) {
super(message);
this.name = "HttpError";
}
}
// ----------------------------- Extraction Metrics ----------------------------- // ----------------------------- Extraction Metrics -----------------------------
/** /**
@@ -274,112 +262,21 @@ function logExtractionMetrics(success: boolean, itemId?: string) {
// ----------------------------- HTTP Client ----------------------------- // ----------------------------- HTTP Client -----------------------------
/** function createFacebookHeaders(cookies: string): Record<string, string> {
Fetch HTML with a basic retry strategy and simple rate-limit delay between calls. return {
- Retries on 429 and 5xx accept:
- Respects X-RateLimit-Reset when present (seconds) "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
- Supports custom cookies for Facebook authentication "accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
*/ "cache-control": "no-cache",
async function fetchHtml( "upgrade-insecure-requests": "1",
url: string, "sec-fetch-dest": "document",
DELAY_MS: number, "sec-fetch-mode": "navigate",
opts?: { "sec-fetch-site": "none",
maxRetries?: number; "sec-fetch-user": "?1",
retryBaseMs?: number; "user-agent":
onRateInfo?: (remaining: string | null, reset: string | null) => void; "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
cookies?: string; cookie: cookies,
}, };
): Promise<{ html: HTMLString; responseUrl: string }> {
const maxRetries = opts?.maxRetries ?? 3;
const retryBaseMs = opts?.retryBaseMs ?? 500;
let lastRateLimitError: HttpError | null = null;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
const headers: Record<string, string> = {
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
"accept-encoding": "gzip, deflate, br",
"cache-control": "no-cache",
"upgrade-insecure-requests": "1",
"sec-fetch-dest": "document",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "none",
"sec-fetch-user": "?1",
"user-agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
};
// Add cookies if provided
if (opts?.cookies) {
headers.cookie = opts.cookies;
}
const res = await fetch(url, {
method: "GET",
headers,
});
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
if (!res.ok) {
// Respect 429 reset if provided
if (res.status === 429) {
lastRateLimitError = new HttpError(
`Request failed with status ${res.status}`,
res.status,
url,
);
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: (attempt + 1) * retryBaseMs;
if (attempt >= maxRetries) {
throw lastRateLimitError;
}
await delay(waitMs);
continue;
}
// For Facebook, 400 often means authentication required
// Don't retry 4xx client errors except 429
if (res.status >= 400 && res.status < 500 && res.status !== 429) {
throw new HttpError(
`Request failed with status ${res.status} (Facebook may require authentication cookies for access)`,
res.status,
url,
);
}
// Retry on 5xx
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
await delay((attempt + 1) * retryBaseMs);
continue;
}
throw new HttpError(
`Request failed with status ${res.status}`,
res.status,
url,
);
}
const html = await res.text();
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS);
return { html, responseUrl: res.url || url };
} catch (err) {
if (err instanceof HttpError) {
throw err;
}
if (attempt >= maxRetries) throw err;
await delay((attempt + 1) * retryBaseMs);
}
}
throw lastRateLimitError ?? new Error("Exhausted retries without response");
} }
// ----------------------------- Parsing ----------------------------- // ----------------------------- Parsing -----------------------------
@@ -1157,6 +1054,8 @@ export default async function fetchFacebookItems(
try { try {
const response = await fetchHtml(searchUrl, DELAY_MS, { const response = await fetchHtml(searchUrl, DELAY_MS, {
maxRetries: 3, maxRetries: 3,
includeResponseUrl: true,
headers: createFacebookHeaders(cookiesHeader),
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
logger.log( logger.log(
@@ -1164,22 +1063,27 @@ export default async function fetchFacebookItems(
); );
} }
}, },
cookies: cookiesHeader,
}); });
searchHtml = response.html; searchHtml = response.html;
searchResponseUrl = response.responseUrl; searchResponseUrl = response.responseUrl;
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
logger.warn( logger.warn(
`\nFacebook marketplace access failed (${err.status}): ${err.message}`, `\nFacebook marketplace access failed (${err.statusCode}): ${err.message}`,
); );
if (err.status === 400 || err.status === 401 || err.status === 403) { if (err.statusCode === 400 || err.statusCode === 401 || err.statusCode === 403) {
logger.warn( logger.warn(
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.", "This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
); );
} }
return finalizeResults([]); return finalizeResults([]);
} }
if (err instanceof RateLimitError) {
logger.warn(
`\nFacebook marketplace access rate limited: ${err.message}`,
);
return finalizeResults([]);
}
throw err; throw err;
} }
@@ -1261,6 +1165,8 @@ export async function fetchFacebookItem(
let itemResponseUrl = itemUrl; let itemResponseUrl = itemUrl;
try { try {
const response = await fetchHtml(itemUrl, 1000, { const response = await fetchHtml(itemUrl, 1000, {
includeResponseUrl: true,
headers: createFacebookHeaders(cookiesHeader),
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
logger.log( logger.log(
@@ -1268,18 +1174,17 @@ export async function fetchFacebookItem(
); );
} }
}, },
cookies: cookiesHeader,
}); });
itemHtml = response.html; itemHtml = response.html;
itemResponseUrl = response.responseUrl; itemResponseUrl = response.responseUrl;
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
logger.warn( logger.warn(
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`, `\nFacebook marketplace item access failed (${err.statusCode}): ${err.message}`,
); );
// Enhanced error handling based on status codes // Enhanced error handling based on status codes
switch (err.status) { switch (err.statusCode) {
case 400: case 400:
case 401: case 401:
case 403: case 403:
@@ -1305,10 +1210,19 @@ export async function fetchFacebookItem(
); );
break; break;
default: default:
logger.warn(`Unexpected error status: ${err.status}`); logger.warn(`Unexpected error status: ${err.statusCode}`);
} }
return null; return null;
} }
if (err instanceof RateLimitError) {
logger.warn(
`\nFacebook marketplace item rate limited for item ${itemId}: ${err.message}`,
);
logger.warn(
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
);
return null;
}
throw err; throw err;
} }

53
packages/core/src/utils/http.ts
View File

@@ -1,3 +1,4 @@
import type { HTMLString } from "../types/common";
import { delay } from "./delay"; import { delay } from "./delay";
/** Custom error class for HTTP-related failures */ /** Custom error class for HTTP-related failures */
@@ -60,10 +61,20 @@ export function isRecord(value: unknown): value is Record<string, unknown> {
/** /**
* Calculate exponential backoff delay with jitter * Calculate exponential backoff delay with jitter
*/ */
function calculateBackoffDelay(attempt: number, baseMs: number): number { function calculateBackoffDelay(
attempt: number,
baseMs: number,
jitter: () => number = Math.random,
): number {
const exponentialDelay = baseMs * 2 ** attempt; const exponentialDelay = baseMs * 2 ** attempt;
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter const jitterDelay = jitter() * 0.1 * exponentialDelay; // 10% jitter
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds return Math.min(exponentialDelay + jitterDelay, 30000); // Cap at 30 seconds
}
/** Result type when includeResponseUrl is true */
export interface FetchHtmlResult {
html: HTMLString;
responseUrl: string;
} }
/** Options for fetchHtml */ /** Options for fetchHtml */
@@ -73,6 +84,8 @@ export interface FetchHtmlOptions {
timeoutMs?: number; timeoutMs?: number;
onRateInfo?: (remaining: string | null, reset: string | null) => void; onRateInfo?: (remaining: string | null, reset: string | null) => void;
headers?: Record<string, string>; headers?: Record<string, string>;
includeResponseUrl?: boolean;
jitter?: () => number;
} }
/** /**
@@ -80,14 +93,24 @@ export interface FetchHtmlOptions {
* @param url - The URL to fetch * @param url - The URL to fetch
* @param delayMs - Delay in milliseconds between requests (rate limiting) * @param delayMs - Delay in milliseconds between requests (rate limiting)
* @param opts - Optional fetch options * @param opts - Optional fetch options
* @returns The HTML content as a string * @returns The HTML content as a string, or an object with html and responseUrl
* @throws HttpError, NetworkError, or RateLimitError on failure * @throws HttpError, NetworkError, or RateLimitError on failure
*/ */
export async function fetchHtml(
url: string,
delayMs: number,
opts: FetchHtmlOptions & { includeResponseUrl: true },
): Promise<FetchHtmlResult>;
export async function fetchHtml( export async function fetchHtml(
url: string, url: string,
delayMs: number, delayMs: number,
opts?: FetchHtmlOptions, opts?: FetchHtmlOptions,
): Promise<string> { ): Promise<HTMLString>;
export async function fetchHtml(
url: string,
delayMs: number,
opts?: FetchHtmlOptions,
): Promise<HTMLString | FetchHtmlResult> {
const maxRetries = opts?.maxRetries ?? 3; const maxRetries = opts?.maxRetries ?? 3;
const retryBaseMs = opts?.retryBaseMs ?? 1000; const retryBaseMs = opts?.retryBaseMs ?? 1000;
const timeoutMs = opts?.timeoutMs ?? 30000; const timeoutMs = opts?.timeoutMs ?? 30000;
@@ -138,10 +161,10 @@ export async function fetchHtml(
: Number.NaN; : Number.NaN;
const waitMs = Number.isFinite(resetSeconds) const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000) ? Math.max(0, resetSeconds * 1000)
: calculateBackoffDelay(attempt, retryBaseMs); : calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random);
if (attempt < maxRetries) { if (attempt < maxRetries) {
await new Promise((resolve) => setTimeout(resolve, waitMs)); await delay(waitMs);
continue; continue;
} }
throw new RateLimitError( throw new RateLimitError(
@@ -153,9 +176,7 @@ export async function fetchHtml(
// Retry on server errors // Retry on server errors
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) { if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
await new Promise((resolve) => await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random));
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
);
continue; continue;
} }
@@ -170,7 +191,9 @@ export async function fetchHtml(
// Respect per-request delay to maintain rate limiting // Respect per-request delay to maintain rate limiting
await delay(delayMs); await delay(delayMs);
return html; return opts?.includeResponseUrl
? { html, responseUrl: res.url || url }
: html;
} catch (err) { } catch (err) {
// Re-throw known errors // Re-throw known errors
if ( if (
@@ -183,9 +206,7 @@ export async function fetchHtml(
if (err instanceof Error && err.name === "AbortError") { if (err instanceof Error && err.name === "AbortError") {
if (attempt < maxRetries) { if (attempt < maxRetries) {
await new Promise((resolve) => await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random));
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
);
continue; continue;
} }
throw new NetworkError(`Request timeout for ${url}`, url, err); throw new NetworkError(`Request timeout for ${url}`, url, err);
@@ -193,9 +214,7 @@ export async function fetchHtml(
// Network or other errors // Network or other errors
if (attempt < maxRetries) { if (attempt < maxRetries) {
await new Promise((resolve) => await delay(calculateBackoffDelay(attempt, retryBaseMs, opts?.jitter ?? Math.random));
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
);
continue; continue;
} }
throw new NetworkError( throw new NetworkError(

3
packages/core/src/utils/logger.ts
View File

@@ -7,4 +7,7 @@ export const logger = {
warn: (...args: Parameters<typeof console.warn>) => { warn: (...args: Parameters<typeof console.warn>) => {
if (!isTest()) console.warn(...args); if (!isTest()) console.warn(...args);
}, },
error: (...args: Parameters<typeof console.error>) => {
if (!isTest()) console.error(...args);
},
}; };

19
packages/core/test/ebay-core.test.ts
View File

@@ -32,6 +32,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => Promise.resolve("<html><body></body></html>"), text: () => Promise.resolve("<html><body></body></html>"),
}), }),
) as unknown as typeof fetch; ) as unknown as typeof fetch;
@@ -64,6 +65,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -88,6 +90,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -114,6 +117,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -146,6 +150,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -188,6 +193,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -214,6 +220,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -243,6 +250,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -272,6 +280,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -301,6 +310,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -343,6 +353,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -375,6 +386,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -407,6 +419,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -440,6 +453,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -467,6 +481,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -499,6 +514,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -529,6 +545,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -574,6 +591,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>
@@ -612,6 +630,7 @@ describe("eBay Scraper Cookie Handling", () => {
global.fetch = mock(() => global.fetch = mock(() =>
Promise.resolve({ Promise.resolve({
ok: true, ok: true,
headers: { get: () => null },
text: () => text: () =>
Promise.resolve(` Promise.resolve(`
<html><body> <html><body>

19
packages/core/test/http.test.ts
View File

@@ -38,4 +38,23 @@ describe("fetchHtml", () => {
expect(scheduledDelays).not.toContain(1000); expect(scheduledDelays).not.toContain(1000);
}); });
test("fetchHtml returns responseUrl when includeResponseUrl is true", async () => {
process.env.NODE_ENV = "test";
global.fetch = mock(() =>
Promise.resolve({
ok: true,
status: 200,
url: "https://example.test/final",
headers: { get: () => null },
text: () => Promise.resolve("<html></html>"),
}),
) as unknown as typeof fetch;
const result = await fetchHtml("https://example.test", 0, {
includeResponseUrl: true,
});
expect(result.html).toBe("<html></html>");
expect(result.responseUrl).toBe("https://example.test/final");
});
}); });