fix: resolve biome lint errors and warnings
This commit is contained in:
@@ -1,45 +1,42 @@
|
||||
// Export all scrapers
|
||||
export {
|
||||
default as fetchKijijiItems,
|
||||
slugify,
|
||||
resolveLocationId,
|
||||
resolveCategoryId,
|
||||
buildSearchUrl,
|
||||
extractApolloState,
|
||||
parseSearch,
|
||||
parseDetailedListing,
|
||||
HttpError,
|
||||
NetworkError,
|
||||
ParseError,
|
||||
RateLimitError,
|
||||
ValidationError,
|
||||
} from "./scrapers/kijiji";
|
||||
export type {
|
||||
KijijiListingDetails,
|
||||
DetailedListing,
|
||||
SearchOptions,
|
||||
ListingFetchOptions,
|
||||
} from "./scrapers/kijiji";
|
||||
|
||||
export {
|
||||
default as fetchFacebookItems,
|
||||
fetchFacebookItem,
|
||||
parseFacebookCookieString,
|
||||
ensureFacebookCookies,
|
||||
extractFacebookMarketplaceData,
|
||||
extractFacebookItemData,
|
||||
parseFacebookAds,
|
||||
parseFacebookItem,
|
||||
} from "./scrapers/facebook";
|
||||
export type { FacebookListingDetails } from "./scrapers/facebook";
|
||||
|
||||
export { default as fetchEbayItems } from "./scrapers/ebay";
|
||||
export type { EbayListingDetails } from "./scrapers/ebay";
|
||||
|
||||
// Export shared utilities
|
||||
export * from "./utils/http";
|
||||
export * from "./utils/delay";
|
||||
export * from "./utils/format";
|
||||
|
||||
export { default as fetchEbayItems } from "./scrapers/ebay";
|
||||
export type { FacebookListingDetails } from "./scrapers/facebook";
|
||||
export {
|
||||
default as fetchFacebookItems,
|
||||
ensureFacebookCookies,
|
||||
extractFacebookItemData,
|
||||
extractFacebookMarketplaceData,
|
||||
fetchFacebookItem,
|
||||
parseFacebookAds,
|
||||
parseFacebookCookieString,
|
||||
parseFacebookItem,
|
||||
} from "./scrapers/facebook";
|
||||
export type {
|
||||
DetailedListing,
|
||||
KijijiListingDetails,
|
||||
ListingFetchOptions,
|
||||
SearchOptions,
|
||||
} from "./scrapers/kijiji";
|
||||
export {
|
||||
buildSearchUrl,
|
||||
default as fetchKijijiItems,
|
||||
extractApolloState,
|
||||
HttpError,
|
||||
NetworkError,
|
||||
ParseError,
|
||||
parseDetailedListing,
|
||||
parseSearch,
|
||||
RateLimitError,
|
||||
resolveCategoryId,
|
||||
resolveLocationId,
|
||||
slugify,
|
||||
ValidationError,
|
||||
} from "./scrapers/kijiji";
|
||||
// Export shared types
|
||||
export * from "./types/common";
|
||||
export * from "./utils/delay";
|
||||
export * from "./utils/format";
|
||||
// Export shared utilities
|
||||
export * from "./utils/http";
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
import { parseHTML } from "linkedom";
|
||||
import type { HTMLString } from "../types/common";
|
||||
import { delay } from "../utils/delay";
|
||||
import { formatCentsToCurrency } from "../utils/format";
|
||||
import { isRecord } from "../utils/http";
|
||||
|
||||
// ----------------------------- Types -----------------------------
|
||||
|
||||
@@ -43,7 +38,7 @@ function parseEbayPrice(
|
||||
|
||||
const amountStr = numberMatches[0].replace(/,/g, "");
|
||||
const dollars = parseFloat(amountStr);
|
||||
if (isNaN(dollars)) return null;
|
||||
if (Number.isNaN(dollars)) return null;
|
||||
|
||||
const cents = Math.round(dollars * 100);
|
||||
|
||||
@@ -185,8 +180,7 @@ function parseEbayListings(
|
||||
const text = el.textContent?.trim();
|
||||
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
|
||||
if (
|
||||
text &&
|
||||
text.includes("$") &&
|
||||
text?.includes("$") &&
|
||||
text.length < 100 &&
|
||||
!text.includes("laptop") &&
|
||||
!text.includes("computer") &&
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -3,18 +3,18 @@ export type HTMLString = string;
|
||||
|
||||
/** Currency price object with formatting options */
|
||||
export interface Price {
|
||||
amountFormatted: string;
|
||||
cents: number;
|
||||
currency: string;
|
||||
amountFormatted: string;
|
||||
cents: number;
|
||||
currency: string;
|
||||
}
|
||||
|
||||
/** Base listing details common across all marketplaces */
|
||||
export interface ListingDetails {
|
||||
url: string;
|
||||
title: string;
|
||||
listingPrice: Price;
|
||||
listingType: string;
|
||||
listingStatus: string;
|
||||
address?: string | null;
|
||||
creationDate?: string;
|
||||
url: string;
|
||||
title: string;
|
||||
listingPrice: Price;
|
||||
listingType: string;
|
||||
listingStatus: string;
|
||||
address?: string | null;
|
||||
creationDate?: string;
|
||||
}
|
||||
|
||||
@@ -4,5 +4,5 @@
|
||||
* @returns A promise that resolves after the specified delay
|
||||
*/
|
||||
export function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
@@ -4,18 +4,21 @@
|
||||
* @param locale - Locale string for formatting (e.g., 'en-CA', 'en-US')
|
||||
* @returns Formatted currency string
|
||||
*/
|
||||
export function formatCentsToCurrency(cents: number, locale: string = "en-CA"): string {
|
||||
try {
|
||||
const formatter = new Intl.NumberFormat(locale, {
|
||||
style: "currency",
|
||||
currency: "CAD",
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: 2,
|
||||
});
|
||||
return formatter.format(cents / 100);
|
||||
} catch (error) {
|
||||
// Fallback if locale is not supported
|
||||
const dollars = (cents / 100).toFixed(2);
|
||||
return `$${dollars}`;
|
||||
}
|
||||
export function formatCentsToCurrency(
|
||||
cents: number,
|
||||
locale: string = "en-CA",
|
||||
): string {
|
||||
try {
|
||||
const formatter = new Intl.NumberFormat(locale, {
|
||||
style: "currency",
|
||||
currency: "CAD",
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: 2,
|
||||
});
|
||||
return formatter.format(cents / 100);
|
||||
} catch {
|
||||
// Fallback if locale is not supported
|
||||
const dollars = (cents / 100).toFixed(2);
|
||||
return `$${dollars}`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,79 +1,79 @@
|
||||
/** Custom error class for HTTP-related failures */
|
||||
export class HttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly statusCode: number,
|
||||
public readonly url?: string
|
||||
) {
|
||||
super(message);
|
||||
this.name = "HttpError";
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly statusCode: number,
|
||||
public readonly url?: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "HttpError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Error class for network failures (timeouts, connection issues) */
|
||||
export class NetworkError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly url: string,
|
||||
public readonly cause?: Error
|
||||
) {
|
||||
super(message);
|
||||
this.name = "NetworkError";
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly url: string,
|
||||
public readonly cause?: Error,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "NetworkError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Error class for parsing failures */
|
||||
export class ParseError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly data?: unknown
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ParseError";
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly data?: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "ParseError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Error class for rate limiting */
|
||||
export class RateLimitError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly url: string,
|
||||
public readonly resetTime?: number
|
||||
) {
|
||||
super(message);
|
||||
this.name = "RateLimitError";
|
||||
}
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly url: string,
|
||||
public readonly resetTime?: number,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "RateLimitError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Error class for validation failures */
|
||||
export class ValidationError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ValidationError";
|
||||
}
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ValidationError";
|
||||
}
|
||||
}
|
||||
|
||||
/** Type guard to check if a value is a record (object) */
|
||||
export function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate exponential backoff delay with jitter
|
||||
*/
|
||||
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
||||
const exponentialDelay = baseMs * 2 ** attempt;
|
||||
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
||||
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
||||
const exponentialDelay = baseMs * 2 ** attempt;
|
||||
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
||||
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
||||
}
|
||||
|
||||
/** Options for fetchHtml */
|
||||
export interface FetchHtmlOptions {
|
||||
maxRetries?: number;
|
||||
retryBaseMs?: number;
|
||||
timeoutMs?: number;
|
||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
||||
headers?: Record<string, string>;
|
||||
maxRetries?: number;
|
||||
retryBaseMs?: number;
|
||||
timeoutMs?: number;
|
||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -85,116 +85,116 @@ export interface FetchHtmlOptions {
|
||||
* @throws HttpError, NetworkError, or RateLimitError on failure
|
||||
*/
|
||||
export async function fetchHtml(
|
||||
url: string,
|
||||
delayMs: number,
|
||||
opts?: FetchHtmlOptions
|
||||
url: string,
|
||||
delayMs: number,
|
||||
opts?: FetchHtmlOptions,
|
||||
): Promise<string> {
|
||||
const maxRetries = opts?.maxRetries ?? 3;
|
||||
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
||||
const timeoutMs = opts?.timeoutMs ?? 30000;
|
||||
const maxRetries = opts?.maxRetries ?? 3;
|
||||
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
||||
const timeoutMs = opts?.timeoutMs ?? 30000;
|
||||
|
||||
const defaultHeaders: Record<string, string> = {
|
||||
accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
||||
"cache-control": "no-cache",
|
||||
"upgrade-insecure-requests": "1",
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
||||
};
|
||||
const defaultHeaders: Record<string, string> = {
|
||||
accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
||||
"cache-control": "no-cache",
|
||||
"upgrade-insecure-requests": "1",
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
||||
};
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
const res = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: { ...defaultHeaders, ...opts?.headers },
|
||||
signal: controller.signal,
|
||||
});
|
||||
const res = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: { ...defaultHeaders, ...opts?.headers },
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
||||
|
||||
if (!res.ok) {
|
||||
// Handle rate limiting
|
||||
if (res.status === 429) {
|
||||
const resetSeconds = rateLimitReset
|
||||
? Number(rateLimitReset)
|
||||
: Number.NaN;
|
||||
const waitMs = Number.isFinite(resetSeconds)
|
||||
? Math.max(0, resetSeconds * 1000)
|
||||
: calculateBackoffDelay(attempt, retryBaseMs);
|
||||
if (!res.ok) {
|
||||
// Handle rate limiting
|
||||
if (res.status === 429) {
|
||||
const resetSeconds = rateLimitReset
|
||||
? Number(rateLimitReset)
|
||||
: Number.NaN;
|
||||
const waitMs = Number.isFinite(resetSeconds)
|
||||
? Math.max(0, resetSeconds * 1000)
|
||||
: calculateBackoffDelay(attempt, retryBaseMs);
|
||||
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
continue;
|
||||
}
|
||||
throw new RateLimitError(
|
||||
`Rate limit exceeded for ${url}`,
|
||||
url,
|
||||
resetSeconds
|
||||
);
|
||||
}
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
continue;
|
||||
}
|
||||
throw new RateLimitError(
|
||||
`Rate limit exceeded for ${url}`,
|
||||
url,
|
||||
resetSeconds,
|
||||
);
|
||||
}
|
||||
|
||||
// Retry on server errors
|
||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs))
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// Retry on server errors
|
||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new HttpError(
|
||||
`Request failed with status ${res.status}`,
|
||||
res.status,
|
||||
url
|
||||
);
|
||||
}
|
||||
throw new HttpError(
|
||||
`Request failed with status ${res.status}`,
|
||||
res.status,
|
||||
url,
|
||||
);
|
||||
}
|
||||
|
||||
const html = await res.text();
|
||||
const html = await res.text();
|
||||
|
||||
// Respect per-request delay to maintain rate limiting
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
return html;
|
||||
} catch (err) {
|
||||
// Re-throw known errors
|
||||
if (
|
||||
err instanceof RateLimitError ||
|
||||
err instanceof HttpError ||
|
||||
err instanceof NetworkError
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
// Respect per-request delay to maintain rate limiting
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
return html;
|
||||
} catch (err) {
|
||||
// Re-throw known errors
|
||||
if (
|
||||
err instanceof RateLimitError ||
|
||||
err instanceof HttpError ||
|
||||
err instanceof NetworkError
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs))
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw new NetworkError(`Request timeout for ${url}`, url, err);
|
||||
}
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw new NetworkError(`Request timeout for ${url}`, url, err);
|
||||
}
|
||||
|
||||
// Network or other errors
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs))
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw new NetworkError(
|
||||
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
url,
|
||||
err instanceof Error ? err : undefined
|
||||
);
|
||||
}
|
||||
}
|
||||
// Network or other errors
|
||||
if (attempt < maxRetries) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
throw new NetworkError(
|
||||
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
url,
|
||||
err instanceof Error ? err : undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
|
||||
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user