refactor: improve Kijiji scraping architecture
Introduces types, utilities, and a dedicated HTTP client for improved structure and maintainability. Includes rate limit handling, retry logic, and error handling.
This commit is contained in:
430
src/kijiji.ts
430
src/kijiji.ts
@@ -1,159 +1,234 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
import { parseHTML } from "linkedom";
|
||||
|
||||
// ----------------------------- Types -----------------------------
|
||||
|
||||
type HTMLString = string;
|
||||
|
||||
type SearchListing = {
|
||||
name: string;
|
||||
listingLink: string;
|
||||
};
|
||||
|
||||
interface ApolloSearchState {
|
||||
[key: string]: {
|
||||
[key: string]: unknown;
|
||||
type ApolloRecord = Record<string, unknown>;
|
||||
|
||||
interface ApolloSearchItem {
|
||||
url?: string;
|
||||
title?: string;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
interface ApolloListingRoot {
|
||||
url?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
price?: { amount?: number | string; currency?: string };
|
||||
type?: string;
|
||||
status?: string;
|
||||
activationDate?: string;
|
||||
endDate?: string;
|
||||
metrics?: { views?: number | string };
|
||||
location?: { address?: string | null };
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
type ListingDetails = {
|
||||
url: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
listingPrice?: {
|
||||
amountFormatted: string;
|
||||
cents?: number;
|
||||
currency?: string;
|
||||
};
|
||||
listingType?: string;
|
||||
listingStatus?: string;
|
||||
creationDate?: string;
|
||||
endDate?: string;
|
||||
numberOfViews?: number;
|
||||
address?: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
interface ApolloListingState {
|
||||
[key: string]: any;
|
||||
}
|
||||
// ----------------------------- Config -----------------------------
|
||||
|
||||
function formatCentsToCurrency(num: number | string, locale = "en-US") {
|
||||
if (typeof num === "string") num = parseInt(num);
|
||||
const numberInDollars = num / 100;
|
||||
const REQUESTS_PER_SECOND = 1;
|
||||
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
|
||||
const BASE_URL = "https://www.kijiji.ca";
|
||||
const SEARCH_QUERY = "playstation 5";
|
||||
|
||||
// ----------------------------- Utilities -----------------------------
|
||||
|
||||
/**
|
||||
* Turns cents to localized currency string.
|
||||
*/
|
||||
function formatCentsToCurrency(
|
||||
num: number | string | undefined,
|
||||
locale = "en-US",
|
||||
): string {
|
||||
if (num == null) return "";
|
||||
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
|
||||
if (Number.isNaN(cents)) return "";
|
||||
const dollars = cents / 100;
|
||||
const formatter = new Intl.NumberFormat(locale, {
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: 2,
|
||||
useGrouping: true,
|
||||
});
|
||||
|
||||
return formatter.format(numberInDollars);
|
||||
return formatter.format(dollars);
|
||||
}
|
||||
|
||||
const searchQuery = "playstation 5";
|
||||
const REQUESTS_PER_SECOND = 1;
|
||||
const DELAY_MS = 1000 / REQUESTS_PER_SECOND;
|
||||
|
||||
// const exampleSearchHTML = Bun.file("./example-kijiji-search.html");
|
||||
// const exampleSearchHTMLData = await exampleSearchHTML.text();
|
||||
|
||||
function parseSearch(htmlString: string) {
|
||||
const { document } = parseHTML(htmlString);
|
||||
const nextData = document.getElementById("__NEXT_DATA__");
|
||||
|
||||
if (!nextData) {
|
||||
console.error("Could not find __NEXT_DATA__ script element.");
|
||||
return [];
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
async function delay(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
class HttpError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly status: number,
|
||||
public readonly url: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "HttpError";
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------- HTTP Client -----------------------------
|
||||
|
||||
/**
|
||||
Fetch HTML with a basic retry strategy and simple rate-limit delay between calls.
|
||||
- Retries on 429 and 5xx
|
||||
- Respects X-RateLimit-Reset when present (seconds)
|
||||
*/
|
||||
async function fetchHtml(
|
||||
url: string,
|
||||
opts?: {
|
||||
maxRetries?: number;
|
||||
retryBaseMs?: number;
|
||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
||||
},
|
||||
): Promise<HTMLString> {
|
||||
const maxRetries = opts?.maxRetries ?? 3;
|
||||
const retryBaseMs = opts?.retryBaseMs ?? 500;
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
try {
|
||||
if (!nextData.textContent) {
|
||||
console.error("__NEXT_DATA__ element is empty!");
|
||||
return [];
|
||||
}
|
||||
const jsonData = JSON.parse(nextData.textContent);
|
||||
const apolloState: ApolloSearchState =
|
||||
jsonData.props.pageProps.__APOLLO_STATE__;
|
||||
|
||||
const listingsKeys: string[] = [];
|
||||
|
||||
for (const key in apolloState) {
|
||||
if (key.includes("Listing")) {
|
||||
listingsKeys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
const searchListings: SearchListing[] = listingsKeys.map((key) => {
|
||||
const listing = apolloState[key];
|
||||
return {
|
||||
listingLink: listing!.url,
|
||||
name: listing!.title,
|
||||
};
|
||||
});
|
||||
|
||||
// console.log(searchListings);
|
||||
|
||||
return searchListings;
|
||||
} catch (error) {
|
||||
console.error("Error parsing __NEXT_DATA__:", error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const makeKijijiRequest = async <T>(url: string): Promise<T> => {
|
||||
console.log(`Making a request at ${new Date()}`);
|
||||
const response = await fetch(url, {
|
||||
const res = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
accept:
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
||||
"cache-control": "max-age=0",
|
||||
priority: "u=0, i",
|
||||
"sec-ch-ua": '"Not)A;Brand";v="8", "Chromium";v="138"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Linux"',
|
||||
"sec-fetch-dest": "document",
|
||||
"sec-fetch-mode": "navigate",
|
||||
"sec-fetch-site": "same-origin",
|
||||
"sec-fetch-user": "?1",
|
||||
"cache-control": "no-cache",
|
||||
"upgrade-insecure-requests": "1",
|
||||
"user-agent":
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
||||
},
|
||||
body: null,
|
||||
method: "GET",
|
||||
});
|
||||
|
||||
const rateLimitRemaining = response.headers.get("X-RateLimit-Remaining");
|
||||
const rateLimitReset = response.headers.get("X-RateLimit-Reset");
|
||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
||||
|
||||
if (rateLimitRemaining !== null && rateLimitReset !== null) {
|
||||
console.log(
|
||||
`Rate limit remaining: ${rateLimitRemaining}, Reset in: ${rateLimitReset} seconds`,
|
||||
if (!res.ok) {
|
||||
// Respect 429 reset if provided
|
||||
if (res.status === 429) {
|
||||
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : NaN;
|
||||
const waitMs = Number.isFinite(resetSeconds)
|
||||
? Math.max(0, resetSeconds * 1000)
|
||||
: (attempt + 1) * retryBaseMs;
|
||||
await delay(waitMs);
|
||||
continue;
|
||||
}
|
||||
// Retry on 5xx
|
||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
||||
await delay((attempt + 1) * retryBaseMs);
|
||||
continue;
|
||||
}
|
||||
throw new HttpError(
|
||||
`Request failed with status ${res.status}`,
|
||||
res.status,
|
||||
url,
|
||||
);
|
||||
}
|
||||
|
||||
const data: T = (await response.text()) as T;
|
||||
return data;
|
||||
};
|
||||
|
||||
async function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const html = await res.text();
|
||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
||||
await delay(DELAY_MS);
|
||||
return html;
|
||||
} catch (err) {
|
||||
if (attempt >= maxRetries) throw err;
|
||||
await delay((attempt + 1) * retryBaseMs);
|
||||
}
|
||||
}
|
||||
|
||||
// const exampleListing = await Bun.file("./examples/apollo_listing.json").json();
|
||||
// const exampleListingApolloState =
|
||||
// exampleListing.props.pageProps.__APOLLO_STATE__;
|
||||
throw new Error("Exhausted retries without response");
|
||||
}
|
||||
|
||||
const parseListing = (htmlString: string) => {
|
||||
// ----------------------------- Parsing -----------------------------
|
||||
|
||||
/**
|
||||
Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML.
|
||||
*/
|
||||
function extractApolloState(htmlString: HTMLString): ApolloRecord | null {
|
||||
const { document } = parseHTML(htmlString);
|
||||
const nextData = document.getElementById("__NEXT_DATA__");
|
||||
if (!nextData || !nextData.textContent) return null;
|
||||
|
||||
if (!nextData) {
|
||||
console.error("Could not find __NEXT_DATA__ script element.");
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!nextData.textContent) {
|
||||
console.error("__NEXT_DATA__ element is empty!");
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const jsonData = JSON.parse(nextData.textContent);
|
||||
const apolloState: ApolloListingState =
|
||||
jsonData.props.pageProps.__APOLLO_STATE__;
|
||||
|
||||
const getListingId = (apolloState: { [key: string]: any }):
|
||||
| string
|
||||
| undefined => {
|
||||
const apolloStateKeys = Object.keys(apolloState);
|
||||
const key = apolloStateKeys.find((key) => key.includes("Listing"));
|
||||
if (!key) return undefined;
|
||||
return key;
|
||||
};
|
||||
|
||||
const listingKey = getListingId(apolloState);
|
||||
|
||||
if (!listingKey) {
|
||||
throw new Error("No listing key found in listing apolloState!");
|
||||
const apollo = jsonData?.props?.pageProps?.__APOLLO_STATE__;
|
||||
return isRecord(apollo) ? apollo : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
Parse search page apollo state into SearchListing[].
|
||||
Filters keys likely to be listing entities and ensures url/title exist.
|
||||
*/
|
||||
function parseSearch(htmlString: HTMLString): SearchListing[] {
|
||||
const apolloState = extractApolloState(htmlString);
|
||||
if (!apolloState) return [];
|
||||
|
||||
const results: SearchListing[] = [];
|
||||
for (const [key, value] of Object.entries(apolloState)) {
|
||||
// Heuristic: Kijiji listing keys usually contain "Listing"
|
||||
if (!key.includes("Listing")) continue;
|
||||
if (!isRecord(value)) continue;
|
||||
|
||||
const item = value as ApolloSearchItem;
|
||||
if (typeof item.url === "string" && typeof item.title === "string") {
|
||||
results.push({
|
||||
listingLink: item.url.startsWith("http")
|
||||
? item.url
|
||||
: `${BASE_URL}${item.url}`,
|
||||
name: item.title,
|
||||
});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
Parse a listing page into a typed object.
|
||||
*/
|
||||
function parseListing(htmlString: HTMLString): ListingDetails | null {
|
||||
const apolloState = extractApolloState(htmlString);
|
||||
if (!apolloState) return null;
|
||||
|
||||
// Find the listing root key
|
||||
const listingKey = Object.keys(apolloState).find((k) =>
|
||||
k.includes("Listing"),
|
||||
);
|
||||
if (!listingKey) return null;
|
||||
|
||||
const root = apolloState[listingKey];
|
||||
if (!isRecord(root)) return null;
|
||||
|
||||
const {
|
||||
url,
|
||||
@@ -165,66 +240,105 @@ const parseListing = (htmlString: string) => {
|
||||
activationDate,
|
||||
endDate,
|
||||
metrics,
|
||||
// attributes,
|
||||
location,
|
||||
} = apolloState[listingKey];
|
||||
} = root as ApolloListingRoot;
|
||||
|
||||
const listingObject = {
|
||||
url,
|
||||
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
||||
const amountFormatted = formatCentsToCurrency(cents);
|
||||
|
||||
const numberOfViews =
|
||||
metrics?.views != null ? Number(metrics.views) : undefined;
|
||||
|
||||
const listingUrl =
|
||||
typeof url === "string"
|
||||
? url.startsWith("http")
|
||||
? url
|
||||
: `${BASE_URL}${url}`
|
||||
: "";
|
||||
|
||||
if (!listingUrl || !title) return null;
|
||||
|
||||
return {
|
||||
url: listingUrl,
|
||||
title,
|
||||
description,
|
||||
listingPrice: {
|
||||
amount: formatCentsToCurrency(price.amount),
|
||||
currency: price.currency,
|
||||
},
|
||||
listingPrice: amountFormatted
|
||||
? {
|
||||
amountFormatted,
|
||||
cents: Number.isFinite(cents!) ? cents : undefined,
|
||||
currency: price?.currency,
|
||||
}
|
||||
: undefined,
|
||||
listingType: type,
|
||||
listingStatus: status,
|
||||
creationDate: activationDate,
|
||||
endDate,
|
||||
numberOfViews: metrics.views,
|
||||
// condition: attributes.all.find(
|
||||
// (attr: { [key: string]: unknown }) => attr.canonicalName === "condition",
|
||||
// ).canonicalValues[0],
|
||||
address: location.address,
|
||||
numberOfViews: Number.isFinite(numberOfViews!) ? numberOfViews : undefined,
|
||||
address: location?.address ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
return listingObject;
|
||||
};
|
||||
// ----------------------------- Main -----------------------------
|
||||
|
||||
const searchHtml: string = await makeKijijiRequest(
|
||||
`https://www.kijiji.ca/b-canada/${searchQuery}/k0l0?dc=true&view=list`,
|
||||
async function main() {
|
||||
const searchUrl = `${BASE_URL}/b-canada/${encodeURIComponent(SEARCH_QUERY)}/k0l0?dc=true&view=list`;
|
||||
|
||||
console.log(`Fetching search: ${searchUrl}`);
|
||||
const searchHtml = await fetchHtml(searchUrl, {
|
||||
onRateInfo: (remaining, reset) => {
|
||||
if (remaining && reset) {
|
||||
console.log(
|
||||
`Search - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||
);
|
||||
|
||||
const searchResults = parseSearch(searchHtml);
|
||||
|
||||
// if (searchResults.length === 0) {
|
||||
// throw new Error("Search didn't return an HTML!")
|
||||
// }
|
||||
// console.log(searchResults);
|
||||
|
||||
const fetchAllWithRateLimit = async (links: string[]) => {
|
||||
const results: string[] = [];
|
||||
for (const link of links) {
|
||||
try {
|
||||
const data: string = await makeKijijiRequest(link);
|
||||
// console.log(data);
|
||||
results.push(data);
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch data from ${link}:`, error);
|
||||
}
|
||||
await delay(DELAY_MS);
|
||||
}
|
||||
return results;
|
||||
};
|
||||
|
||||
const listingsLinks: string[] = searchResults.map((item) => {
|
||||
return item.listingLink;
|
||||
},
|
||||
});
|
||||
|
||||
// console.log(listingsLinks);
|
||||
const searchResults = parseSearch(searchHtml);
|
||||
if (searchResults.length === 0) {
|
||||
console.warn("No search results parsed from page.");
|
||||
return;
|
||||
}
|
||||
|
||||
const fetchResults = await fetchAllWithRateLimit(listingsLinks);
|
||||
// Deduplicate links
|
||||
const listingLinks = Array.from(
|
||||
new Set(searchResults.map((r) => r.listingLink)),
|
||||
);
|
||||
|
||||
const itemsData = fetchResults.map((itemHtml) => parseListing(itemHtml));
|
||||
console.log(
|
||||
`Found ${listingLinks.length} listing links. Fetching details...`,
|
||||
);
|
||||
|
||||
console.log(itemsData);
|
||||
const items: ListingDetails[] = [];
|
||||
for (const link of listingLinks) {
|
||||
try {
|
||||
const html = await fetchHtml(link, {
|
||||
onRateInfo: (remaining, reset) => {
|
||||
if (remaining && reset) {
|
||||
console.log(
|
||||
`Item - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
const parsed = parseListing(html);
|
||||
if (parsed) items.push(parsed);
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError) {
|
||||
console.error(`Failed to fetch ${link} - ${err.status} ${err.message}`);
|
||||
} else {
|
||||
console.error(
|
||||
`Failed to fetch ${link} - ${String((err as Error)?.message || err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Parsed ${items.length} listings.`);
|
||||
console.log(items);
|
||||
}
|
||||
|
||||
void main().catch((err) => {
|
||||
console.error("Fatal error:", err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user