old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: biome lint and formatting

Browse Source 
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
This commit is contained in:
Dmytro Stanchiev
2026-04-28 19:21:16 -04:00
parent 49e90d45f8
commit 957e0f137b
10 changed files with 465 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
packages/core/src/scrapers/ebay.ts
View File

@@ -4,13 +4,13 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type CookieConfig,
ensureCookies,
formatCookiesForHeader,
} from "../utils/cookies";
import { delay } from "../utils/delay";
import { classifyUnstableListings } from "../utils/unstable";
// eBay cookie configuration
const EBAY_COOKIE_CONFIG: CookieConfig = {
@@ -44,7 +44,9 @@ function canonicalizeEbayItemUrl(url: string): string {
try {
const parsed = new URL(url, "https://www.ebay.ca");
const match = parsed.pathname.match(/\/itm\/(?:[^/?#]+\/)?\d+/);
return match ? `${parsed.origin}${match[0]}` : `${parsed.origin}${parsed.pathname}`;
return match
? `${parsed.origin}${match[0]}`
: `${parsed.origin}${parsed.pathname}`;
} catch {
return url;
}
@@ -275,11 +277,7 @@ function parseEbayListings(
const actualPrices: HTMLElement[] = [];
for (const el of allPriceElements) {
const text = el.textContent?.trim();
if (
text &&
EBAY_PRICE_TEXT_RE.test(text) &&
text.length < 50
) {
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
actualPrices.push(el);
}
}
@@ -386,16 +384,18 @@ async function loadEbayCookies(): Promise<string | undefined> {
export default async function fetchEbayItems(
SEARCH_QUERY: string,
REQUESTS_PER_SECOND: number | undefined,
opts: {
minPrice?: number;
maxPrice?: number;
strictMode?: boolean;
exclusions?: string[];
keywords?: string[];
buyItNowOnly?: boolean;
canadaOnly?: boolean;
maxItems?: number;
} | undefined,
opts:
| {
minPrice?: number;
maxPrice?: number;
strictMode?: boolean;
exclusions?: string[];
keywords?: string[];
buyItNowOnly?: boolean;
canadaOnly?: boolean;
maxItems?: number;
}
| undefined,
unstableMode: { hideUnstableResults: true },
): Promise<UnstableListingBuckets<EbayListingDetails>>;
export default async function fetchEbayItems(
@@ -529,7 +529,9 @@ export default async function fetchEbayItems(
// Filter by price range (additional safety check)
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return typeof cents === "number" && cents >= minPrice && cents <= maxPrice;
return (
typeof cents === "number" && cents >= minPrice && cents <= maxPrice
);
});
console.log(`Parsed ${filteredListings.length} eBay listings.`);

62
packages/core/src/scrapers/facebook.ts
View File

@@ -5,7 +5,6 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type Cookie,
type CookieConfig,
@@ -16,6 +15,7 @@ import {
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http";
import { classifyUnstableListings } from "../utils/unstable";
/**
* Facebook Marketplace Scraper
@@ -408,7 +408,11 @@ export function classifyFacebookResponse(
htmlString.includes("This listing is no longer available") ||
htmlString.includes("listing has been removed");
if (unavailable) {
return { kind: "unavailable" as const, authGated: false, unavailable: true };
return {
kind: "unavailable" as const,
authGated: false,
unavailable: true,
};
}
if (responseUrl.includes("/marketplace/item/")) {
@@ -455,7 +459,8 @@ function isFacebookSearchEdgeArray(value: unknown): value is FacebookEdge[] {
Array.isArray(value) &&
value.length > 0 &&
value.every(
(edge) => isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
(edge) =>
isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
)
);
}
@@ -552,8 +557,7 @@ function scoreMarketplaceItemPath(path: string[]): number {
if (
path.some(
(segment) =>
segment.includes("recommend") || segment.includes("related"),
(segment) => segment.includes("recommend") || segment.includes("related"),
)
) {
score -= 10;
@@ -567,7 +571,9 @@ function collectMarketplaceItemCandidates(
path: string[] = [],
): FacebookMarketplaceItemMatch[] {
if (Array.isArray(candidate)) {
return candidate.flatMap((item) => collectMarketplaceItemCandidates(item, path));
return candidate.flatMap((item) =>
collectMarketplaceItemCandidates(item, path),
);
}
if (!isRecord(candidate)) {
@@ -628,7 +634,9 @@ function extractRenderedText(node: ParentNode, selector: string): string[] {
.filter((text): text is string => Boolean(text));
}
function extractMarketplaceItemIdFromElement(element: Element | null): string | null {
function extractMarketplaceItemIdFromElement(
element: Element | null,
): string | null {
const href = element?.getAttribute("href") || "";
return href.match(FACEBOOK_ITEM_HREF_RE)?.[1] ?? null;
}
@@ -666,7 +674,9 @@ function extractFacebookPermalinkItemId(document: Document): string | null {
return extractMarketplaceItemIdFromElement(itemLinks.at(-1) ?? null);
}
function extractFacebookDescriptionText(document: Document): string | undefined {
function extractFacebookDescriptionText(
document: Document,
): string | undefined {
const labels = Array.from(document.querySelectorAll("div, span, h2, h3, p"));
for (const label of labels) {
@@ -759,7 +769,10 @@ function extractFacebookItemHtmlFallback(
const priceText = texts.find((text) => FACEBOOK_PRICE_TEXT_RE.test(text));
const parsedPrice = priceText ? parseFacebookRenderedPrice(priceText) : null;
const location = texts.find(
(text) => text !== title && text !== priceText && FACEBOOK_LOCATION_TEXT_RE.test(text),
(text) =>
text !== title &&
text !== priceText &&
FACEBOOK_LOCATION_TEXT_RE.test(text),
);
const description = extractFacebookDescriptionText(document);
@@ -841,7 +854,8 @@ export function extractFacebookItemData(
if (
!bestMatch ||
match.score > bestMatch.score ||
(match.score === bestMatch.score && match.path.length < bestMatch.path.length)
(match.score === bestMatch.score &&
match.path.length < bestMatch.path.length)
) {
bestMatch = match;
}
@@ -1101,7 +1115,9 @@ export default async function fetchFacebookItems(
const finalizeResults = (
listings: FacebookListingDetails[],
): FacebookListingDetails[] | UnstableListingBuckets<FacebookListingDetails> => {
):
| FacebookListingDetails[]
| UnstableListingBuckets<FacebookListingDetails> => {
if (!unstableMode.hideUnstableResults) {
return listings.slice(0, MAX_ITEMS);
}
@@ -1166,9 +1182,14 @@ export default async function fetchFacebookItems(
throw err;
}
const classification = classifyFacebookResponse(searchHtml, searchResponseUrl);
const classification = classifyFacebookResponse(
searchHtml,
searchResponseUrl,
);
if (classification.authGated) {
console.warn("Facebook marketplace search redirected to login. Cookies may be expired.");
console.warn(
"Facebook marketplace search redirected to login. Cookies may be expired.",
);
return finalizeResults([]);
}
@@ -1204,7 +1225,8 @@ export default async function fetchFacebookItems(
// Filter to only priced items (already done in parseFacebookAds)
const pricedItems = items.filter(
(item) =>
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
typeof item.listingPrice?.cents === "number" &&
item.listingPrice.cents >= 0,
);
progressBar?.update(totalProgress);
@@ -1293,7 +1315,9 @@ export async function fetchFacebookItem(
if (classification.authGated) {
logExtractionMetrics(false, itemId);
console.warn(`Authentication failed for item ${itemId}. Cookies may be expired.`);
console.warn(
`Authentication failed for item ${itemId}. Cookies may be expired.`,
);
return null;
}
@@ -1301,7 +1325,9 @@ export async function fetchFacebookItem(
if (classification.unavailable && !itemData) {
logExtractionMetrics(false, itemId);
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}
@@ -1317,7 +1343,9 @@ export async function fetchFacebookItem(
logExtractionMetrics(false, itemId);
if (itemHtml.includes("This item has been sold")) {
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}

26
packages/core/src/scrapers/kijiji.ts
View File

@@ -6,7 +6,6 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type CookieConfig,
formatCookiesForHeader,
@@ -22,6 +21,7 @@ import {
RateLimitError,
ValidationError,
} from "../utils/http";
import { classifyUnstableListings } from "../utils/unstable";
// Kijiji cookie configuration
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
@@ -203,11 +203,17 @@ const SORT_MAPPINGS: Record<string, string> = {
};
const LOCATION_SLUGS = Object.fromEntries(
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [
id,
slug.replace(/\s+/g, "-"),
]),
) as Record<number, string>;
const CATEGORY_SLUGS = Object.fromEntries(
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [
id,
slug.replace(/\s+/g, "-"),
]),
) as Record<number, string>;
// ----------------------------- Utilities -----------------------------
@@ -816,7 +822,10 @@ export default async function fetchKijijiItems(
: undefined;
// Set defaults for configuration
const finalSearchOptions: Omit<Required<SearchOptions>, "priceMin" | "priceMax"> & {
const finalSearchOptions: Omit<
Required<SearchOptions>,
"priceMin" | "priceMax"
> & {
priceMin?: number;
priceMax?: number;
} = {
@@ -903,7 +912,9 @@ export default async function fetchKijijiItems(
const batchPromises = batch.map(async (link, batchIndex) => {
try {
if (batchIndex > 0) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS * batchIndex));
await new Promise((resolve) =>
setTimeout(resolve, DELAY_MS * batchIndex),
);
}
const html = await fetchHtml(link, 0, {
@@ -949,7 +960,6 @@ export default async function fetchKijijiItems(
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
}
allListings.push(
@@ -968,9 +978,7 @@ export default async function fetchKijijiItems(
matchesPriceFilters(listing, finalSearchOptions),
);
console.log(
`\nParsed ${filteredListings.length} detailed listings.`,
);
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
return finalizeResults(filteredListings);
}