old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: format ebay scraper with consistent indentation

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-01-23 09:52:25 -05:00
parent 6e0487f8f3
commit 7b106c91ce
1 changed files with 338 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
packages/core/src/scrapers/ebay.ts
View File

@@ -1,9 +1,9 @@
/* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import { isRecord } from "../utils/http"; import type { HTMLString } from "../types/common";
import { delay } from "../utils/delay"; import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format"; import { formatCentsToCurrency } from "../utils/format";
import type { HTMLString } from "../types/common"; import { isRecord } from "../utils/http";
// ----------------------------- Types ----------------------------- // ----------------------------- Types -----------------------------
@@ -29,8 +29,10 @@ export interface EbayListingDetails {
/** /**
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents * Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
*/ */
function parseEbayPrice(priceText: string): { cents: number; currency: string } | null { function parseEbayPrice(
if (!priceText || typeof priceText !== 'string') return null; priceText: string,
): { cents: number; currency: string } | null {
if (!priceText || typeof priceText !== "string") return null;
// Clean up the price text and extract currency and amount // Clean up the price text and extract currency and amount
const cleaned = priceText.trim(); const cleaned = priceText.trim();
@@ -39,19 +41,23 @@ function parseEbayPrice(priceText: string): { cents: number; currency: string }
const numberMatches = cleaned.match(/[\d,]+\.?\d*/); const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
if (!numberMatches) return null; if (!numberMatches) return null;
const amountStr = numberMatches[0].replace(/,/g, ''); const amountStr = numberMatches[0].replace(/,/g, "");
const dollars = parseFloat(amountStr); const dollars = parseFloat(amountStr);
if (isNaN(dollars)) return null; if (isNaN(dollars)) return null;
const cents = Math.round(dollars * 100); const cents = Math.round(dollars * 100);
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc. // Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
let currency = 'USD'; // Default let currency = "USD"; // Default
if (cleaned.toUpperCase().includes('CAD') || cleaned.includes('CA$') || cleaned.includes('C $')) { if (
currency = 'CAD'; cleaned.toUpperCase().includes("CAD") ||
} else if (cleaned.toUpperCase().includes('USD') || cleaned.includes('$')) { cleaned.includes("CA$") ||
currency = 'USD'; cleaned.includes("C $")
) {
currency = "CAD";
} else if (cleaned.toUpperCase().includes("USD") || cleaned.includes("$")) {
currency = "USD";
} }
return { cents, currency }; return { cents, currency };
@@ -77,7 +83,7 @@ function parseEbayListings(
htmlString: HTMLString, htmlString: HTMLString,
keywords: string[], keywords: string[],
exclusions: string[], exclusions: string[],
strictMode: boolean strictMode: boolean,
): EbayListingDetails[] { ): EbayListingDetails[] {
const { document } = parseHTML(htmlString); const { document } = parseHTML(htmlString);
const results: EbayListingDetails[] = []; const results: EbayListingDetails[] = [];
@@ -85,16 +91,17 @@ function parseEbayListings(
// Find all listing links by looking for eBay item URLs (/itm/) // Find all listing links by looking for eBay item URLs (/itm/)
const linkElements = document.querySelectorAll('a[href*="itm/"]'); const linkElements = document.querySelectorAll('a[href*="itm/"]');
for (const linkElement of linkElements) { for (const linkElement of linkElements) {
try { try {
// Get href attribute // Get href attribute
let href = linkElement.getAttribute('href'); let href = linkElement.getAttribute("href");
if (!href) continue; if (!href) continue;
// Make href absolute // Make href absolute
if (!href.startsWith('http')) { if (!href.startsWith("http")) {
href = href.startsWith('//') ? `https:${href}` : `https://www.ebay.com${href}`; href = href.startsWith("//")
? `https:${href}`
: `https://www.ebay.com${href}`;
} }
// Find the container - go up several levels to find the item container // Find the container - go up several levels to find the item container
@@ -108,15 +115,23 @@ function parseEbayListings(
// Extract title - look for heading or title-related elements near the link // Extract title - look for heading or title-related elements near the link
// Modern eBay often uses h3, span, or div with text content near the link // Modern eBay often uses h3, span, or div with text content near the link
let titleElement = container.querySelector('h3, [role="heading"], .s-item__title span'); let titleElement = container.querySelector(
'h3, [role="heading"], .s-item__title span',
);
// If no direct title element, try finding text content around the link // If no direct title element, try finding text content around the link
if (!titleElement) { if (!titleElement) {
// Look for spans or divs with text near this link // Look for spans or divs with text near this link
const nearbySpans = container.querySelectorAll('span, div'); const nearbySpans = container.querySelectorAll("span, div");
for (const span of nearbySpans) { for (const span of nearbySpans) {
const text = span.textContent?.trim(); const text = span.textContent?.trim();
if (text && text.length > 10 && text.length < 200 && !text.includes('$') && !text.includes('item')) { if (
text &&
text.length > 10 &&
text.length < 200 &&
!text.includes("$") &&
!text.includes("item")
) {
titleElement = span; titleElement = span;
break; break;
} }
@@ -129,12 +144,12 @@ function parseEbayListings(
if (title) { if (title) {
// Remove common eBay UI strings that appear at the end of titles // Remove common eBay UI strings that appear at the end of titles
const uiStrings = [ const uiStrings = [
'Opens in a new window', "Opens in a new window",
'Opens in a new tab', "Opens in a new tab",
'Opens in a new window or tab', "Opens in a new window or tab",
'opens in a new window', "opens in a new window",
'opens in a new tab', "opens in a new tab",
'opens in a new window or tab' "opens in a new window or tab",
]; ];
for (const uiString of uiStrings) { for (const uiString of uiStrings) {
@@ -157,18 +172,29 @@ function parseEbayListings(
if (title === "Shop on eBay" || title.length < 3) continue; if (title === "Shop on eBay" || title.length < 3) continue;
// Extract price - look for eBay's price classes, preferring sale/discount prices // Extract price - look for eBay's price classes, preferring sale/discount prices
let priceElement = container.querySelector('[class*="s-item__price"], .s-item__price, [class*="price"]'); let priceElement = container.querySelector(
'[class*="s-item__price"], .s-item__price, [class*="price"]',
);
// If no direct price class, look for spans containing $ (but not titles) // If no direct price class, look for spans containing $ (but not titles)
if (!priceElement) { if (!priceElement) {
const spansAndElements = container.querySelectorAll('span, div, b, em, strong'); const spansAndElements = container.querySelectorAll(
"span, div, b, em, strong",
);
for (const el of spansAndElements) { for (const el of spansAndElements) {
const text = el.textContent?.trim(); const text = el.textContent?.trim();
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words // Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
if (text && text.includes('$') && text.length < 100 && if (
!text.includes('laptop') && !text.includes('computer') && !text.includes('intel') && text &&
!text.includes('core') && !text.includes('ram') && !text.includes('ssd') && text.includes("$") &&
! /\d{4}/.test(text) && // Avoid years like "2024" text.length < 100 &&
!text.includes("laptop") &&
!text.includes("computer") &&
!text.includes("intel") &&
!text.includes("core") &&
!text.includes("ram") &&
!text.includes("ssd") &&
!/\d{4}/.test(text) && // Avoid years like "2024"
!text.includes('"') // Avoid measurements !text.includes('"') // Avoid measurements
) { ) {
priceElement = el; priceElement = el;
@@ -181,17 +207,26 @@ function parseEbayListings(
// Prefer sale/current price over original/strikethrough price // Prefer sale/current price over original/strikethrough price
if (priceElement) { if (priceElement) {
// Check if this element or its parent contains multiple price elements // Check if this element or its parent contains multiple price elements
const priceContainer = priceElement.closest('[class*="s-item__price"]') || priceElement.parentElement; const priceContainer =
priceElement.closest('[class*="s-item__price"]') ||
priceElement.parentElement;
if (priceContainer) { if (priceContainer) {
// Look for all price elements within this container, including strikethrough prices // Look for all price elements within this container, including strikethrough prices
const allPriceElements = priceContainer.querySelectorAll('[class*="s-item__price"], span, b, em, strong, s, del, strike'); const allPriceElements = priceContainer.querySelectorAll(
'[class*="s-item__price"], span, b, em, strong, s, del, strike',
);
// Filter to only elements that actually contain prices (not labels) // Filter to only elements that actually contain prices (not labels)
const actualPrices: HTMLElement[] = []; const actualPrices: HTMLElement[] = [];
for (const el of allPriceElements) { for (const el of allPriceElements) {
const text = el.textContent?.trim(); const text = el.textContent?.trim();
if (text && /^\s*[$£€¥]/u.test(text) && text.length < 50 && !/\d{4}/.test(text)) { if (
text &&
/^\s*[$£€¥]/u.test(text) &&
text.length < 50 &&
!/\d{4}/.test(text)
) {
actualPrices.push(el); actualPrices.push(el);
} }
} }
@@ -199,11 +234,18 @@ function parseEbayListings(
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices) // Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
if (actualPrices.length > 1) { if (actualPrices.length > 1) {
// First, look for prices that are NOT struck through // First, look for prices that are NOT struck through
const nonStrikethroughPrices = actualPrices.filter(el => { const nonStrikethroughPrices = actualPrices.filter((el) => {
const tagName = el.tagName.toLowerCase(); const tagName = el.tagName.toLowerCase();
const styles = el.classList.contains('s-strikethrough') || el.classList.contains('u-flStrike') || const styles =
el.closest('s, del, strike'); el.classList.contains("s-strikethrough") ||
return tagName !== 's' && tagName !== 'del' && tagName !== 'strike' && !styles; el.classList.contains("u-flStrike") ||
el.closest("s, del, strike");
return (
tagName !== "s" &&
tagName !== "del" &&
tagName !== "strike" &&
!styles
);
}); });
if (nonStrikethroughPrices.length > 0) { if (nonStrikethroughPrices.length > 0) {
@@ -227,12 +269,22 @@ function parseEbayListings(
if (!priceInfo) continue; if (!priceInfo) continue;
// Apply exclusion filters // Apply exclusion filters
if (exclusions.some(exclusion => title.toLowerCase().includes(exclusion.toLowerCase()))) { if (
exclusions.some((exclusion) =>
title.toLowerCase().includes(exclusion.toLowerCase()),
)
) {
continue; continue;
} }
// Apply strict mode filter (title must contain at least one keyword) // Apply strict mode filter (title must contain at least one keyword)
if (strictMode && !keywords.some(keyword => title!.toLowerCase().includes(keyword.toLowerCase()))) { if (
strictMode &&
title &&
!keywords.some((keyword) =>
title.toLowerCase().includes(keyword.toLowerCase()),
)
) {
continue; continue;
} }
@@ -307,18 +359,19 @@ export default async function fetchEbayItems(
try { try {
// Use custom headers modeled after real browser requests to bypass bot detection // Use custom headers modeled after real browser requests to bypass bot detection
const headers: Record<string, string> = { const headers: Record<string, string> = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0', "User-Agent":
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', "Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
'Accept-Language': 'en-US,en;q=0.5', Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
'Accept-Encoding': 'gzip, deflate, br', "Accept-Language": "en-US,en;q=0.5",
'Referer': 'https://www.ebay.ca/', "Accept-Encoding": "gzip, deflate, br",
'Connection': 'keep-alive', Referer: "https://www.ebay.ca/",
'Upgrade-Insecure-Requests': '1', Connection: "keep-alive",
'Sec-Fetch-Dest': 'document', "Upgrade-Insecure-Requests": "1",
'Sec-Fetch-Mode': 'navigate', "Sec-Fetch-Dest": "document",
'Sec-Fetch-Site': 'same-origin', "Sec-Fetch-Mode": "navigate",
'Sec-Fetch-User': '?1', "Sec-Fetch-Site": "same-origin",
'Priority': 'u=0, i' "Sec-Fetch-User": "?1",
Priority: "u=0, i",
}; };
const res = await fetch(searchUrl, { const res = await fetch(searchUrl, {
@@ -340,17 +393,21 @@ export default async function fetchEbayItems(
console.log(`\nParsing eBay listings...`); console.log(`\nParsing eBay listings...`);
const listings = parseEbayListings(searchHtml, keywords, exclusions, strictMode); const listings = parseEbayListings(
searchHtml,
keywords,
exclusions,
strictMode,
);
// Filter by price range (additional safety check) // Filter by price range (additional safety check)
const filteredListings = listings.filter(listing => { const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents; const cents = listing.listingPrice?.cents;
return cents && cents >= minPrice && cents <= maxPrice; return cents && cents >= minPrice && cents <= maxPrice;
}); });
console.log(`Parsed ${filteredListings.length} eBay listings.`); console.log(`Parsed ${filteredListings.length} eBay listings.`);
return filteredListings; return filteredListings;
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
console.error( console.error(