old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: format ebay scraper with consistent indentation

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-01-23 09:52:25 -05:00
parent 6e0487f8f3
commit 7b106c91ce
1 changed files with 338 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
packages/core/src/scrapers/ebay.ts
View File

@@ -1,9 +1,9 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom";
import { isRecord } from "../utils/http";
import type { HTMLString } from "../types/common";
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format";
import type { HTMLString } from "../types/common";
import { isRecord } from "../utils/http";
// ----------------------------- Types -----------------------------
@@ -29,8 +29,10 @@ export interface EbayListingDetails {
/**
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
*/
function parseEbayPrice(priceText: string): { cents: number; currency: string } | null {
if (!priceText || typeof priceText !== 'string') return null;
function parseEbayPrice(
priceText: string,
): { cents: number; currency: string } | null {
if (!priceText || typeof priceText !== "string") return null;
// Clean up the price text and extract currency and amount
const cleaned = priceText.trim();
@@ -39,19 +41,23 @@ function parseEbayPrice(priceText: string): { cents: number; currency: string }
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
if (!numberMatches) return null;
const amountStr = numberMatches[0].replace(/,/g, '');
const amountStr = numberMatches[0].replace(/,/g, "");
const dollars = parseFloat(amountStr);
if (isNaN(dollars)) return null;
const cents = Math.round(dollars * 100);
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
let currency = 'USD'; // Default
let currency = "USD"; // Default
if (cleaned.toUpperCase().includes('CAD') || cleaned.includes('CA$') || cleaned.includes('C $')) {
currency = 'CAD';
} else if (cleaned.toUpperCase().includes('USD') || cleaned.includes('$')) {
currency = 'USD';
if (
cleaned.toUpperCase().includes("CAD") ||
cleaned.includes("CA$") ||
cleaned.includes("C $")
) {
currency = "CAD";
} else if (cleaned.toUpperCase().includes("USD") || cleaned.includes("$")) {
currency = "USD";
}
return { cents, currency };
@@ -77,7 +83,7 @@ function parseEbayListings(
htmlString: HTMLString,
keywords: string[],
exclusions: string[],
strictMode: boolean
strictMode: boolean,
): EbayListingDetails[] {
const { document } = parseHTML(htmlString);
const results: EbayListingDetails[] = [];
@@ -85,16 +91,17 @@ function parseEbayListings(
// Find all listing links by looking for eBay item URLs (/itm/)
const linkElements = document.querySelectorAll('a[href*="itm/"]');
for (const linkElement of linkElements) {
try {
// Get href attribute
let href = linkElement.getAttribute('href');
let href = linkElement.getAttribute("href");
if (!href) continue;
// Make href absolute
if (!href.startsWith('http')) {
href = href.startsWith('//') ? `https:${href}` : `https://www.ebay.com${href}`;
if (!href.startsWith("http")) {
href = href.startsWith("//")
? `https:${href}`
: `https://www.ebay.com${href}`;
}
// Find the container - go up several levels to find the item container
@@ -108,15 +115,23 @@ function parseEbayListings(
// Extract title - look for heading or title-related elements near the link
// Modern eBay often uses h3, span, or div with text content near the link
let titleElement = container.querySelector('h3, [role="heading"], .s-item__title span');
let titleElement = container.querySelector(
'h3, [role="heading"], .s-item__title span',
);
// If no direct title element, try finding text content around the link
if (!titleElement) {
// Look for spans or divs with text near this link
const nearbySpans = container.querySelectorAll('span, div');
const nearbySpans = container.querySelectorAll("span, div");
for (const span of nearbySpans) {
const text = span.textContent?.trim();
if (text && text.length > 10 && text.length < 200 && !text.includes('$') && !text.includes('item')) {
if (
text &&
text.length > 10 &&
text.length < 200 &&
!text.includes("$") &&
!text.includes("item")
) {
titleElement = span;
break;
}
@@ -129,12 +144,12 @@ function parseEbayListings(
if (title) {
// Remove common eBay UI strings that appear at the end of titles
const uiStrings = [
'Opens in a new window',
'Opens in a new tab',
'Opens in a new window or tab',
'opens in a new window',
'opens in a new tab',
'opens in a new window or tab'
"Opens in a new window",
"Opens in a new tab",
"Opens in a new window or tab",
"opens in a new window",
"opens in a new tab",
"opens in a new window or tab",
];
for (const uiString of uiStrings) {
@@ -157,18 +172,29 @@ function parseEbayListings(
if (title === "Shop on eBay" || title.length < 3) continue;
// Extract price - look for eBay's price classes, preferring sale/discount prices
let priceElement = container.querySelector('[class*="s-item__price"], .s-item__price, [class*="price"]');
let priceElement = container.querySelector(
'[class*="s-item__price"], .s-item__price, [class*="price"]',
);
// If no direct price class, look for spans containing $ (but not titles)
if (!priceElement) {
const spansAndElements = container.querySelectorAll('span, div, b, em, strong');
const spansAndElements = container.querySelectorAll(
"span, div, b, em, strong",
);
for (const el of spansAndElements) {
const text = el.textContent?.trim();
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
if (text && text.includes('$') && text.length < 100 &&
!text.includes('laptop') && !text.includes('computer') && !text.includes('intel') &&
!text.includes('core') && !text.includes('ram') && !text.includes('ssd') &&
! /\d{4}/.test(text) && // Avoid years like "2024"
if (
text &&
text.includes("$") &&
text.length < 100 &&
!text.includes("laptop") &&
!text.includes("computer") &&
!text.includes("intel") &&
!text.includes("core") &&
!text.includes("ram") &&
!text.includes("ssd") &&
!/\d{4}/.test(text) && // Avoid years like "2024"
!text.includes('"') // Avoid measurements
) {
priceElement = el;
@@ -181,17 +207,26 @@ function parseEbayListings(
// Prefer sale/current price over original/strikethrough price
if (priceElement) {
// Check if this element or its parent contains multiple price elements
const priceContainer = priceElement.closest('[class*="s-item__price"]') || priceElement.parentElement;
const priceContainer =
priceElement.closest('[class*="s-item__price"]') ||
priceElement.parentElement;
if (priceContainer) {
// Look for all price elements within this container, including strikethrough prices
const allPriceElements = priceContainer.querySelectorAll('[class*="s-item__price"], span, b, em, strong, s, del, strike');
const allPriceElements = priceContainer.querySelectorAll(
'[class*="s-item__price"], span, b, em, strong, s, del, strike',
);
// Filter to only elements that actually contain prices (not labels)
const actualPrices: HTMLElement[] = [];
for (const el of allPriceElements) {
const text = el.textContent?.trim();
if (text && /^\s*[$£€¥]/u.test(text) && text.length < 50 && !/\d{4}/.test(text)) {
if (
text &&
/^\s*[$£€¥]/u.test(text) &&
text.length < 50 &&
!/\d{4}/.test(text)
) {
actualPrices.push(el);
}
}
@@ -199,11 +234,18 @@ function parseEbayListings(
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
if (actualPrices.length > 1) {
// First, look for prices that are NOT struck through
const nonStrikethroughPrices = actualPrices.filter(el => {
const nonStrikethroughPrices = actualPrices.filter((el) => {
const tagName = el.tagName.toLowerCase();
const styles = el.classList.contains('s-strikethrough') || el.classList.contains('u-flStrike') ||
el.closest('s, del, strike');
return tagName !== 's' && tagName !== 'del' && tagName !== 'strike' && !styles;
const styles =
el.classList.contains("s-strikethrough") ||
el.classList.contains("u-flStrike") ||
el.closest("s, del, strike");
return (
tagName !== "s" &&
tagName !== "del" &&
tagName !== "strike" &&
!styles
);
});
if (nonStrikethroughPrices.length > 0) {
@@ -227,12 +269,22 @@ function parseEbayListings(
if (!priceInfo) continue;
// Apply exclusion filters
if (exclusions.some(exclusion => title.toLowerCase().includes(exclusion.toLowerCase()))) {
if (
exclusions.some((exclusion) =>
title.toLowerCase().includes(exclusion.toLowerCase()),
)
) {
continue;
}
// Apply strict mode filter (title must contain at least one keyword)
if (strictMode && !keywords.some(keyword => title!.toLowerCase().includes(keyword.toLowerCase()))) {
if (
strictMode &&
title &&
!keywords.some((keyword) =>
title.toLowerCase().includes(keyword.toLowerCase()),
)
) {
continue;
}
@@ -307,18 +359,19 @@ export default async function fetchEbayItems(
try {
// Use custom headers modeled after real browser requests to bypass bot detection
const headers: Record<string, string> = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.ebay.ca/',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Priority': 'u=0, i'
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
Referer: "https://www.ebay.ca/",
Connection: "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-User": "?1",
Priority: "u=0, i",
};
const res = await fetch(searchUrl, {
@@ -340,17 +393,21 @@ export default async function fetchEbayItems(
console.log(`\nParsing eBay listings...`);
const listings = parseEbayListings(searchHtml, keywords, exclusions, strictMode);
const listings = parseEbayListings(
searchHtml,
keywords,
exclusions,
strictMode,
);
// Filter by price range (additional safety check)
const filteredListings = listings.filter(listing => {
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return cents && cents >= minPrice && cents <= maxPrice;
});
console.log(`Parsed ${filteredListings.length} eBay listings.`);
return filteredListings;
} catch (err) {
if (err instanceof HttpError) {
console.error(