Compare commits
7 Commits
da23ca1c3f
...
441ff436c4
| Author | SHA1 | Date | |
|---|---|---|---|
| 441ff436c4 | |||
| 1f53ec912a | |||
| 053efd815b | |||
| d619fa5d77 | |||
| 050fd0adba | |||
| 7b106c91ce | |||
| 6e0487f8f3 |
@@ -1,7 +1,7 @@
|
|||||||
import { statusRoute } from "./routes/status";
|
|
||||||
import { kijijiRoute } from "./routes/kijiji";
|
|
||||||
import { facebookRoute } from "./routes/facebook";
|
|
||||||
import { ebayRoute } from "./routes/ebay";
|
import { ebayRoute } from "./routes/ebay";
|
||||||
|
import { facebookRoute } from "./routes/facebook";
|
||||||
|
import { kijijiRoute } from "./routes/kijiji";
|
||||||
|
import { statusRoute } from "./routes/status";
|
||||||
|
|
||||||
const PORT = process.env.PORT || 4005;
|
const PORT = process.env.PORT || 4005;
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ const server = Bun.serve({
|
|||||||
},
|
},
|
||||||
|
|
||||||
// Fallback for all other routes
|
// Fallback for all other routes
|
||||||
fetch(req: Request) {
|
fetch(_req: Request) {
|
||||||
return new Response("Not Found", { status: 404 });
|
return new Response("Not Found", { status: 404 });
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { fetchEbayItems } from "@marketplace-scrapers/core";
|
|||||||
* Search eBay for listings (default: Buy It Now only, Canada only)
|
* Search eBay for listings (default: Buy It Now only, Canada only)
|
||||||
*/
|
*/
|
||||||
export async function ebayRoute(req: Request): Promise<Response> {
|
export async function ebayRoute(req: Request): Promise<Response> {
|
||||||
|
try {
|
||||||
const reqUrl = new URL(req.url);
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
const SEARCH_QUERY =
|
const SEARCH_QUERY =
|
||||||
@@ -18,23 +19,26 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
|
|
||||||
// Parse optional parameters with defaults
|
const minPriceParam = reqUrl.searchParams.get("minPrice");
|
||||||
const minPrice = reqUrl.searchParams.get("minPrice")
|
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
|
||||||
? parseInt(reqUrl.searchParams.get("minPrice")!)
|
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
|
||||||
: undefined;
|
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
|
||||||
const maxPrice = reqUrl.searchParams.get("maxPrice")
|
|
||||||
? parseInt(reqUrl.searchParams.get("maxPrice")!)
|
|
||||||
: undefined;
|
|
||||||
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
||||||
const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false";
|
const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false";
|
||||||
const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false";
|
const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false";
|
||||||
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
||||||
const exclusions = exclusionsParam ? exclusionsParam.split(",").map(s => s.trim()) : [];
|
const exclusions = exclusionsParam
|
||||||
|
? exclusionsParam.split(",").map((s) => s.trim())
|
||||||
|
: [];
|
||||||
const keywordsParam = reqUrl.searchParams.get("keywords");
|
const keywordsParam = reqUrl.searchParams.get("keywords");
|
||||||
const keywords = keywordsParam ? keywordsParam.split(",").map(s => s.trim()) : [SEARCH_QUERY];
|
const keywords = keywordsParam
|
||||||
|
? keywordsParam.split(",").map((s) => s.trim())
|
||||||
|
: [SEARCH_QUERY];
|
||||||
|
|
||||||
try {
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
const items = await fetchEbayItems(SEARCH_QUERY, 5, {
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
|
||||||
|
|
||||||
|
const items = await fetchEbayItems(SEARCH_QUERY, 1, {
|
||||||
minPrice,
|
minPrice,
|
||||||
maxPrice,
|
maxPrice,
|
||||||
strictMode,
|
strictMode,
|
||||||
@@ -43,18 +47,19 @@ export async function ebayRoute(req: Request): Promise<Response> {
|
|||||||
buyItNowOnly,
|
buyItNowOnly,
|
||||||
canadaOnly,
|
canadaOnly,
|
||||||
});
|
});
|
||||||
if (!items || items.length === 0)
|
|
||||||
|
const results = maxItems ? items.slice(0, maxItems) : items;
|
||||||
|
|
||||||
|
if (!results || results.length === 0)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
return Response.json(items, { status: 200 });
|
return Response.json(results, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("eBay scraping error:", error);
|
console.error("eBay scraping error:", error);
|
||||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
|
const errorMessage =
|
||||||
return Response.json(
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
{ message: errorMessage },
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,17 +12,25 @@ export async function facebookRoute(req: Request): Promise<Response> {
|
|||||||
if (!SEARCH_QUERY)
|
if (!SEARCH_QUERY)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{
|
{
|
||||||
message:
|
message: "Request didn't have 'query' header or 'q' search parameter!",
|
||||||
"Request didn't have 'query' header or 'q' search parameter!",
|
|
||||||
},
|
},
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
|
|
||||||
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
|
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
|
||||||
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
||||||
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : 25;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const items = await fetchFacebookItems(SEARCH_QUERY, 5, LOCATION, 25, COOKIES_SOURCE);
|
const items = await fetchFacebookItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
1,
|
||||||
|
LOCATION,
|
||||||
|
maxItems,
|
||||||
|
COOKIES_SOURCE,
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
if (!items || items.length === 0)
|
if (!items || items.length === 0)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
@@ -31,10 +39,8 @@ export async function facebookRoute(req: Request): Promise<Response> {
|
|||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Facebook scraping error:", error);
|
console.error("Facebook scraping error:", error);
|
||||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
|
const errorMessage =
|
||||||
return Response.json(
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
{ message: errorMessage },
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,14 +12,45 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
|||||||
if (!SEARCH_QUERY)
|
if (!SEARCH_QUERY)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{
|
{
|
||||||
message:
|
message: "Request didn't have 'query' header or 'q' search parameter!",
|
||||||
"Request didn't have 'query' header or 'q' search parameter!",
|
|
||||||
},
|
},
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const maxPagesParam = reqUrl.searchParams.get("maxPages");
|
||||||
|
const maxPages = maxPagesParam ? parseInt(maxPagesParam, 10) : 5;
|
||||||
|
const priceMinParam = reqUrl.searchParams.get("priceMin");
|
||||||
|
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
|
||||||
|
const priceMaxParam = reqUrl.searchParams.get("priceMax");
|
||||||
|
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
|
||||||
|
|
||||||
|
const searchOptions = {
|
||||||
|
location: reqUrl.searchParams.get("location") || undefined,
|
||||||
|
category: reqUrl.searchParams.get("category") || undefined,
|
||||||
|
keywords: reqUrl.searchParams.get("keywords") || undefined,
|
||||||
|
sortBy: reqUrl.searchParams.get("sortBy") as
|
||||||
|
| "relevancy"
|
||||||
|
| "date"
|
||||||
|
| "price"
|
||||||
|
| "distance"
|
||||||
|
| undefined,
|
||||||
|
sortOrder: reqUrl.searchParams.get("sortOrder") as
|
||||||
|
| "desc"
|
||||||
|
| "asc"
|
||||||
|
| undefined,
|
||||||
|
maxPages,
|
||||||
|
priceMin,
|
||||||
|
priceMax,
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const items = await fetchKijijiItems(SEARCH_QUERY, 5);
|
const items = await fetchKijijiItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
1,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
searchOptions,
|
||||||
|
{},
|
||||||
|
);
|
||||||
if (!items)
|
if (!items)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
@@ -28,10 +59,8 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
|||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Kijiji scraping error:", error);
|
console.error("Kijiji scraping error:", error);
|
||||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
|
const errorMessage =
|
||||||
return Response.json(
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
{ message: errorMessage },
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
import { isRecord } from "../utils/http";
|
import type { HTMLString } from "../types/common";
|
||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
import { formatCentsToCurrency } from "../utils/format";
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
import type { HTMLString } from "../types/common";
|
import { isRecord } from "../utils/http";
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
@@ -29,8 +29,10 @@ export interface EbayListingDetails {
|
|||||||
/**
|
/**
|
||||||
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
||||||
*/
|
*/
|
||||||
function parseEbayPrice(priceText: string): { cents: number; currency: string } | null {
|
function parseEbayPrice(
|
||||||
if (!priceText || typeof priceText !== 'string') return null;
|
priceText: string,
|
||||||
|
): { cents: number; currency: string } | null {
|
||||||
|
if (!priceText || typeof priceText !== "string") return null;
|
||||||
|
|
||||||
// Clean up the price text and extract currency and amount
|
// Clean up the price text and extract currency and amount
|
||||||
const cleaned = priceText.trim();
|
const cleaned = priceText.trim();
|
||||||
@@ -39,19 +41,23 @@ function parseEbayPrice(priceText: string): { cents: number; currency: string }
|
|||||||
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
|
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
|
||||||
if (!numberMatches) return null;
|
if (!numberMatches) return null;
|
||||||
|
|
||||||
const amountStr = numberMatches[0].replace(/,/g, '');
|
const amountStr = numberMatches[0].replace(/,/g, "");
|
||||||
const dollars = parseFloat(amountStr);
|
const dollars = parseFloat(amountStr);
|
||||||
if (isNaN(dollars)) return null;
|
if (isNaN(dollars)) return null;
|
||||||
|
|
||||||
const cents = Math.round(dollars * 100);
|
const cents = Math.round(dollars * 100);
|
||||||
|
|
||||||
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
|
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
|
||||||
let currency = 'USD'; // Default
|
let currency = "USD"; // Default
|
||||||
|
|
||||||
if (cleaned.toUpperCase().includes('CAD') || cleaned.includes('CA$') || cleaned.includes('C $')) {
|
if (
|
||||||
currency = 'CAD';
|
cleaned.toUpperCase().includes("CAD") ||
|
||||||
} else if (cleaned.toUpperCase().includes('USD') || cleaned.includes('$')) {
|
cleaned.includes("CA$") ||
|
||||||
currency = 'USD';
|
cleaned.includes("C $")
|
||||||
|
) {
|
||||||
|
currency = "CAD";
|
||||||
|
} else if (cleaned.toUpperCase().includes("USD") || cleaned.includes("$")) {
|
||||||
|
currency = "USD";
|
||||||
}
|
}
|
||||||
|
|
||||||
return { cents, currency };
|
return { cents, currency };
|
||||||
@@ -77,7 +83,7 @@ function parseEbayListings(
|
|||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
keywords: string[],
|
keywords: string[],
|
||||||
exclusions: string[],
|
exclusions: string[],
|
||||||
strictMode: boolean
|
strictMode: boolean,
|
||||||
): EbayListingDetails[] {
|
): EbayListingDetails[] {
|
||||||
const { document } = parseHTML(htmlString);
|
const { document } = parseHTML(htmlString);
|
||||||
const results: EbayListingDetails[] = [];
|
const results: EbayListingDetails[] = [];
|
||||||
@@ -85,16 +91,17 @@ function parseEbayListings(
|
|||||||
// Find all listing links by looking for eBay item URLs (/itm/)
|
// Find all listing links by looking for eBay item URLs (/itm/)
|
||||||
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
||||||
|
|
||||||
|
|
||||||
for (const linkElement of linkElements) {
|
for (const linkElement of linkElements) {
|
||||||
try {
|
try {
|
||||||
// Get href attribute
|
// Get href attribute
|
||||||
let href = linkElement.getAttribute('href');
|
let href = linkElement.getAttribute("href");
|
||||||
if (!href) continue;
|
if (!href) continue;
|
||||||
|
|
||||||
// Make href absolute
|
// Make href absolute
|
||||||
if (!href.startsWith('http')) {
|
if (!href.startsWith("http")) {
|
||||||
href = href.startsWith('//') ? `https:${href}` : `https://www.ebay.com${href}`;
|
href = href.startsWith("//")
|
||||||
|
? `https:${href}`
|
||||||
|
: `https://www.ebay.com${href}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find the container - go up several levels to find the item container
|
// Find the container - go up several levels to find the item container
|
||||||
@@ -108,15 +115,23 @@ function parseEbayListings(
|
|||||||
|
|
||||||
// Extract title - look for heading or title-related elements near the link
|
// Extract title - look for heading or title-related elements near the link
|
||||||
// Modern eBay often uses h3, span, or div with text content near the link
|
// Modern eBay often uses h3, span, or div with text content near the link
|
||||||
let titleElement = container.querySelector('h3, [role="heading"], .s-item__title span');
|
let titleElement = container.querySelector(
|
||||||
|
'h3, [role="heading"], .s-item__title span',
|
||||||
|
);
|
||||||
|
|
||||||
// If no direct title element, try finding text content around the link
|
// If no direct title element, try finding text content around the link
|
||||||
if (!titleElement) {
|
if (!titleElement) {
|
||||||
// Look for spans or divs with text near this link
|
// Look for spans or divs with text near this link
|
||||||
const nearbySpans = container.querySelectorAll('span, div');
|
const nearbySpans = container.querySelectorAll("span, div");
|
||||||
for (const span of nearbySpans) {
|
for (const span of nearbySpans) {
|
||||||
const text = span.textContent?.trim();
|
const text = span.textContent?.trim();
|
||||||
if (text && text.length > 10 && text.length < 200 && !text.includes('$') && !text.includes('item')) {
|
if (
|
||||||
|
text &&
|
||||||
|
text.length > 10 &&
|
||||||
|
text.length < 200 &&
|
||||||
|
!text.includes("$") &&
|
||||||
|
!text.includes("item")
|
||||||
|
) {
|
||||||
titleElement = span;
|
titleElement = span;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -129,12 +144,12 @@ function parseEbayListings(
|
|||||||
if (title) {
|
if (title) {
|
||||||
// Remove common eBay UI strings that appear at the end of titles
|
// Remove common eBay UI strings that appear at the end of titles
|
||||||
const uiStrings = [
|
const uiStrings = [
|
||||||
'Opens in a new window',
|
"Opens in a new window",
|
||||||
'Opens in a new tab',
|
"Opens in a new tab",
|
||||||
'Opens in a new window or tab',
|
"Opens in a new window or tab",
|
||||||
'opens in a new window',
|
"opens in a new window",
|
||||||
'opens in a new tab',
|
"opens in a new tab",
|
||||||
'opens in a new window or tab'
|
"opens in a new window or tab",
|
||||||
];
|
];
|
||||||
|
|
||||||
for (const uiString of uiStrings) {
|
for (const uiString of uiStrings) {
|
||||||
@@ -157,17 +172,28 @@ function parseEbayListings(
|
|||||||
if (title === "Shop on eBay" || title.length < 3) continue;
|
if (title === "Shop on eBay" || title.length < 3) continue;
|
||||||
|
|
||||||
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
||||||
let priceElement = container.querySelector('[class*="s-item__price"], .s-item__price, [class*="price"]');
|
let priceElement = container.querySelector(
|
||||||
|
'[class*="s-item__price"], .s-item__price, [class*="price"]',
|
||||||
|
);
|
||||||
|
|
||||||
// If no direct price class, look for spans containing $ (but not titles)
|
// If no direct price class, look for spans containing $ (but not titles)
|
||||||
if (!priceElement) {
|
if (!priceElement) {
|
||||||
const spansAndElements = container.querySelectorAll('span, div, b, em, strong');
|
const spansAndElements = container.querySelectorAll(
|
||||||
|
"span, div, b, em, strong",
|
||||||
|
);
|
||||||
for (const el of spansAndElements) {
|
for (const el of spansAndElements) {
|
||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
|
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
|
||||||
if (text && text.includes('$') && text.length < 100 &&
|
if (
|
||||||
!text.includes('laptop') && !text.includes('computer') && !text.includes('intel') &&
|
text &&
|
||||||
!text.includes('core') && !text.includes('ram') && !text.includes('ssd') &&
|
text.includes("$") &&
|
||||||
|
text.length < 100 &&
|
||||||
|
!text.includes("laptop") &&
|
||||||
|
!text.includes("computer") &&
|
||||||
|
!text.includes("intel") &&
|
||||||
|
!text.includes("core") &&
|
||||||
|
!text.includes("ram") &&
|
||||||
|
!text.includes("ssd") &&
|
||||||
!/\d{4}/.test(text) && // Avoid years like "2024"
|
!/\d{4}/.test(text) && // Avoid years like "2024"
|
||||||
!text.includes('"') // Avoid measurements
|
!text.includes('"') // Avoid measurements
|
||||||
) {
|
) {
|
||||||
@@ -181,17 +207,26 @@ function parseEbayListings(
|
|||||||
// Prefer sale/current price over original/strikethrough price
|
// Prefer sale/current price over original/strikethrough price
|
||||||
if (priceElement) {
|
if (priceElement) {
|
||||||
// Check if this element or its parent contains multiple price elements
|
// Check if this element or its parent contains multiple price elements
|
||||||
const priceContainer = priceElement.closest('[class*="s-item__price"]') || priceElement.parentElement;
|
const priceContainer =
|
||||||
|
priceElement.closest('[class*="s-item__price"]') ||
|
||||||
|
priceElement.parentElement;
|
||||||
|
|
||||||
if (priceContainer) {
|
if (priceContainer) {
|
||||||
// Look for all price elements within this container, including strikethrough prices
|
// Look for all price elements within this container, including strikethrough prices
|
||||||
const allPriceElements = priceContainer.querySelectorAll('[class*="s-item__price"], span, b, em, strong, s, del, strike');
|
const allPriceElements = priceContainer.querySelectorAll(
|
||||||
|
'[class*="s-item__price"], span, b, em, strong, s, del, strike',
|
||||||
|
);
|
||||||
|
|
||||||
// Filter to only elements that actually contain prices (not labels)
|
// Filter to only elements that actually contain prices (not labels)
|
||||||
const actualPrices: HTMLElement[] = [];
|
const actualPrices: HTMLElement[] = [];
|
||||||
for (const el of allPriceElements) {
|
for (const el of allPriceElements) {
|
||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
if (text && /^\s*[$£€¥]/u.test(text) && text.length < 50 && !/\d{4}/.test(text)) {
|
if (
|
||||||
|
text &&
|
||||||
|
/^\s*[$£€¥]/u.test(text) &&
|
||||||
|
text.length < 50 &&
|
||||||
|
!/\d{4}/.test(text)
|
||||||
|
) {
|
||||||
actualPrices.push(el);
|
actualPrices.push(el);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -199,11 +234,18 @@ function parseEbayListings(
|
|||||||
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
|
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
|
||||||
if (actualPrices.length > 1) {
|
if (actualPrices.length > 1) {
|
||||||
// First, look for prices that are NOT struck through
|
// First, look for prices that are NOT struck through
|
||||||
const nonStrikethroughPrices = actualPrices.filter(el => {
|
const nonStrikethroughPrices = actualPrices.filter((el) => {
|
||||||
const tagName = el.tagName.toLowerCase();
|
const tagName = el.tagName.toLowerCase();
|
||||||
const styles = el.classList.contains('s-strikethrough') || el.classList.contains('u-flStrike') ||
|
const styles =
|
||||||
el.closest('s, del, strike');
|
el.classList.contains("s-strikethrough") ||
|
||||||
return tagName !== 's' && tagName !== 'del' && tagName !== 'strike' && !styles;
|
el.classList.contains("u-flStrike") ||
|
||||||
|
el.closest("s, del, strike");
|
||||||
|
return (
|
||||||
|
tagName !== "s" &&
|
||||||
|
tagName !== "del" &&
|
||||||
|
tagName !== "strike" &&
|
||||||
|
!styles
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
if (nonStrikethroughPrices.length > 0) {
|
if (nonStrikethroughPrices.length > 0) {
|
||||||
@@ -227,12 +269,22 @@ function parseEbayListings(
|
|||||||
if (!priceInfo) continue;
|
if (!priceInfo) continue;
|
||||||
|
|
||||||
// Apply exclusion filters
|
// Apply exclusion filters
|
||||||
if (exclusions.some(exclusion => title.toLowerCase().includes(exclusion.toLowerCase()))) {
|
if (
|
||||||
|
exclusions.some((exclusion) =>
|
||||||
|
title.toLowerCase().includes(exclusion.toLowerCase()),
|
||||||
|
)
|
||||||
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply strict mode filter (title must contain at least one keyword)
|
// Apply strict mode filter (title must contain at least one keyword)
|
||||||
if (strictMode && !keywords.some(keyword => title!.toLowerCase().includes(keyword.toLowerCase()))) {
|
if (
|
||||||
|
strictMode &&
|
||||||
|
title &&
|
||||||
|
!keywords.some((keyword) =>
|
||||||
|
title.toLowerCase().includes(keyword.toLowerCase()),
|
||||||
|
)
|
||||||
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -307,18 +359,19 @@ export default async function fetchEbayItems(
|
|||||||
try {
|
try {
|
||||||
// Use custom headers modeled after real browser requests to bypass bot detection
|
// Use custom headers modeled after real browser requests to bypass bot detection
|
||||||
const headers: Record<string, string> = {
|
const headers: Record<string, string> = {
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0',
|
"User-Agent":
|
||||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
|
||||||
'Accept-Language': 'en-US,en;q=0.5',
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
'Accept-Encoding': 'gzip, deflate, br',
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
'Referer': 'https://www.ebay.ca/',
|
"Accept-Encoding": "gzip, deflate, br",
|
||||||
'Connection': 'keep-alive',
|
Referer: "https://www.ebay.ca/",
|
||||||
'Upgrade-Insecure-Requests': '1',
|
Connection: "keep-alive",
|
||||||
'Sec-Fetch-Dest': 'document',
|
"Upgrade-Insecure-Requests": "1",
|
||||||
'Sec-Fetch-Mode': 'navigate',
|
"Sec-Fetch-Dest": "document",
|
||||||
'Sec-Fetch-Site': 'same-origin',
|
"Sec-Fetch-Mode": "navigate",
|
||||||
'Sec-Fetch-User': '?1',
|
"Sec-Fetch-Site": "same-origin",
|
||||||
'Priority': 'u=0, i'
|
"Sec-Fetch-User": "?1",
|
||||||
|
Priority: "u=0, i",
|
||||||
};
|
};
|
||||||
|
|
||||||
const res = await fetch(searchUrl, {
|
const res = await fetch(searchUrl, {
|
||||||
@@ -340,17 +393,21 @@ export default async function fetchEbayItems(
|
|||||||
|
|
||||||
console.log(`\nParsing eBay listings...`);
|
console.log(`\nParsing eBay listings...`);
|
||||||
|
|
||||||
const listings = parseEbayListings(searchHtml, keywords, exclusions, strictMode);
|
const listings = parseEbayListings(
|
||||||
|
searchHtml,
|
||||||
|
keywords,
|
||||||
|
exclusions,
|
||||||
|
strictMode,
|
||||||
|
);
|
||||||
|
|
||||||
// Filter by price range (additional safety check)
|
// Filter by price range (additional safety check)
|
||||||
const filteredListings = listings.filter(listing => {
|
const filteredListings = listings.filter((listing) => {
|
||||||
const cents = listing.listingPrice?.cents;
|
const cents = listing.listingPrice?.cents;
|
||||||
return cents && cents >= minPrice && cents <= maxPrice;
|
return cents && cents >= minPrice && cents <= maxPrice;
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||||
return filteredListings;
|
return filteredListings;
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
console.error(
|
console.error(
|
||||||
|
|||||||
@@ -97,7 +97,23 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
error: { code: -32602, message: "query parameter is required" },
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const items = await fetchKijijiItems(query, args.maxItems || 5);
|
const searchOptions = {
|
||||||
|
location: args.location,
|
||||||
|
category: args.category,
|
||||||
|
keywords: args.keywords,
|
||||||
|
sortBy: args.sortBy,
|
||||||
|
sortOrder: args.sortOrder,
|
||||||
|
maxPages: args.maxPages || 5,
|
||||||
|
priceMin: args.priceMin,
|
||||||
|
priceMax: args.priceMax,
|
||||||
|
};
|
||||||
|
const items = await fetchKijijiItems(
|
||||||
|
query,
|
||||||
|
1,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
searchOptions,
|
||||||
|
{}
|
||||||
|
);
|
||||||
result = items || [];
|
result = items || [];
|
||||||
} else if (name === "search_facebook") {
|
} else if (name === "search_facebook") {
|
||||||
const query = args.query;
|
const query = args.query;
|
||||||
@@ -110,10 +126,11 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
}
|
}
|
||||||
const items = await fetchFacebookItems(
|
const items = await fetchFacebookItems(
|
||||||
query,
|
query,
|
||||||
args.maxItems || 5,
|
1,
|
||||||
args.location || "toronto",
|
args.location || "toronto",
|
||||||
25,
|
args.maxItems || 25,
|
||||||
args.cookiesSource
|
args.cookiesSource,
|
||||||
|
undefined
|
||||||
);
|
);
|
||||||
result = items || [];
|
result = items || [];
|
||||||
} else if (name === "search_ebay") {
|
} else if (name === "search_ebay") {
|
||||||
@@ -125,7 +142,7 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
error: { code: -32602, message: "query parameter is required" },
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
const items = await fetchEbayItems(query, args.maxItems || 5, {
|
const items = await fetchEbayItems(query, 1, {
|
||||||
minPrice: args.minPrice,
|
minPrice: args.minPrice,
|
||||||
maxPrice: args.maxPrice,
|
maxPrice: args.maxPrice,
|
||||||
strictMode: args.strictMode || false,
|
strictMode: args.strictMode || false,
|
||||||
@@ -134,7 +151,9 @@ export async function handleMcpRequest(req: Request): Promise<Response> {
|
|||||||
buyItNowOnly: args.buyItNowOnly !== false,
|
buyItNowOnly: args.buyItNowOnly !== false,
|
||||||
canadaOnly: args.canadaOnly !== false,
|
canadaOnly: args.canadaOnly !== false,
|
||||||
});
|
});
|
||||||
result = items || [];
|
|
||||||
|
const results = args.maxItems ? items.slice(0, args.maxItems) : items;
|
||||||
|
result = results || [];
|
||||||
} else {
|
} else {
|
||||||
return Response.json({
|
return Response.json({
|
||||||
jsonrpc: "2.0",
|
jsonrpc: "2.0",
|
||||||
|
|||||||
@@ -13,11 +13,43 @@ export const tools = [
|
|||||||
type: "string",
|
type: "string",
|
||||||
description: "Search query for Kijiji listings",
|
description: "Search query for Kijiji listings",
|
||||||
},
|
},
|
||||||
maxItems: {
|
location: {
|
||||||
|
type: "string",
|
||||||
|
description: "Location name or ID (e.g., 'toronto', 'gta', 'ontario')",
|
||||||
|
},
|
||||||
|
category: {
|
||||||
|
type: "string",
|
||||||
|
description: "Category name or ID (e.g., 'computers', 'furniture', 'bikes')",
|
||||||
|
},
|
||||||
|
keywords: {
|
||||||
|
type: "string",
|
||||||
|
description: "Additional keywords to filter results",
|
||||||
|
},
|
||||||
|
sortBy: {
|
||||||
|
type: "string",
|
||||||
|
description: "Sort results by field",
|
||||||
|
enum: ["relevancy", "date", "price", "distance"],
|
||||||
|
default: "relevancy",
|
||||||
|
},
|
||||||
|
sortOrder: {
|
||||||
|
type: "string",
|
||||||
|
description: "Sort order",
|
||||||
|
enum: ["asc", "desc"],
|
||||||
|
default: "desc",
|
||||||
|
},
|
||||||
|
maxPages: {
|
||||||
type: "number",
|
type: "number",
|
||||||
description: "Maximum number of items to return",
|
description: "Maximum pages to fetch (~40 items per page)",
|
||||||
default: 5,
|
default: 5,
|
||||||
},
|
},
|
||||||
|
priceMin: {
|
||||||
|
type: "number",
|
||||||
|
description: "Minimum price in cents",
|
||||||
|
},
|
||||||
|
priceMax: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum price in cents",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
required: ["query"],
|
required: ["query"],
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in New Issue
Block a user