Compare commits

...

19 Commits

Author SHA1 Message Date
e741ead690 chore: disable mcp servers by default
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 23:54:34 -05:00
6ab9c4c3a5 chore: biome lint
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 22:34:05 -05:00
3919ec0727 chore: biome init
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 22:33:52 -05:00
1090ad5bfb fix: update Facebook API endpoint with cookiePath parameter 2026-01-22 19:57:27 -05:00
c937a70db7 feat: add Facebook marketplace item fetching API 2026-01-22 19:56:31 -05:00
59fcbf9ed2 feat: update fetchFacebookItems with cookie auto-loading 2026-01-22 19:56:02 -05:00
d8542eb8f7 feat: parse Facebook marketplace item details and test exports 2026-01-22 19:54:44 -05:00
0a114cf323 feat: extract individual Facebook marketplace items 2026-01-22 19:54:14 -05:00
5f7de1167e fix: add currency style and USD to formatCentsToCurrency 2026-01-22 19:53:53 -05:00
9edafc88c8 feat: add extraction monitoring and metrics logging 2026-01-22 19:52:39 -05:00
5871644e8b refactor: improve search extraction with edge case handling 2026-01-22 19:52:09 -05:00
d5d050013e feat: add Facebook cookie parsing and auto-loading 2026-01-22 19:51:35 -05:00
ff56a29171 feat: add cookiePath parameter to loadFacebookCookies 2026-01-22 19:51:18 -05:00
6a36214528 feat: add FacebookMarketplaceItem interface 2026-01-22 19:48:41 -05:00
7af1be3977 feat: improve Cookie interface type safety 2026-01-22 19:47:37 -05:00
844e566b57 feat: add Facebook cookie parser script 2026-01-22 19:35:47 -05:00
b3be32835a test: add Facebook marketplace test suite 2026-01-22 19:35:38 -05:00
baa34eefdf chore: agent.md
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 16:22:06 -05:00
9011ab4793 feat: fmarketplace docs
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 01:02:36 -05:00
16 changed files with 5076 additions and 2094 deletions

382
FMARKETPLACE.md Normal file
View File

@@ -0,0 +1,382 @@
# Facebook Marketplace API Reverse Engineering
## Overview
This document tracks findings from reverse-engineering Facebook Marketplace APIs for listing details.
## Current Implementation Status
- Search functionality: Implemented in `src/facebook.ts`
- Individual listing details: Not yet implemented
## Findings
### Step 1: Initial Setup
- Using Chrome DevTools to inspect Facebook Marketplace
- Need to authenticate with Facebook account to access marketplace data
- Cookies required for full access
- Current status: Successfully logged in and accessed marketplace data
### Step 2: Individual Listing Details Analysis - COMPLETED
- **Data Location**: Embedded in HTML script tags within `require` array structure
- **Path**: `require[0][3].__bbox.result.data.viewer.marketplace_product_details_page.target`
- **Authentication**: Required for full data access
- **Current Status**: Successfully reverse-engineered the API structure and data extraction method
### API Endpoints Discovered
#### Search Endpoint
- URL: `https://www.facebook.com/marketplace/{location}/search`
- Parameters: `query`, `sortBy`, `exact`
- Data embedded in HTML script tags with `require` structure
- Authentication: Required (cookies)
#### Listing Details Endpoint
- **URL Structure**: `https://www.facebook.com/marketplace/item/{listing_id}/`
- **Data Source**: Server-side rendered HTML with embedded JSON data in script tags
- **Data Structure**: Relay/GraphQL style data structure under `require[0][3].__bbox.require[...].__bbox.result.data.viewer.marketplace_product_details_page.target`
- **Extraction Method**: Parse JSON from script tags containing marketplace data, navigate to the target object
- **Authentication**: Required (cookies)
### Listing Data Structure Discovered (Current - 2026)
The current Facebook Marketplace API returns a comprehensive `GroupCommerceProductItem` object with the following key properties:
```typescript
interface FacebookMarketplaceItem {
// Basic identification
id: string;
__typename: "GroupCommerceProductItem";
// Listing content
marketplace_listing_title: string;
redacted_description: {
text: string;
};
custom_title?: string;
// Pricing
formatted_price: {
text: string;
};
listing_price: {
amount: string;
currency: string;
amount_with_offset: string;
};
// Location
location_text: {
text: string;
};
location: {
latitude: number;
longitude: number;
reverse_geocode_detailed: {
country_alpha_two: string;
postal_code_trimmed: string;
};
};
// Status flags
is_live: boolean;
is_sold: boolean;
is_pending: boolean;
is_hidden: boolean;
is_draft: boolean;
// Timing
creation_time: number;
// Seller information
marketplace_listing_seller: {
__typename: "User";
id: string;
name: string;
profile_picture?: {
uri: string;
};
join_time?: number;
};
// Vehicle-specific fields (for automotive listings)
vehicle_make_display_name?: string;
vehicle_model_display_name?: string;
vehicle_odometer_data?: {
unit: "KILOMETERS" | "MILES";
value: number;
};
vehicle_transmission_type?: "AUTOMATIC" | "MANUAL";
vehicle_exterior_color?: string;
vehicle_interior_color?: string;
vehicle_condition?: "EXCELLENT" | "GOOD" | "FAIR" | "POOR";
vehicle_fuel_type?: string;
vehicle_trim_display_name?: string;
// Category and commerce
marketplace_listing_category_id: string;
condition?: string;
// Commerce features
delivery_types?: string[];
is_shipping_offered?: boolean;
is_buy_now_enabled?: boolean;
can_buyer_make_checkout_offer?: boolean;
// Communication
messaging_enabled?: boolean;
first_message_suggested_value?: string;
// Metadata
logging_id: string;
reportable_ent_id: string;
origin_target?: {
__typename: "Marketplace";
id: string;
};
// Related listings (for part-out sellers)
marketplace_listing_sets?: {
edges: Array<{
node: {
canonical_listing: {
id: string;
marketplace_listing_title: string;
is_live: boolean;
is_sold: boolean;
formatted_price: { text: string };
};
};
}>;
};
}
```
### Example Data Extracted (Current Structure)
```json
{
"__typename": "GroupCommerceProductItem",
"marketplace_listing_title": "2012 Mazda MAZDA 3 PART-OUT",
"id": "1211645920845312",
"redacted_description": {
"text": "FOR PARTS ONLY!!!"
},
"custom_title": "2012 Mazda 3 part-out",
"creation_time": 1760450080,
"location_text": {
"text": "Toronto, ON"
},
"is_live": true,
"is_sold": false,
"is_pending": false,
"is_hidden": false,
"formatted_price": {
"text": "FREE"
},
"listing_price": {
"amount_with_offset": "0",
"currency": "CAD",
"amount": "0.00"
},
"condition": "USED",
"logging_id": "24676483845336407",
"marketplace_listing_category_id": "807311116002614",
"marketplace_listing_seller": {
"__typename": "User",
"id": "61570613529010",
"name": "Jay Heshin",
"profile_picture": {
"uri": "https://scontent-yyz1-1.xx.fbcdn.net/v/t39.30808-1/480952111_122133462296687117_4145652046222010716_n.jpg?stp=cp6_dst-jpg_s50x50_tt6&_nc_cat=108&ccb=1-7&_nc_sid=e99d92&_nc_ohc=x_DTkeriVbgQ7kNvwEqT_x3&_nc_oc=Adnqnqf4YsZxgMIkR2mSFrdLb6-BDw4omCWqG_cqB-H0uXGgK1l4-T-fLSGB_CQJEKo&_nc_zt=24&_nc_ht=scontent-yyz1-1.xx&_nc_gid=7GnSwn4MSbllAgGWJy0RTQ&oh=00_AfpY66l8w-LvHvZ6tTgiD9Qh-Or_Udc-OaFiVL9pQ0YXsg&oe=697797CD"
}
},
"vehicle_condition": "FAIR",
"vehicle_exterior_color": "white",
"vehicle_interior_color": "",
"vehicle_make_display_name": "Mazda",
"vehicle_model_display_name": "3 part-out",
"vehicle_odometer_data": {
"unit": "KILOMETERS",
"value": 999999
},
"vehicle_transmission_type": "AUTOMATIC",
"location": {
"latitude": 43.651428222656,
"longitude": -79.436645507812,
"reverse_geocode_detailed": {
"country_alpha_two": "CA",
"postal_code_trimmed": "M6H 1C1"
}
},
"delivery_types": ["IN_PERSON"],
"messaging_enabled": true,
"first_message_suggested_value": "Hi, is this available?",
"marketplace_listing_sets": {
"edges": [
{
"node": {
"canonical_listing": {
"id": "1435935788228627",
"marketplace_listing_title": "2004 Land Rover LR2 PART-OUT",
"is_live": true,
"formatted_price": {"text": "FREE"}
}
}
}
]
}
}
```
## Data Extraction Method
### Current Method (2026)
Facebook Marketplace listing data is embedded in JSON within `<script>` tags in the HTML response. The extraction process:
1. **Find the Correct Script**: Look for script tags containing marketplace listing data by searching for key fields like `marketplace_listing_title`, `redacted_description`, and `formatted_price`.
2. **Parse JSON Structure**: The data is nested within a `require` array structure:
```
require[0][3].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target
```
3. **Navigate to Target Object**: The actual listing data is a `GroupCommerceProductItem` object containing comprehensive information about the listing, seller, and vehicle details.
4. **Handle Dynamic Structure**: Facebook may change the exact path, so robust extraction should search for the target object recursively within the parsed JSON.
### Authentication Requirements
- Valid Facebook session cookies are required
- User must be logged in to Facebook
- Marketplace access may be location-restricted
## Tools Used
- Chrome DevTools Protocol
- Network monitoring
- HTML/script parsing
- JSON structure analysis
## Implementation Status
- ✅ Successfully reverse-engineered Facebook Marketplace API for listing details
- ✅ Identified current data structure and extraction method (2026)
- ✅ Documented comprehensive GroupCommerceProductItem interface
- ✅ Implemented `extractFacebookItemData()` function with script parsing logic
- ✅ Implemented `parseFacebookItem()` function to convert GroupCommerceProductItem to ListingDetails
- ✅ Implemented `fetchFacebookItem()` function with authentication and error handling
- ✅ Updated TypeScript interfaces to match current API structure
- ✅ Added robust extraction with fallback methods for changing API paths
## Implementation Details
### Core Functions Implemented
1. **`extractFacebookItemData(htmlString)`**: Extracts marketplace item data from HTML-embedded JSON in script tags
- Searches for scripts containing marketplace listing data
- Uses primary path: `require[0][3][0].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target`
- Falls back to recursive search for GroupCommerceProductItem objects
2. **`parseFacebookItem(item)`**: Converts Facebook's GroupCommerceProductItem to unified ListingDetails format
- Handles pricing (FREE listings, CAD currency)
- Extracts seller information, location, and status
- Supports vehicle-specific metadata
- Maps Facebook-specific fields to common interface
3. **`fetchFacebookItem(itemId, cookiesSource?)`**: Fetches individual listing details
- Loads Facebook authentication cookies
- Makes authenticated HTTP requests
- Handles rate limiting and retries
- Returns parsed ListingDetails or null on failure
### Authentication Requirements
- Facebook session cookies required in `./cookies/facebook.json` or provided as parameter
- Cookies must include valid authentication tokens for marketplace access
- Handles cookie expiration and domain validation
## Current Implementation Status - 2026 Verification
### Step 3: API Verification and Current Structure Analysis (January 2026)
- **Verification Date**: January 22, 2026
- **Status**: Successfully verified current Facebook Marketplace API structure
- **Data Source**: Embedded JSON in HTML script tags (server-side rendered)
- **Extraction Path**: `require[0][3].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target`
#### Verified Listing Structure (Real Example - 2006 Hyundai Tiburon)
- **Listing ID**: 1226468515995685
- **Title**: "2006 Hyundai Tiburon"
- **Price**: CA$3,000 (formatted_price.text)
- **Raw Price Data**: {"amount_with_offset": "300000", "currency": "CAD", "amount": "3000.00"}
- **Location**: Hamilton, ON (with coordinates: 43.250427246094, -79.963989257812)
- **Description**: "As is" (redacted_description.text)
- **Vehicle Details**:
- Make: Hyundai
- Model: Tiburon
- Odometer: 194,000 km
- Transmission: AUTOMATIC
- Exterior Color: blue
- Interior Color: black
- Fuel Type: GASOLINE
- Number of Owners: TWO
- **Seller Information**:
- Name: Ajitpal Kaler
- ID: 100009257293466
- Profile Picture Available
- Join Time: 1426564800 (2015)
- **Listing Status**: Active (is_live: true, is_sold: false, is_pending: false)
- **Category**: 807311116002614 (Vehicles)
- **Delivery Types**: ["IN_PERSON"]
- **Messaging**: Enabled
#### Current API Characteristics
- **Authentication**: Still requires valid Facebook session cookies
- **Data Format**: Server-side rendered HTML with embedded GraphQL/Relay JSON
- **Structure Stability**: Primary extraction path remains functional
- **Additional Features**: Includes marketplace ratings, seller verification badges, cross-posting info
### API Changes Observed Since 2024 Documentation
- **Minimal Changes**: Core data structure largely unchanged
- **Enhanced Fields**: Added more detailed vehicle specifications and seller profile information
- **GraphQL Integration**: Deeper integration with Facebook's GraphQL infrastructure
- **Security Features**: Additional integrity checks and reporting mechanisms
### Multi-Category Testing Results (January 2026)
Successfully tested extraction across different listing categories:
#### 1. Vehicle Listings (Automotive)
- **Example**: 2006 Hyundai Tiburon (ID: 1226468515995685)
- **Status**: ✅ Fully functional
- **Data Extracted**: Complete vehicle specs, pricing, seller info, location coordinates
- **Unique Fields**: vehicle_make_display_name, vehicle_odometer_data, vehicle_transmission_type, vehicle_exterior_color, vehicle_interior_color, vehicle_fuel_type
#### 2. Electronics Listings
- **Example**: Nintendo Switch (ID: 3903865769914262)
- **Status**: ✅ Fully functional
- **Data Extracted**: Title, price (CA$140), location (Toronto, ON), condition (Used - like new), seller (Yitao Hou)
- **Category**: Electronics (category_id: 479353692612078)
- **Notes**: Standard GroupCommerceProductItem structure applies
#### 3. Home Goods/Furniture Listings
- **Example**: Tabletop Mirror (cat not included) (ID: 1082389057290709)
- **Status**: ✅ Fully functional
- **Data Extracted**: Title, price (CA$5), location (Mississauga, ON), condition (Used - like new), seller (Rohit Rehan)
- **Category**: Home Goods (category_id: 1569171756675761)
- **Notes**: Includes detailed description and delivery options
#### Testing Summary
- **Extraction Method**: Consistent across all categories
- **Data Structure**: GroupCommerceProductItem interface works for all listing types
- **Authentication**: Required for all categories
- **Rate Limiting**: Standard Facebook rate limits apply
- **Edge Cases**: All tested listings were active/in-person pickup
## Implementation Status - COMPLETED (January 2026)
- ✅ Successfully reverse-engineered Facebook Marketplace API for listing details
- ✅ Verified current API structure and extraction method (January 2026)
- ✅ Tested extraction across multiple listing categories (vehicles, electronics, home goods)
- ✅ Implemented comprehensive error handling for sold/removed listings and authentication failures
- ✅ Enhanced rate limiting and retry logic (already robust)
- ✅ Added monitoring and metrics for API stability detection
- ✅ Updated all scraper functions to use verified extraction methods
- ✅ Documented comprehensive GroupCommerceProductItem interface with real examples
## Next Steps (Future Maintenance)
1. Monitor extraction success rates for API change detection
2. Update extraction paths if Facebook changes their API structure
3. Add support for additional marketplace features as they become available
4. Implement caching mechanisms for improved performance
5. Add support for marketplace messaging and negotiation features

30
biome.json Normal file
View File

@@ -0,0 +1,30 @@
{
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
"vcs": {
"enabled": false,
"clientKind": "git",
"useIgnoreFile": false
},
"files": {
"ignoreUnknown": false,
"ignore": []
},
"formatter": {
"enabled": true,
"indentStyle": "space"
},
"organizeImports": {
"enabled": true
},
"linter": {
"enabled": true,
"rules": {
"recommended": true
}
},
"javascript": {
"formatter": {
"quoteStyle": "double"
}
}
}

View File

@@ -14,12 +14,14 @@
"-e",
"/nix/store/lz8ajxhnkkw2llj752bdz41wqr645h9c-google-chrome-dev-146.0.7635.0/bin/google-chrome-unstable",
"--ignore-default-chrome-arg='--disable-extensions'"
]
],
"enabled": false
},
"bun-docs": {
"type": "remote",
"url": "https://bun.com/docs/mcp",
"timeout": 3000
"timeout": 3000,
"enabled": false
}
}
}

View File

@@ -0,0 +1,183 @@
#!/usr/bin/env bun
/**
* Facebook Cookie Parser CLI
*
* Parses Facebook cookie strings into JSON format for the marketplace scraper
*
* Usage:
* bun run scripts/parse-facebook-cookies.ts "c_user=123; xs=abc"
* bun run scripts/parse-facebook-cookies.ts --input cookies.txt
* echo "c_user=123; xs=abc" | bun run scripts/parse-facebook-cookies.ts
* bun run scripts/parse-facebook-cookies.ts "cookie_string" --output my-cookies.json
*/
import { parseFacebookCookieString } from "../src/facebook";
interface Cookie {
name: string;
value: string;
domain: string;
path: string;
secure?: boolean;
httpOnly?: boolean;
sameSite?: "strict" | "lax" | "none" | "unspecified";
expirationDate?: number;
storeId?: string;
}
function parseFacebookCookieStringCLI(cookieString: string): Cookie[] {
if (!cookieString || !cookieString.trim()) {
console.error("❌ Error: Empty or invalid cookie string provided");
process.exit(1);
}
const cookies = parseFacebookCookieString(cookieString);
if (cookies.length === 0) {
console.error("❌ Error: No valid cookies found in input string");
console.error('Expected format: "name1=value1; name2=value2;"');
process.exit(1);
}
return cookies;
}
async function main() {
const args = process.argv.slice(2);
if (args.length === 0 && process.stdin.isTTY === false) {
// Read from stdin
let input = "";
for await (const chunk of process.stdin) {
input += chunk;
}
input = input.trim();
if (!input) {
console.error("❌ Error: No input provided via stdin");
process.exit(1);
}
const cookies = parseFacebookCookieStringCLI(input);
await writeOutput(cookies, "./cookies/facebook.json");
return;
}
let cookieString = "";
let outputPath = "./cookies/facebook.json";
let inputPath = "";
// Parse command line arguments
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg === "--input" || arg === "-i") {
inputPath = args[i + 1];
i++; // Skip next arg
} else if (arg === "--output" || arg === "-o") {
outputPath = args[i + 1];
i++; // Skip next arg
} else if (arg === "--help" || arg === "-h") {
showHelp();
return;
} else if (!arg.startsWith("-")) {
// Assume this is the cookie string
cookieString = arg;
} else {
console.error(`❌ Unknown option: ${arg}`);
showHelp();
process.exit(1);
}
}
// Read from file if specified
if (inputPath) {
try {
const file = Bun.file(inputPath);
if (!(await file.exists())) {
console.error(`❌ Error: Input file not found: ${inputPath}`);
process.exit(1);
}
cookieString = await file.text();
} catch (error) {
console.error(`❌ Error reading input file: ${error}`);
process.exit(1);
}
}
if (!cookieString.trim()) {
console.error("❌ Error: No cookie string provided");
console.error(
"Provide cookie string as argument, --input file, or via stdin",
);
showHelp();
process.exit(1);
}
const cookies = parseFacebookCookieStringCLI(cookieString);
await writeOutput(cookies, outputPath);
}
async function writeOutput(cookies: Cookie[], outputPath: string) {
try {
await Bun.write(outputPath, JSON.stringify(cookies, null, 2));
console.log(`✅ Successfully parsed ${cookies.length} Facebook cookies`);
console.log(`📁 Saved to: ${outputPath}`);
// Show summary of parsed cookies
console.log("\n📋 Parsed cookies:");
for (const cookie of cookies) {
console.log(
`${cookie.name}: ${cookie.value.substring(0, 20)}${cookie.value.length > 20 ? "..." : ""}`,
);
}
} catch (error) {
console.error(`❌ Error writing to output file: ${error}`);
process.exit(1);
}
}
function showHelp() {
console.log(`
Facebook Cookie Parser CLI
Parses Facebook cookie strings into JSON format for the marketplace scraper.
USAGE:
bun run scripts/parse-facebook-cookies.ts [OPTIONS] [COOKIE_STRING]
EXAMPLES:
# Parse from command line argument
bun run scripts/parse-facebook-cookies.ts "c_user=123; xs=abc"
# Parse from file
bun run scripts/parse-facebook-cookies.ts --input cookies.txt
# Parse from stdin
echo "c_user=123; xs=abc" | bun run scripts/parse-facebook-cookies.ts
# Output to custom file
bun run scripts/parse-facebook-cookies.ts "cookie_string" --output my-cookies.json
OPTIONS:
-i, --input FILE Read cookie string from file
-o, --output FILE Output file path (default: ./cookies/facebook.json)
-h, --help Show this help message
COOKIE FORMAT:
Semicolon-separated name=value pairs
Example: "c_user=123456789; xs=abcdef123456; fr=xyz789"
OUTPUT:
JSON array of cookie objects saved to ./cookies/facebook.json
`);
}
// Run the CLI
if (import.meta.main) {
main().catch((error) => {
console.error(`❌ Unexpected error: ${error}`);
process.exit(1);
});
}

View File

@@ -1,6 +1,6 @@
import cliProgress from "cli-progress";
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom";
import cliProgress from "cli-progress";
// ----------------------------- Types -----------------------------
@@ -55,8 +55,10 @@ function formatCentsToCurrency(
/**
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
*/
function parseEbayPrice(priceText: string): { cents: number; currency: string } | null {
if (!priceText || typeof priceText !== 'string') return null;
function parseEbayPrice(
priceText: string,
): { cents: number; currency: string } | null {
if (!priceText || typeof priceText !== "string") return null;
// Clean up the price text and extract currency and amount
const cleaned = priceText.trim();
@@ -65,19 +67,23 @@ function parseEbayPrice(priceText: string): { cents: number; currency: string }
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
if (!numberMatches) return null;
const amountStr = numberMatches[0].replace(/,/g, '');
const dollars = parseFloat(amountStr);
if (isNaN(dollars)) return null;
const amountStr = numberMatches[0].replace(/,/g, "");
const dollars = Number.parseFloat(amountStr);
if (Number.isNaN(dollars)) return null;
const cents = Math.round(dollars * 100);
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
let currency = 'USD'; // Default
let currency = "USD"; // Default
if (cleaned.toUpperCase().includes('CAD') || cleaned.includes('CA$') || cleaned.includes('C $')) {
currency = 'CAD';
} else if (cleaned.toUpperCase().includes('USD') || cleaned.includes('$')) {
currency = 'USD';
if (
cleaned.toUpperCase().includes("CAD") ||
cleaned.includes("CA$") ||
cleaned.includes("C $")
) {
currency = "CAD";
} else if (cleaned.toUpperCase().includes("USD") || cleaned.includes("$")) {
currency = "USD";
}
return { cents, currency };
@@ -135,7 +141,9 @@ async function fetchHtml(
if (!res.ok) {
// Respect 429 reset if provided
if (res.status === 429) {
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : NaN;
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: (attempt + 1) * retryBaseMs;
@@ -176,7 +184,7 @@ function parseEbayListings(
htmlString: HTMLString,
keywords: string[],
exclusions: string[],
strictMode: boolean
strictMode: boolean,
): ListingDetails[] {
const { document } = parseHTML(htmlString);
const results: ListingDetails[] = [];
@@ -184,16 +192,17 @@ function parseEbayListings(
// Find all listing links by looking for eBay item URLs (/itm/)
const linkElements = document.querySelectorAll('a[href*="itm/"]');
for (const linkElement of linkElements) {
try {
// Get href attribute
let href = linkElement.getAttribute('href');
let href = linkElement.getAttribute("href");
if (!href) continue;
// Make href absolute
if (!href.startsWith('http')) {
href = href.startsWith('//') ? `https:${href}` : `https://www.ebay.com${href}`;
if (!href.startsWith("http")) {
href = href.startsWith("//")
? `https:${href}`
: `https://www.ebay.com${href}`;
}
// Find the container - go up several levels to find the item container
@@ -207,15 +216,23 @@ function parseEbayListings(
// Extract title - look for heading or title-related elements near the link
// Modern eBay often uses h3, span, or div with text content near the link
let titleElement = container.querySelector('h3, [role="heading"], .s-item__title span');
let titleElement = container.querySelector(
'h3, [role="heading"], .s-item__title span',
);
// If no direct title element, try finding text content around the link
if (!titleElement) {
// Look for spans or divs with text near this link
const nearbySpans = container.querySelectorAll('span, div');
const nearbySpans = container.querySelectorAll("span, div");
for (const span of nearbySpans) {
const text = span.textContent?.trim();
if (text && text.length > 10 && text.length < 200 && !text.includes('$') && !text.includes('item')) {
if (
text &&
text.length > 10 &&
text.length < 200 &&
!text.includes("$") &&
!text.includes("item")
) {
titleElement = span;
break;
}
@@ -228,12 +245,12 @@ function parseEbayListings(
if (title) {
// Remove common eBay UI strings that appear at the end of titles
const uiStrings = [
'Opens in a new window',
'Opens in a new tab',
'Opens in a new window or tab',
'opens in a new window',
'opens in a new tab',
'opens in a new window or tab'
"Opens in a new window",
"Opens in a new tab",
"Opens in a new window or tab",
"opens in a new window",
"opens in a new tab",
"opens in a new window or tab",
];
for (const uiString of uiStrings) {
@@ -256,17 +273,27 @@ function parseEbayListings(
if (title === "Shop on eBay" || title.length < 3) continue;
// Extract price - look for eBay's price classes, preferring sale/discount prices
let priceElement = container.querySelector('[class*="s-item__price"], .s-item__price, [class*="price"]');
let priceElement = container.querySelector(
'[class*="s-item__price"], .s-item__price, [class*="price"]',
);
// If no direct price class, look for spans containing $ (but not titles)
if (!priceElement) {
const spansAndElements = container.querySelectorAll('span, div, b, em, strong');
const spansAndElements = container.querySelectorAll(
"span, div, b, em, strong",
);
for (const el of spansAndElements) {
const text = el.textContent?.trim();
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
if (text && text.includes('$') && text.length < 100 &&
!text.includes('laptop') && !text.includes('computer') && !text.includes('intel') &&
!text.includes('core') && !text.includes('ram') && !text.includes('ssd') &&
if (
text?.includes("$") &&
text.length < 100 &&
!text.includes("laptop") &&
!text.includes("computer") &&
!text.includes("intel") &&
!text.includes("core") &&
!text.includes("ram") &&
!text.includes("ssd") &&
!/\d{4}/.test(text) && // Avoid years like "2024"
!text.includes('"') // Avoid measurements
) {
@@ -280,17 +307,26 @@ function parseEbayListings(
// Prefer sale/current price over original/strikethrough price
if (priceElement) {
// Check if this element or its parent contains multiple price elements
const priceContainer = priceElement.closest('[class*="s-item__price"]') || priceElement.parentElement;
const priceContainer =
priceElement.closest('[class*="s-item__price"]') ||
priceElement.parentElement;
if (priceContainer) {
// Look for all price elements within this container, including strikethrough prices
const allPriceElements = priceContainer.querySelectorAll('[class*="s-item__price"], span, b, em, strong, s, del, strike');
const allPriceElements = priceContainer.querySelectorAll(
'[class*="s-item__price"], span, b, em, strong, s, del, strike',
);
// Filter to only elements that actually contain prices (not labels)
const actualPrices: HTMLElement[] = [];
for (const el of allPriceElements) {
const text = el.textContent?.trim();
if (text && /^\s*[\$£¥]/u.test(text) && text.length < 50 && !/\d{4}/.test(text)) {
if (
text &&
/^\s*[\$£¥]/u.test(text) &&
text.length < 50 &&
!/\d{4}/.test(text)
) {
actualPrices.push(el);
}
}
@@ -298,11 +334,18 @@ function parseEbayListings(
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
if (actualPrices.length > 1) {
// First, look for prices that are NOT struck through
const nonStrikethroughPrices = actualPrices.filter(el => {
const nonStrikethroughPrices = actualPrices.filter((el) => {
const tagName = el.tagName.toLowerCase();
const styles = el.classList.contains('s-strikethrough') || el.classList.contains('u-flStrike') ||
el.closest('s, del, strike');
return tagName !== 's' && tagName !== 'del' && tagName !== 'strike' && !styles;
const styles =
el.classList.contains("s-strikethrough") ||
el.classList.contains("u-flStrike") ||
el.closest("s, del, strike");
return (
tagName !== "s" &&
tagName !== "del" &&
tagName !== "strike" &&
!styles
);
});
if (nonStrikethroughPrices.length > 0) {
@@ -317,7 +360,7 @@ function parseEbayListings(
}
}
let priceText = priceElement?.textContent?.trim();
const priceText = priceElement?.textContent?.trim();
if (!priceText) continue;
@@ -326,12 +369,21 @@ function parseEbayListings(
if (!priceInfo) continue;
// Apply exclusion filters
if (exclusions.some(exclusion => title.toLowerCase().includes(exclusion.toLowerCase()))) {
if (
exclusions.some((exclusion) =>
title.toLowerCase().includes(exclusion.toLowerCase()),
)
) {
continue;
}
// Apply strict mode filter (title must contain at least one keyword)
if (strictMode && !keywords.some(keyword => title!.toLowerCase().includes(keyword.toLowerCase()))) {
if (
strictMode &&
!keywords.some((keyword) =>
title?.toLowerCase().includes(keyword.toLowerCase()),
)
) {
continue;
}
@@ -351,7 +403,6 @@ function parseEbayListings(
results.push(listing);
} catch (err) {
console.warn(`Error parsing eBay listing: ${err}`);
continue;
}
}
@@ -376,7 +427,7 @@ export default async function fetchEbayItems(
maxPrice = Number.MAX_SAFE_INTEGER,
strictMode = false,
exclusions = [],
keywords = [SEARCH_QUERY] // Default to search query if no keywords provided
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
} = opts;
// Build eBay search URL - use Canadian site and tracking parameters like real browser
@@ -389,18 +440,19 @@ export default async function fetchEbayItems(
try {
// Use custom headers modeled after real browser requests to bypass bot detection
const headers: Record<string, string> = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.ebay.ca/',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Priority': 'u=0, i'
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
Referer: "https://www.ebay.ca/",
Connection: "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-origin",
"Sec-Fetch-User": "?1",
Priority: "u=0, i",
};
const res = await fetch(searchUrl, {
@@ -420,19 +472,23 @@ export default async function fetchEbayItems(
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS);
console.log(`\nParsing eBay listings...`);
console.log("\nParsing eBay listings...");
const listings = parseEbayListings(searchHtml, keywords, exclusions, strictMode);
const listings = parseEbayListings(
searchHtml,
keywords,
exclusions,
strictMode,
);
// Filter by price range (additional safety check)
const filteredListings = listings.filter(listing => {
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return cents && cents >= minPrice && cents <= maxPrice;
});
console.log(`Parsed ${filteredListings.length} eBay listings.`);
return filteredListings;
} catch (err) {
if (err instanceof HttpError) {
console.error(

View File

@@ -1,6 +1,6 @@
import cliProgress from "cli-progress";
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom";
import cliProgress from "cli-progress";
/**
* Facebook Marketplace Scraper
@@ -24,7 +24,7 @@ interface Cookie {
sameSite?: "strict" | "lax" | "none" | "unspecified";
session?: boolean;
expirationDate?: number;
partitionKey?: any;
partitionKey?: Record<string, unknown>;
storeId?: string;
}
@@ -68,6 +68,112 @@ interface FacebookRequireData {
[k: string]: unknown;
}
interface FacebookMarketplaceItem {
// Basic identification
id: string;
__typename: "GroupCommerceProductItem";
// Listing content
marketplace_listing_title: string;
redacted_description?: {
text: string;
};
custom_title?: string;
// Pricing
formatted_price?: {
text: string;
};
listing_price?: {
amount: string;
currency: string;
amount_with_offset: string;
};
// Location
location_text?: {
text: string;
};
location?: {
latitude: number;
longitude: number;
reverse_geocode_detailed?: {
country_alpha_two: string;
postal_code_trimmed: string;
};
};
// Status flags
is_live?: boolean;
is_sold?: boolean;
is_pending?: boolean;
is_hidden?: boolean;
is_draft?: boolean;
// Timing
creation_time?: number;
// Seller information
marketplace_listing_seller?: {
__typename: "User";
id: string;
name: string;
profile_picture?: {
uri: string;
};
join_time?: number;
};
// Vehicle-specific fields (for automotive listings)
vehicle_make_display_name?: string;
vehicle_model_display_name?: string;
vehicle_odometer_data?: {
unit: "KILOMETERS" | "MILES";
value: number;
};
vehicle_transmission_type?: "AUTOMATIC" | "MANUAL";
vehicle_exterior_color?: string;
vehicle_interior_color?: string;
vehicle_condition?: "EXCELLENT" | "GOOD" | "FAIR" | "POOR";
vehicle_fuel_type?: string;
vehicle_trim_display_name?: string;
// Category and commerce
marketplace_listing_category_id?: string;
condition?: string;
// Commerce features
delivery_types?: string[];
is_shipping_offered?: boolean;
is_buy_now_enabled?: boolean;
can_buyer_make_checkout_offer?: boolean;
// Communication
messaging_enabled?: boolean;
first_message_suggested_value?: string;
// Metadata
logging_id?: string;
reportable_ent_id?: string;
// Related listings (for part-out sellers)
marketplace_listing_sets?: {
edges: Array<{
node: {
canonical_listing: {
id: string;
marketplace_listing_title: string;
is_live: boolean;
is_sold: boolean;
formatted_price: { text: string };
};
};
}>;
};
[k: string]: unknown;
}
type ListingDetails = {
url: string;
title: string;
@@ -107,7 +213,10 @@ async function delay(ms: number): Promise<void> {
/**
* Load Facebook cookies from file or string
*/
async function loadFacebookCookies(cookiesSource?: string): Promise<Cookie[]> {
async function loadFacebookCookies(
cookiesSource?: string,
cookiePath = "./cookies/facebook.json",
): Promise<Cookie[]> {
// First try to load from provided string parameter
if (cookiesSource) {
try {
@@ -125,9 +234,9 @@ async function loadFacebookCookies(cookiesSource?: string): Promise<Cookie[]> {
}
}
// Try to load from ./cookies/facebook.json
// Try to load from specified path
try {
const cookiesPath = "./cookies/facebook.json";
const cookiesPath = cookiePath;
const file = Bun.file(cookiesPath);
if (await file.exists()) {
const content = await file.text();
@@ -148,6 +257,89 @@ async function loadFacebookCookies(cookiesSource?: string): Promise<Cookie[]> {
return [];
}
/**
* Parse Facebook cookie string into Cookie array format
*/
function parseFacebookCookieString(cookieString: string): Cookie[] {
if (!cookieString || !cookieString.trim()) {
return [];
}
return cookieString
.split(";")
.map((pair) => pair.trim())
.filter((pair) => pair.includes("="))
.map((pair) => {
const [name, value] = pair.split("=", 2);
const trimmedName = name.trim();
const trimmedValue = value.trim();
// Skip empty names or values
if (!trimmedName || !trimmedValue) {
return null;
}
return {
name: trimmedName,
value: decodeURIComponent(trimmedValue),
domain: ".facebook.com",
path: "/",
secure: true,
httpOnly: false,
sameSite: "lax" as const,
expirationDate: undefined, // Session cookies
};
})
.filter((cookie): cookie is Cookie => cookie !== null);
}
/**
* Ensure Facebook cookies are available, parsing from env var if needed
*/
async function ensureFacebookCookies(
cookiePath = "./cookies/facebook.json",
): Promise<Cookie[]> {
// First try to load existing cookies
try {
const existing = await loadFacebookCookies(undefined, cookiePath);
if (existing.length > 0) {
return existing;
}
} catch (error) {
// File doesn't exist or is invalid, continue to check env var
}
// Try to parse from environment variable
const cookieString = process.env.FACEBOOK_COOKIE;
if (!cookieString || !cookieString.trim()) {
throw new Error(
"No valid Facebook cookies found. Either:\n" +
" 1. Set FACEBOOK_COOKIE environment variable with cookie string, or\n" +
" 2. Create ./cookies/facebook.json manually with cookie array",
);
}
// Parse the cookie string
const cookies = parseFacebookCookieString(cookieString);
if (cookies.length === 0) {
throw new Error(
"FACEBOOK_COOKIE environment variable contains no valid cookies. " +
'Expected format: "name1=value1; name2=value2;"',
);
}
// Save to file for future use
try {
await Bun.write(cookiePath, JSON.stringify(cookies, null, 2));
console.log(`✅ Saved ${cookies.length} Facebook cookies to ${cookiePath}`);
} catch (error) {
console.warn(`! Could not save cookies to ${cookiePath}: ${error}`);
// Continue anyway, we have the cookies in memory
}
return cookies;
}
/**
* Format cookies array into Cookie header string
*/
@@ -161,10 +353,9 @@ function formatCookiesForHeader(cookies: Cookie[], domain: string): string {
domain.endsWith(cookie.domain.slice(1)) ||
domain === cookie.domain.slice(1)
);
} else {
}
// Host-only cookie
return cookie.domain === domain;
}
})
.filter((cookie) => {
// Check expiration
@@ -230,7 +421,7 @@ async function fetchHtml(
// Add cookies if provided
if (opts?.cookies) {
headers["cookie"] = opts.cookies;
headers.cookie = opts.cookies;
}
const res = await fetch(url, {
@@ -245,7 +436,9 @@ async function fetchHtml(
if (!res.ok) {
// Respect 429 reset if provided
if (res.status === 429) {
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : NaN;
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: (attempt + 1) * retryBaseMs;
@@ -300,7 +493,7 @@ function extractFacebookMarketplaceData(
let marketplaceData: FacebookMarketplaceSearch | null = null;
// Find the script containing the require data with marketplace_search
for (const script of scripts as unknown as HTMLScriptElement[]) {
for (const script of Array.from(scripts) as HTMLScriptElement[]) {
const scriptText = script.textContent;
if (!scriptText) continue;
@@ -312,58 +505,114 @@ function extractFacebookMarketplaceData(
// Try multiple navigation paths to find marketplace_search
const paths = [
// Original path from example
() => parsed.require[0][3][0]['__bbox']['require'][0][3][1]['__bbox']['result']['data']['marketplace_search'],
() =>
parsed.require[0][3][0].__bbox.require[0][3][1].__bbox.result.data
.marketplace_search,
// Alternative path structure
() => parsed.require[0][3][1]?.__bbox?.result?.data?.marketplace_search,
() =>
parsed.require[0][3][1]?.__bbox?.result?.data?.marketplace_search,
// Another variation
() => parsed.require[0][3][0]['__bbox']['result']['data']['marketplace_search'],
() => parsed.require[0][3][0].__bbox.result.data.marketplace_search,
// Direct access for some responses
() => {
for (const item of parsed.require) {
if (item && item.length >= 4 && item[3]) {
const bbox = item[3]?.['__bbox']?.result?.data?.marketplace_search;
const bbox = item[3]?.__bbox?.result?.data?.marketplace_search;
if (bbox) return bbox;
}
}
return null;
}
},
];
for (const getData of paths) {
try {
const result = getData();
if (result && isRecord(result) && result.feed_units?.edges) {
if (
result &&
isRecord(result) &&
result.feed_units?.edges?.length > 0
) {
marketplaceData = result as FacebookMarketplaceSearch;
break;
}
} catch {
continue;
}
} catch {}
}
if (marketplaceData) break;
}
// Also check for direct marketplace_search in the parsed data
if (parsed.marketplace_search && isRecord(parsed.marketplace_search) && parsed.marketplace_search.feed_units?.edges) {
marketplaceData = parsed.marketplace_search as FacebookMarketplaceSearch;
if (parsed.marketplace_search && isRecord(parsed.marketplace_search)) {
const searchData =
parsed.marketplace_search as FacebookMarketplaceSearch;
if (searchData.feed_units?.edges?.length > 0) {
marketplaceData = searchData;
break;
}
}
} catch {
// Ignore parsing errors for other scripts
continue;
}
}
if (!marketplaceData?.feed_units?.edges) {
if (!marketplaceData?.feed_units?.edges?.length) {
console.warn("No marketplace data found in HTML response");
return null;
}
console.log(`Successfully parsed ${marketplaceData.feed_units.edges.length} Facebook marketplace listings`);
console.log(
`Successfully parsed ${marketplaceData.feed_units.edges.length} Facebook marketplace listings`,
);
return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node }));
}
/**
* Monitor API extraction success/failure for detecting changes
*/
const extractionStats = {
totalExtractions: 0,
successfulExtractions: 0,
failedExtractions: 0,
lastApiChangeDetected: null as Date | null,
};
/**
* Log extraction metrics for monitoring API stability
*/
function logExtractionMetrics(success: boolean, itemId?: string) {
extractionStats.totalExtractions++;
if (success) {
extractionStats.successfulExtractions++;
} else {
extractionStats.failedExtractions++;
}
// Log warning if extraction success rate drops below 80%
const successRate =
extractionStats.successfulExtractions / extractionStats.totalExtractions;
if (
extractionStats.totalExtractions > 10 &&
successRate < 0.8 &&
!extractionStats.lastApiChangeDetected
) {
console.warn(
"! Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
);
extractionStats.lastApiChangeDetected = new Date();
}
if (success) {
console.log(
`📊 Facebook API extraction stats: ${extractionStats.successfulExtractions}/${extractionStats.totalExtractions} successful`,
);
} else {
console.warn(
`❌ Facebook API extraction failed for item ${itemId || "unknown"}`,
);
}
}
/**
* Turns cents to localized currency string.
*/
@@ -376,6 +625,8 @@ function formatCentsToCurrency(
if (Number.isNaN(cents)) return "";
const dollars = cents / 100;
const formatter = new Intl.NumberFormat(locale, {
style: "currency",
currency: "USD",
minimumFractionDigits: 2,
maximumFractionDigits: 2,
useGrouping: true,
@@ -383,6 +634,145 @@ function formatCentsToCurrency(
return formatter.format(dollars);
}
/**
Extract marketplace item details from Facebook item page HTML
Updated for 2026 Facebook Marketplace API structure with multiple extraction paths
*/
function extractFacebookItemData(
htmlString: HTMLString,
): FacebookMarketplaceItem | null {
const { document } = parseHTML(htmlString);
const scripts = document.querySelectorAll("script");
for (const script of scripts) {
const scriptText = script.textContent;
if (!scriptText) continue;
try {
const parsed = JSON.parse(scriptText);
// Check for the 2026 require structure with marketplace product details
if (parsed.require && Array.isArray(parsed.require)) {
// Try multiple extraction paths discovered from reverse engineering
const extractionPaths = [
// Path 1: Primary path from current API structure
() =>
parsed.require[0][3].__bbox.result.data.viewer
.marketplace_product_details_page.target,
// Path 2: Alternative path with nested require
() =>
parsed.require[0][3][0].__bbox.require[3][3][1].__bbox.result.data
.viewer.marketplace_product_details_page.target,
// Path 3: Variation without the [0] index
() =>
parsed.require[0][3].__bbox.require[3][3][1].__bbox.result.data
.viewer.marketplace_product_details_page.target,
// Path 4-5: Additional fallback paths for edge cases
() =>
parsed.require[0][3][1]?.__bbox?.result?.data?.viewer
?.marketplace_product_details_page?.target,
() =>
parsed.require[0][3][2]?.__bbox?.result?.data?.viewer
?.marketplace_product_details_page?.target,
];
let pathIndex = 0;
for (const getPath of extractionPaths) {
try {
const targetData = getPath();
if (
targetData &&
typeof targetData === "object" &&
targetData.id &&
targetData.marketplace_listing_title &&
targetData.__typename === "GroupCommerceProductItem"
) {
console.log(
`Successfully extracted Facebook item data using extraction path ${pathIndex + 1}`,
);
return targetData as FacebookMarketplaceItem;
}
} catch {
// Path not found or invalid, try next path
}
pathIndex++;
}
// Fallback: Search recursively for marketplace data in the parsed structure
const findMarketplaceData = (
obj: unknown,
depth = 0,
maxDepth = 10,
): FacebookMarketplaceItem | null => {
if (depth > maxDepth) return null; // Prevent infinite recursion
if (isRecord(obj)) {
// Check if this object matches the expected marketplace item structure
if (
obj.marketplace_listing_title &&
obj.id &&
obj.__typename === "GroupCommerceProductItem" &&
obj.redacted_description
) {
return obj as FacebookMarketplaceItem;
}
// Recursively search nested objects and arrays
for (const key in obj) {
const value = obj[key];
if (isRecord(value) || Array.isArray(value)) {
const result = findMarketplaceData(value, depth + 1, maxDepth);
if (result) return result;
}
}
} else if (Array.isArray(obj)) {
// Search through arrays
for (const item of obj) {
const result = findMarketplaceData(item, depth + 1, maxDepth);
if (result) return result;
}
}
return null;
};
// Search through the entire require structure
const recursiveResult = findMarketplaceData(parsed.require);
if (recursiveResult) {
console.log(
"Successfully extracted Facebook item data using recursive search",
);
return recursiveResult;
}
// Additional search in other potential locations
if (
parsed.__bbox?.result?.data?.viewer?.marketplace_product_details_page
?.target
) {
const bboxData =
parsed.__bbox.result.data.viewer.marketplace_product_details_page
.target;
if (
bboxData &&
typeof bboxData === "object" &&
bboxData.id &&
bboxData.marketplace_listing_title &&
bboxData.__typename === "GroupCommerceProductItem"
) {
console.log(
"Successfully extracted Facebook item data from __bbox structure",
);
return bboxData as FacebookMarketplaceItem;
}
}
}
} catch (error) {
// Log parsing errors for debugging but continue to next script
console.debug(`Failed to parse script for Facebook item data: ${error}`);
}
}
return null;
}
/**
Parse Facebook marketplace search results into ListingDetails[]
*/
@@ -406,7 +796,8 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
// - formatted_amount: human-readable price (like "CA$1")
let cents: number;
if (priceObj.amount != null) {
const dollars = typeof priceObj.amount === 'string'
const dollars =
typeof priceObj.amount === "string"
? Number.parseFloat(priceObj.amount)
: priceObj.amount;
cents = Math.round(dollars * 100);
@@ -420,7 +811,7 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
if (priceObj.formatted_amount) {
const match = priceObj.formatted_amount.match(/[\d,]+\.?\d*/);
if (match) {
const dollars = Number.parseFloat(match[0].replace(',', ''));
const dollars = Number.parseFloat(match[0].replace(",", ""));
if (!Number.isNaN(dollars)) {
cents = Math.round(dollars * 100);
} else {
@@ -465,19 +856,24 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
// Extract image and video URLs
const imageUrl = listing.primary_listing_photo?.image?.uri;
const videoUrl = listing.listing_video ? `https://www.facebook.com/${listing.listing_video.id}/` : undefined;
const videoUrl = listing.listing_video
? `https://www.facebook.com/${listing.listing_video.id}/`
: undefined;
// Extract seller information
const seller = listing.marketplace_listing_seller ? {
const seller = listing.marketplace_listing_seller
? {
name: listing.marketplace_listing_seller.name,
id: listing.marketplace_listing_seller.id
} : undefined;
id: listing.marketplace_listing_seller.id,
}
: undefined;
const listingDetails: ListingDetails = {
url,
title,
listingPrice: {
amountFormatted: priceObj.formatted_amount || formatCentsToCurrency(cents),
amountFormatted:
priceObj.formatted_amount || formatCentsToCurrency(cents),
cents,
currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD
},
@@ -493,15 +889,121 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
};
results.push(listingDetails);
} catch {
// Skip malformed ads
continue;
}
} catch {}
}
return results;
}
/**
Parse Facebook marketplace item details into ListingDetails format
Updated for 2026 GroupCommerceProductItem structure
*/
function parseFacebookItem(
item: FacebookMarketplaceItem,
): ListingDetails | null {
try {
const title = item.marketplace_listing_title || item.custom_title;
if (!title) return null;
const url = `https://www.facebook.com/marketplace/item/${item.id}`;
// Extract price information
let cents = 0;
let currency = "CAD"; // Default
let amountFormatted = item.formatted_price?.text || "FREE";
if (item.listing_price) {
currency = item.listing_price.currency || "CAD";
if (item.listing_price.amount && item.listing_price.amount !== "0.00") {
const amount = Number.parseFloat(item.listing_price.amount);
if (!Number.isNaN(amount)) {
cents = Math.round(amount * 100);
amountFormatted =
item.formatted_price?.text || formatCentsToCurrency(cents);
}
}
}
// Extract description
const description = item.redacted_description?.text;
// Extract location
const address = item.location_text?.text || null;
// Extract seller information
const seller = item.marketplace_listing_seller
? {
name: item.marketplace_listing_seller.name,
id: item.marketplace_listing_seller.id,
}
: undefined;
// Determine listing status
let listingStatus: string | undefined;
if (item.is_sold) {
listingStatus = "SOLD";
} else if (item.is_pending) {
listingStatus = "PENDING";
} else if (item.is_live) {
listingStatus = "ACTIVE";
} else if (item.is_hidden) {
listingStatus = "HIDDEN";
}
// Format creation date
const creationDate = item.creation_time
? new Date(item.creation_time * 1000).toISOString()
: undefined;
// Determine listing type based on category or vehicle data
let listingType = "item";
if (item.vehicle_make_display_name || item.vehicle_odometer_data) {
listingType = "vehicle";
} else if (item.marketplace_listing_category_id) {
// Could map category IDs to types, but keeping simple for now
listingType = "item";
}
const listingDetails: ListingDetails = {
url,
title,
description,
listingPrice: {
amountFormatted,
cents,
currency,
},
address,
creationDate,
listingType,
listingStatus,
categoryId: item.marketplace_listing_category_id,
seller,
deliveryTypes: item.delivery_types,
};
return listingDetails;
} catch (error) {
console.warn(`Failed to parse Facebook item ${item.id}:`, error);
return null;
}
}
// ----------------------------- Exports for Testing -----------------------------
// Export internal functions for comprehensive testing
export {
extractFacebookItemData,
extractFacebookMarketplaceData,
parseFacebookItem,
parseFacebookAds,
formatCentsToCurrency,
loadFacebookCookies,
formatCookiesForHeader,
parseFacebookCookieString,
ensureFacebookCookies,
};
// ----------------------------- Main -----------------------------
export default async function fetchFacebookItems(
@@ -510,9 +1012,18 @@ export default async function fetchFacebookItems(
LOCATION = "toronto",
MAX_ITEMS = 25,
cookiesSource?: string,
cookiePath?: string,
) {
// Load Facebook cookies - required for Facebook Marketplace access
const cookies = await loadFacebookCookies(cookiesSource);
let cookies: Cookie[];
if (cookiesSource) {
// Use provided cookie source (backward compatibility)
cookies = await loadFacebookCookies(cookiesSource);
} else {
// Auto-load from file or parse from env var
cookies = await ensureFacebookCookies(cookiePath);
}
if (cookies.length === 0) {
throw new Error(
"Facebook cookies are required for marketplace access. " +
@@ -546,8 +1057,7 @@ export default async function fetchFacebookItems(
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
"\n" +
`Facebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
},
@@ -581,7 +1091,7 @@ export default async function fetchFacebookItems(
cliProgress.Presets.shades_classic,
);
const totalProgress = ads.length;
let currentProgress = 0;
const currentProgress = 0;
progressBar.start(totalProgress, currentProgress);
const items = parseFacebookAds(ads);
@@ -597,3 +1107,158 @@ export default async function fetchFacebookItems(
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
return pricedItems.slice(0, MAX_ITEMS); // Limit results
}
/**
* Fetch individual Facebook marketplace item details with enhanced error handling
*/
export async function fetchFacebookItem(
itemId: string,
cookiesSource?: string,
cookiePath?: string,
): Promise<ListingDetails | null> {
// Load Facebook cookies - required for Facebook Marketplace access
let cookies: Cookie[];
if (cookiesSource) {
// Use provided cookie source (backward compatibility)
cookies = await loadFacebookCookies(cookiesSource);
} else {
// Auto-load from file or parse from env var
cookies = await ensureFacebookCookies(cookiePath);
}
if (cookies.length === 0) {
throw new Error(
"Facebook cookies are required for marketplace access. " +
"Please provide cookies via 'cookies' parameter or create ./cookies/facebook.json file with valid Facebook session cookies.",
);
}
// Format cookies for HTTP header
const domain = "www.facebook.com";
const cookiesHeader = formatCookiesForHeader(cookies, domain);
if (!cookiesHeader) {
throw new Error(
"No valid Facebook cookies found. Please check that cookies are not expired and apply to facebook.com domain.",
);
}
const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`;
console.log(`Fetching Facebook marketplace item: ${itemUrl}`);
let itemHtml: string;
try {
itemHtml = await fetchHtml(itemUrl, 1000, {
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
},
cookies: cookiesHeader,
});
} catch (err) {
if (err instanceof HttpError) {
console.warn(
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
);
// Enhanced error handling based on status codes
switch (err.status) {
case 400:
case 401:
case 403:
console.warn(
"Authentication error: Invalid or expired cookies. Please update ./cookies/facebook.json with fresh session cookies.",
);
console.warn(
"Try logging out and back into Facebook, then export fresh cookies.",
);
break;
case 404:
console.warn(
"Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.",
);
break;
case 429:
console.warn(
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
);
break;
case 500:
case 502:
case 503:
console.warn(
"Facebook server error: Marketplace may be temporarily unavailable.",
);
break;
default:
console.warn(`Unexpected error status: ${err.status}`);
}
return null;
}
throw err;
}
const itemData = extractFacebookItemData(itemHtml);
if (!itemData) {
logExtractionMetrics(false, itemId);
// Enhanced checking for specific failure scenarios
if (
itemHtml.includes("This listing is no longer available") ||
itemHtml.includes("listing has been removed") ||
itemHtml.includes("This item has been sold")
) {
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}
if (
itemHtml.includes("log in to Facebook") ||
itemHtml.includes("You must log in") ||
itemHtml.includes("authentication required")
) {
console.warn(
`Authentication failed for item ${itemId}. Cookies may be expired.`,
);
return null;
}
console.warn(
`No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`,
);
console.warn(" - The listing was removed or sold");
console.warn(" - Authentication issues");
console.warn(" - Facebook changed their API structure");
console.warn(" - Network or parsing issues");
return null;
}
logExtractionMetrics(true, itemId);
console.log(`Successfully extracted data for item ${itemId}`);
const parsedItem = parseFacebookItem(itemData);
if (!parsedItem) {
console.warn(`Failed to parse item ${itemId}: Invalid data structure`);
return null;
}
// Check for sold/removed status in the parsed data with proper precedence
if (itemData.is_sold) {
console.warn(`Item ${itemId} is marked as sold in the marketplace.`);
// Still return the data but mark it as sold
parsedItem.listingStatus = "SOLD";
} else if (!itemData.is_live) {
console.warn(`Item ${itemId} is not live/active in the marketplace.`);
parsedItem.listingStatus = itemData.is_hidden
? "HIDDEN"
: itemData.is_pending
? "PENDING"
: "INACTIVE";
}
return parsedItem;
}

View File

@@ -1,6 +1,6 @@
import fetchKijijiItems from "@/kijiji";
import fetchFacebookItems from "@/facebook";
import fetchEbayItems from "@/ebay";
import fetchFacebookItems from "@/facebook";
import fetchKijijiItems from "@/kijiji";
const PORT = process.env.PORT || 4005;
@@ -30,34 +30,54 @@ const server = Bun.serve({
const location = reqUrl.searchParams.get("location");
const category = reqUrl.searchParams.get("category");
const maxPagesParam = reqUrl.searchParams.get("maxPages");
const maxPages = maxPagesParam
? Number.parseInt(maxPagesParam, 10)
: 5; // Default: 5 pages
const sortBy = reqUrl.searchParams.get("sortBy") as 'relevancy' | 'date' | 'price' | 'distance' | undefined;
const sortOrder = reqUrl.searchParams.get("sortOrder") as 'asc' | 'desc' | undefined;
const maxPages = maxPagesParam ? Number.parseInt(maxPagesParam, 10) : 5; // Default: 5 pages
const sortBy = reqUrl.searchParams.get("sortBy") as
| "relevancy"
| "date"
| "price"
| "distance"
| undefined;
const sortOrder = reqUrl.searchParams.get("sortOrder") as
| "asc"
| "desc"
| undefined;
// Build search options
const locationValue = location ? (/^\d+$/.test(location) ? Number(location) : location) : 1700272;
const categoryValue = category ? (/^\d+$/.test(category) ? Number(category) : category) : 0;
const locationValue = location
? /^\d+$/.test(location)
? Number(location)
: location
: 1700272;
const categoryValue = category
? /^\d+$/.test(category)
? Number(category)
: category
: 0;
const searchOptions: import("@/kijiji").SearchOptions = {
location: locationValue,
category: categoryValue,
keywords: SEARCH_QUERY,
sortBy: sortBy || 'relevancy',
sortOrder: sortOrder || 'desc',
sortBy: sortBy || "relevancy",
sortOrder: sortOrder || "desc",
maxPages,
};
// Build listing fetch options with enhanced defaults
const listingOptions: import("@/kijiji").ListingFetchOptions = {
includeImages: true, // Always include full image arrays
sellerDataDepth: 'detailed', // Default: detailed seller info
sellerDataDepth: "detailed", // Default: detailed seller info
includeClientSideData: false, // GraphQL reviews disabled by default
};
try {
const items = await fetchKijijiItems(SEARCH_QUERY, 1, undefined, searchOptions, listingOptions);
const items = await fetchKijijiItems(
SEARCH_QUERY,
1,
undefined,
searchOptions,
listingOptions,
);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
@@ -66,12 +86,13 @@ const server = Bun.serve({
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Kijiji scraping error:", error);
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json(
{
message: `Scraping failed: ${errorMessage}`,
query: SEARCH_QUERY,
options: { searchOptions, listingOptions }
options: { searchOptions, listingOptions },
},
{ status: 500 },
);
@@ -96,7 +117,14 @@ const server = Bun.serve({
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
try {
const items = await fetchFacebookItems(SEARCH_QUERY, 5, LOCATION, 25, COOKIES_SOURCE);
const items = await fetchFacebookItems(
SEARCH_QUERY,
5,
LOCATION,
25,
COOKIES_SOURCE,
"./cookies/facebook.json",
);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
@@ -105,11 +133,9 @@ const server = Bun.serve({
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Facebook scraping error:", error);
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
return Response.json(
{ message: errorMessage },
{ status: 400 },
);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
},
@@ -138,9 +164,13 @@ const server = Bun.serve({
: undefined;
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
const exclusionsParam = reqUrl.searchParams.get("exclusions");
const exclusions = exclusionsParam ? exclusionsParam.split(",").map(s => s.trim()) : [];
const exclusions = exclusionsParam
? exclusionsParam.split(",").map((s) => s.trim())
: [];
const keywordsParam = reqUrl.searchParams.get("keywords");
const keywords = keywordsParam ? keywordsParam.split(",").map(s => s.trim()) : [SEARCH_QUERY];
const keywords = keywordsParam
? keywordsParam.split(",").map((s) => s.trim())
: [SEARCH_QUERY];
try {
const items = await fetchEbayItems(SEARCH_QUERY, 5, {
@@ -158,11 +188,9 @@ const server = Bun.serve({
return Response.json(items, { status: 200 });
} catch (error) {
console.error("eBay scraping error:", error);
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
return Response.json(
{ message: errorMessage },
{ status: 400 },
);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
},

View File

@@ -1,7 +1,7 @@
import cliProgress from "cli-progress";
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom";
import unidecode from "unidecode";
import cliProgress from "cli-progress";
// const unidecode = require("unidecode");
@@ -99,8 +99,8 @@ interface SearchOptions {
location?: number | string; // Location ID or name
category?: number | string; // Category ID or name
keywords?: string;
sortBy?: 'relevancy' | 'date' | 'price' | 'distance';
sortOrder?: 'desc' | 'asc';
sortBy?: "relevancy" | "date" | "price" | "distance";
sortOrder?: "desc" | "asc";
maxPages?: number; // Default: 5
priceMin?: number;
priceMax?: number;
@@ -108,7 +108,7 @@ interface SearchOptions {
interface ListingFetchOptions {
includeImages?: boolean; // Default: true
sellerDataDepth?: 'basic' | 'detailed' | 'full'; // Default: 'detailed'
sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed'
includeClientSideData?: boolean; // Default: false
}
@@ -116,65 +116,65 @@ interface ListingFetchOptions {
// Location mappings from KIJIJI.md
const LOCATION_MAPPINGS: Record<string, number> = {
'canada': 0,
'ontario': 9004,
'toronto': 1700273,
'gta': 1700272,
'oshawa': 1700275,
'quebec': 9001,
'nova scotia': 9002,
'alberta': 9003,
'new brunswick': 9005,
'manitoba': 9006,
'british columbia': 9007,
'newfoundland': 9008,
'saskatchewan': 9009,
'territories': 9010,
'pei': 9011,
'prince edward island': 9011,
canada: 0,
ontario: 9004,
toronto: 1700273,
gta: 1700272,
oshawa: 1700275,
quebec: 9001,
"nova scotia": 9002,
alberta: 9003,
"new brunswick": 9005,
manitoba: 9006,
"british columbia": 9007,
newfoundland: 9008,
saskatchewan: 9009,
territories: 9010,
pei: 9011,
"prince edward island": 9011,
};
// Category mappings from KIJIJI.md (Buy & Sell main categories)
const CATEGORY_MAPPINGS: Record<string, number> = {
'all': 0,
'buy-sell': 10,
'arts-collectibles': 12,
'audio': 767,
'baby-items': 253,
'bags-luggage': 931,
'bikes': 644,
'books': 109,
'cameras': 103,
'cds': 104,
'clothing': 274,
'computers': 16,
'computer-accessories': 128,
'electronics': 29659001,
'free-stuff': 17220001,
'furniture': 235,
'garage-sales': 638,
'health-special-needs': 140,
'hobbies-crafts': 139,
'home-appliances': 107,
'home-indoor': 717,
'home-outdoor': 727,
'jewellery': 133,
'musical-instruments': 17,
'phones': 132,
'sporting-goods': 111,
'tools': 110,
'toys-games': 108,
'tvs-video': 15093001,
'video-games': 141,
'other': 26,
all: 0,
"buy-sell": 10,
"arts-collectibles": 12,
audio: 767,
"baby-items": 253,
"bags-luggage": 931,
bikes: 644,
books: 109,
cameras: 103,
cds: 104,
clothing: 274,
computers: 16,
"computer-accessories": 128,
electronics: 29659001,
"free-stuff": 17220001,
furniture: 235,
"garage-sales": 638,
"health-special-needs": 140,
"hobbies-crafts": 139,
"home-appliances": 107,
"home-indoor": 717,
"home-outdoor": 727,
jewellery: 133,
"musical-instruments": 17,
phones: 132,
"sporting-goods": 111,
tools: 110,
"toys-games": 108,
"tvs-video": 15093001,
"video-games": 141,
other: 26,
};
// Sort parameter mappings
const SORT_MAPPINGS: Record<string, string> = {
'relevancy': 'MATCH',
'date': 'DATE',
'price': 'PRICE',
'distance': 'DISTANCE',
relevancy: "MATCH",
date: "DATE",
price: "PRICE",
distance: "DISTANCE",
};
// ----------------------------- Exports for Testing -----------------------------
@@ -193,9 +193,9 @@ const SEPS = new Set([" ", "", "—", "/", ":", ";", ",", ".", "-"]);
* Resolve location ID from name or return numeric ID
*/
function resolveLocationId(location?: number | string): number {
if (typeof location === 'number') return location;
if (typeof location === 'string') {
const normalized = location.toLowerCase().replace(/\s+/g, '-');
if (typeof location === "number") return location;
if (typeof location === "string") {
const normalized = location.toLowerCase().replace(/\s+/g, "-");
return LOCATION_MAPPINGS[normalized] ?? 0; // Default to Canada (0)
}
return 0; // Default to Canada
@@ -205,9 +205,9 @@ function resolveLocationId(location?: number | string): number {
* Resolve category ID from name or return numeric ID
*/
function resolveCategoryId(category?: number | string): number {
if (typeof category === 'number') return category;
if (typeof category === 'string') {
const normalized = category.toLowerCase().replace(/\s+/g, '-');
if (typeof category === "number") return category;
if (typeof category === "string") {
const normalized = category.toLowerCase().replace(/\s+/g, "-");
return CATEGORY_MAPPINGS[normalized] ?? 0; // Default to all categories
}
return 0; // Default to all categories
@@ -219,19 +219,22 @@ function resolveCategoryId(category?: number | string): number {
function buildSearchUrl(
keywords: string,
options: SearchOptions & { page?: number },
BASE_URL = "https://www.kijiji.ca"
BASE_URL = "https://www.kijiji.ca",
): string {
const locationId = resolveLocationId(options.location);
const categoryId = resolveCategoryId(options.category);
const categorySlug = categoryId === 0 ? 'buy-sell' : 'buy-sell'; // Could be enhanced
const locationSlug = locationId === 0 ? 'canada' : 'canada'; // Could be enhanced
const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell"; // Could be enhanced
const locationSlug = locationId === 0 ? "canada" : "canada"; // Could be enhanced
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
const sortParam = options.sortBy ? `&sort=${SORT_MAPPINGS[options.sortBy]}` : '';
const sortOrder = options.sortOrder === 'asc' ? 'ASC' : 'DESC';
const pageParam = options.page && options.page > 1 ? `&page=${options.page}` : '';
const sortParam = options.sortBy
? `&sort=${SORT_MAPPINGS[options.sortBy]}`
: "";
const sortOrder = options.sortOrder === "asc" ? "ASC" : "DESC";
const pageParam =
options.page && options.page > 1 ? `&page=${options.page}` : "";
url += `?sort=relevancyDesc&view=list${sortParam}&order=${sortOrder}${pageParam}`;
@@ -278,8 +281,8 @@ export function formatCentsToCurrency(
if (Number.isNaN(cents)) return "";
const dollars = cents / 100;
const formatter = new Intl.NumberFormat(locale, {
style: 'currency',
currency: 'USD',
style: "currency",
currency: "USD",
minimumFractionDigits: 2,
maximumFractionDigits: 2,
});
@@ -394,7 +397,9 @@ async function fetchHtml(
if (!res.ok) {
// Handle rate limiting
if (res.status === 429) {
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : Number.NaN;
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: calculateBackoffDelay(attempt, retryBaseMs);
@@ -428,14 +433,13 @@ async function fetchHtml(
// Respect per-request delay to maintain rate limiting
await delay(DELAY_MS);
return html;
} catch (err) {
// Handle different error types
if (err instanceof RateLimitError || err instanceof HttpError) {
throw err; // Re-throw known errors
}
if (err instanceof Error && err.name === 'AbortError') {
if (err instanceof Error && err.name === "AbortError") {
if (attempt < maxRetries) {
await delay(calculateBackoffDelay(attempt, retryBaseMs));
continue;
@@ -451,7 +455,7 @@ async function fetchHtml(
throw new NetworkError(
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
url,
err instanceof Error ? err : undefined
err instanceof Error ? err : undefined,
);
}
}
@@ -463,7 +467,7 @@ async function fetchHtml(
* Calculate exponential backoff delay with jitter
*/
function calculateBackoffDelay(attempt: number, baseMs: number): number {
const exponentialDelay = baseMs * (2 ** attempt);
const exponentialDelay = baseMs * 2 ** attempt;
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
}
@@ -476,16 +480,16 @@ function calculateBackoffDelay(attempt: number, baseMs: number): number {
async function fetchGraphQLData(
query: string,
variables: Record<string, unknown>,
BASE_URL = "https://www.kijiji.ca"
BASE_URL = "https://www.kijiji.ca",
): Promise<unknown> {
const endpoint = `${BASE_URL}/anvil/api`;
try {
const response = await fetch(endpoint, {
method: 'POST',
method: "POST",
headers: {
'Content-Type': 'application/json',
'apollo-require-preflight': 'true',
"Content-Type": "application/json",
"apollo-require-preflight": "true",
},
body: JSON.stringify({
query,
@@ -497,14 +501,17 @@ async function fetchGraphQLData(
throw new HttpError(
`GraphQL request failed with status ${response.status}`,
response.status,
endpoint
endpoint,
);
}
const result = await response.json();
if (result.errors) {
throw new ParseError(`GraphQL errors: ${JSON.stringify(result.errors)}`, result.errors);
throw new ParseError(
`GraphQL errors: ${JSON.stringify(result.errors)}`,
result.errors,
);
}
return result.data;
@@ -515,7 +522,7 @@ async function fetchGraphQLData(
throw new NetworkError(
`Failed to fetch GraphQL data: ${err instanceof Error ? err.message : String(err)}`,
endpoint,
err instanceof Error ? err : undefined
err instanceof Error ? err : undefined,
);
}
}
@@ -567,12 +574,25 @@ const GRAPHQL_QUERIES = {
*/
async function fetchSellerDetails(
posterId: string,
BASE_URL = "https://www.kijiji.ca"
): Promise<{ reviewCount?: number; reviewScore?: number; memberSince?: string; accountType?: string }> {
BASE_URL = "https://www.kijiji.ca",
): Promise<{
reviewCount?: number;
reviewScore?: number;
memberSince?: string;
accountType?: string;
}> {
try {
const [reviewData, profileData] = await Promise.all([
fetchGraphQLData(GRAPHQL_QUERIES.getReviewSummary, { userId: posterId }, BASE_URL),
fetchGraphQLData(GRAPHQL_QUERIES.getProfileMetrics, { profileId: posterId }, BASE_URL),
fetchGraphQLData(
GRAPHQL_QUERIES.getReviewSummary,
{ userId: posterId },
BASE_URL,
),
fetchGraphQLData(
GRAPHQL_QUERIES.getProfileMetrics,
{ profileId: posterId },
BASE_URL,
),
]);
const reviewResponse = reviewData as GraphQLReviewResponse;
@@ -586,7 +606,10 @@ async function fetchSellerDetails(
};
} catch (err) {
// Silently fail for GraphQL errors - not critical for basic functionality
console.warn(`Failed to fetch seller details for ${posterId}:`, err instanceof Error ? err.message : String(err));
console.warn(
`Failed to fetch seller details for ${posterId}:`,
err instanceof Error ? err.message : String(err),
);
return {};
}
}
@@ -694,7 +717,8 @@ function parseListing(
listingPrice: amountFormatted
? {
amountFormatted,
cents: cents !== undefined && Number.isFinite(cents) ? cents : undefined,
cents:
cents !== undefined && Number.isFinite(cents) ? cents : undefined,
currency: price?.currency,
}
: undefined,
@@ -702,7 +726,10 @@ function parseListing(
listingStatus: status,
creationDate: activationDate,
endDate,
numberOfViews: numberOfViews !== undefined && Number.isFinite(numberOfViews) ? numberOfViews : undefined,
numberOfViews:
numberOfViews !== undefined && Number.isFinite(numberOfViews)
? numberOfViews
: undefined,
address: location?.address ?? null,
};
}
@@ -713,7 +740,7 @@ function parseListing(
async function parseDetailedListing(
htmlString: HTMLString,
BASE_URL: string,
options: ListingFetchOptions = {}
options: ListingFetchOptions = {},
): Promise<DetailedListing | null> {
const apolloState = extractApolloState(htmlString);
if (!apolloState) return null;
@@ -766,8 +793,9 @@ async function parseDetailedListing(
if (!amountFormatted || cents === undefined) return null;
// Extract images if requested
const images = options.includeImages !== false && Array.isArray(imageUrls)
? imageUrls.filter((url): url is string => typeof url === 'string')
const images =
options.includeImages !== false && Array.isArray(imageUrls)
? imageUrls.filter((url): url is string => typeof url === "string")
: [];
// Extract attributes as key-value pairs
@@ -781,26 +809,35 @@ async function parseDetailedListing(
}
// Extract seller info based on depth setting
let sellerInfo: DetailedListing['sellerInfo'];
const depth = options.sellerDataDepth ?? 'detailed';
let sellerInfo: DetailedListing["sellerInfo"];
const depth = options.sellerDataDepth ?? "detailed";
if (posterInfo?.posterId) {
sellerInfo = {
posterId: posterInfo.posterId,
rating: typeof posterInfo.rating === 'number' ? posterInfo.rating : undefined,
rating:
typeof posterInfo.rating === "number" ? posterInfo.rating : undefined,
};
// Add more detailed info if requested and client-side data is enabled
if ((depth === 'detailed' || depth === 'full') && options.includeClientSideData) {
if (
(depth === "detailed" || depth === "full") &&
options.includeClientSideData
) {
try {
const additionalData = await fetchSellerDetails(posterInfo.posterId, BASE_URL);
const additionalData = await fetchSellerDetails(
posterInfo.posterId,
BASE_URL,
);
sellerInfo = {
...sellerInfo,
...additionalData,
};
} catch (err) {
// Silently fail - GraphQL data is optional
console.warn(`Failed to fetch additional seller data for ${posterInfo.posterId}`);
console.warn(
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
);
}
}
}
@@ -818,23 +855,28 @@ async function parseDetailedListing(
listingStatus: status,
creationDate: activationDate,
endDate,
numberOfViews: numberOfViews !== undefined && Number.isFinite(numberOfViews) ? numberOfViews : undefined,
numberOfViews:
numberOfViews !== undefined && Number.isFinite(numberOfViews)
? numberOfViews
: undefined,
address: location?.address ?? null,
images,
categoryId: typeof categoryId === 'number' ? categoryId : 0,
adSource: typeof adSource === 'string' ? adSource : 'UNKNOWN',
categoryId: typeof categoryId === "number" ? categoryId : 0,
adSource: typeof adSource === "string" ? adSource : "UNKNOWN",
flags: {
topAd: flags?.topAd === true,
priceDrop: flags?.priceDrop === true,
},
attributes: attributeMap,
location: {
id: typeof location?.id === 'number' ? location.id : 0,
name: typeof location?.name === 'string' ? location.name : 'Unknown',
coordinates: location?.coordinates ? {
id: typeof location?.id === "number" ? location.id : 0,
name: typeof location?.name === "string" ? location.name : "Unknown",
coordinates: location?.coordinates
? {
latitude: location.coordinates.latitude,
longitude: location.coordinates.longitude,
} : undefined,
}
: undefined,
},
sellerInfo,
};
@@ -856,8 +898,8 @@ export default async function fetchKijijiItems(
location: searchOptions.location ?? 1700272, // Default to GTA
category: searchOptions.category ?? 0, // Default to all categories
keywords: searchOptions.keywords ?? SEARCH_QUERY,
sortBy: searchOptions.sortBy ?? 'relevancy',
sortOrder: searchOptions.sortOrder ?? 'desc',
sortBy: searchOptions.sortBy ?? "relevancy",
sortOrder: searchOptions.sortOrder ?? "desc",
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
priceMin: searchOptions.priceMin,
priceMax: searchOptions.priceMax,
@@ -865,7 +907,7 @@ export default async function fetchKijijiItems(
const finalListingOptions: Required<ListingFetchOptions> = {
includeImages: listingOptions.includeImages ?? true,
sellerDataDepth: listingOptions.sellerDataDepth ?? 'detailed',
sellerDataDepth: listingOptions.sellerDataDepth ?? "detailed",
includeClientSideData: listingOptions.includeClientSideData ?? false,
};
@@ -874,24 +916,32 @@ export default async function fetchKijijiItems(
// Fetch multiple pages
for (let page = 1; page <= finalSearchOptions.maxPages; page++) {
const searchUrl = buildSearchUrl(finalSearchOptions.keywords, {
const searchUrl = buildSearchUrl(
finalSearchOptions.keywords,
{
...finalSearchOptions,
// Add page parameter for pagination
...(page > 1 && { page }),
}, BASE_URL);
},
BASE_URL,
);
console.log(`Fetching search page ${page}: ${searchUrl}`);
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`);
console.log(
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
},
});
const searchResults = parseSearch(searchHtml, BASE_URL);
if (searchResults.length === 0) {
console.log(`No more results found on page ${page}. Stopping pagination.`);
console.log(
`No more results found on page ${page}. Stopping pagination.`,
);
break;
}
@@ -904,7 +954,9 @@ export default async function fetchKijijiItems(
seenUrls.add(link);
}
console.log(`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`);
console.log(
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
);
// Fetch details for this page's listings
const progressBar = new cliProgress.SingleBar(
@@ -920,19 +972,29 @@ export default async function fetchKijijiItems(
const html = await fetchHtml(link, DELAY_MS, {
onRateInfo: (remaining, reset) => {
if (remaining && reset) {
console.log(`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`);
console.log(
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
);
}
},
});
const parsed = await parseDetailedListing(html, BASE_URL, finalListingOptions);
const parsed = await parseDetailedListing(
html,
BASE_URL,
finalListingOptions,
);
if (parsed) {
allListings.push(parsed);
}
} catch (err) {
if (err instanceof HttpError) {
console.error(`\nFailed to fetch ${link}\n - ${err.status} ${err.message}`);
console.error(
`\nFailed to fetch ${link}\n - ${err.status} ${err.message}`,
);
} else {
console.error(`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`);
console.error(
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
);
}
} finally {
currentProgress++;

834
test/facebook-core.test.ts Normal file
View File

@@ -0,0 +1,834 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import {
extractFacebookItemData,
extractFacebookMarketplaceData,
fetchFacebookItem,
formatCentsToCurrency,
formatCookiesForHeader,
loadFacebookCookies,
parseFacebookAds,
parseFacebookCookieString,
parseFacebookItem,
} from "../src/facebook";
// Mock fetch globally
const originalFetch = global.fetch;
describe("Facebook Marketplace Scraper Core Tests", () => {
beforeEach(() => {
global.fetch = mock(() => {
throw new Error("fetch should be mocked in individual tests");
});
});
afterEach(() => {
global.fetch = originalFetch;
});
describe("Cookie Parsing", () => {
describe("parseFacebookCookieString", () => {
test("should parse valid cookie string", () => {
const cookieString = "c_user=123456789; xs=abcdef123456; fr=xyz789";
const result = parseFacebookCookieString(cookieString);
expect(result).toHaveLength(3);
expect(result[0]).toEqual({
name: "c_user",
value: "123456789",
domain: ".facebook.com",
path: "/",
secure: true,
httpOnly: false,
sameSite: "lax",
expirationDate: undefined,
});
expect(result[1]).toEqual({
name: "xs",
value: "abcdef123456",
domain: ".facebook.com",
path: "/",
secure: true,
httpOnly: false,
sameSite: "lax",
expirationDate: undefined,
});
});
test("should handle URL-encoded values", () => {
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
const result = parseFacebookCookieString(cookieString);
expect(result[0].value).toBe("123+456");
expect(result[1].value).toBe("abc=def");
});
test("should filter out malformed cookies", () => {
const cookieString = "c_user=123; invalid; xs=abc; =empty";
const result = parseFacebookCookieString(cookieString);
expect(result).toHaveLength(2);
expect(result.map((c) => c.name)).toEqual(["c_user", "xs"]);
});
test("should handle empty input", () => {
expect(parseFacebookCookieString("")).toEqual([]);
expect(parseFacebookCookieString(" ")).toEqual([]);
});
test("should handle extra whitespace", () => {
const cookieString = " c_user = 123 ; xs=abc ";
const result = parseFacebookCookieString(cookieString);
expect(result).toHaveLength(2);
expect(result[0].name).toBe("c_user");
expect(result[0].value).toBe("123");
expect(result[1].name).toBe("xs");
expect(result[1].value).toBe("abc");
});
});
});
describe("Facebook Item Fetching", () => {
describe("fetchFacebookItem", () => {
const mockCookies = JSON.stringify([
{ name: "c_user", value: "12345", domain: ".facebook.com" },
{ name: "xs", value: "abc123", domain: ".facebook.com" },
]);
test("should handle authentication errors", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: false,
status: 401,
text: () => Promise.resolve("Authentication required"),
headers: {
get: () => null,
},
}),
);
const result = await fetchFacebookItem("123", mockCookies);
expect(result).toBeNull();
});
test("should handle item not found", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: false,
status: 404,
text: () => Promise.resolve("Not found"),
headers: {
get: () => null,
},
}),
);
const result = await fetchFacebookItem("nonexistent", mockCookies);
expect(result).toBeNull();
});
test("should handle rate limiting", async () => {
let attempts = 0;
global.fetch = mock(() => {
attempts++;
if (attempts === 1) {
return Promise.resolve({
ok: false,
status: 429,
headers: {
get: (header: string) => {
if (header === "X-RateLimit-Reset") return "1";
return null;
},
},
text: () => Promise.resolve("Rate limited"),
});
}
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
viewer: {
marketplace_product_details_page: {
target: {
id: "123",
__typename: "GroupCommerceProductItem",
marketplace_listing_title: "Test Item",
is_live: true,
},
},
},
},
},
},
},
],
],
};
return Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
),
headers: {
get: () => null,
},
});
});
const result = await fetchFacebookItem("123", mockCookies);
expect(attempts).toBe(2);
// Should eventually succeed after retry
});
test("should handle sold items", async () => {
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
viewer: {
marketplace_product_details_page: {
target: {
id: "456",
__typename: "GroupCommerceProductItem",
marketplace_listing_title: "Sold Item",
is_sold: true,
is_live: false,
},
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const result = await fetchFacebookItem("456", mockCookies);
expect(result?.listingStatus).toBe("SOLD");
});
test("should handle missing authentication cookies", async () => {
// Use a test-specific cookie file that doesn't exist
const testCookiePath = "./cookies/facebook-test.json";
// Test with no cookies available (test file doesn't exist)
await expect(
fetchFacebookItem("123", undefined, testCookiePath),
).rejects.toThrow("No valid Facebook cookies found");
});
test("should handle successful item extraction", async () => {
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
viewer: {
marketplace_product_details_page: {
target: {
id: "789",
__typename: "GroupCommerceProductItem",
marketplace_listing_title: "Working Item",
formatted_price: { text: "$299.00" },
listing_price: {
amount: "299.00",
currency: "CAD",
},
is_live: true,
creation_time: 1640995200,
},
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const result = await fetchFacebookItem("789", mockCookies);
expect(result).not.toBeNull();
expect(result?.title).toBe("Working Item");
expect(result?.listingPrice?.amountFormatted).toBe("$299.00");
expect(result?.listingStatus).toBe("ACTIVE");
});
test("should handle server errors", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: false,
status: 500,
text: () => Promise.resolve("Internal Server Error"),
headers: {
get: () => null,
},
}),
);
const result = await fetchFacebookItem("error", mockCookies);
expect(result).toBeNull();
});
});
});
describe("Data Extraction", () => {
describe("extractFacebookItemData", () => {
test("should extract item data from standard require structure", () => {
const mockItemData = {
id: "123456",
__typename: "GroupCommerceProductItem",
marketplace_listing_title: "Test Item",
formatted_price: { text: "$100.00" },
listing_price: { amount: "100.00", currency: "CAD" },
is_live: true,
};
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
viewer: {
marketplace_product_details_page: {
target: mockItemData,
},
},
},
},
},
},
],
],
};
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
const result = extractFacebookItemData(html);
expect(result).not.toBeNull();
expect(result?.id).toBe("123456");
expect(result?.marketplace_listing_title).toBe("Test Item");
});
test("should handle missing item data", () => {
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
viewer: {
marketplace_product_details_page: {},
},
},
},
},
},
],
],
};
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
const result = extractFacebookItemData(html);
expect(result).toBeNull();
});
test("should handle malformed HTML", () => {
const result = extractFacebookItemData(
"<html><body>Invalid HTML</body></html>",
);
expect(result).toBeNull();
});
test("should handle invalid JSON in script tags", () => {
const html =
"<html><body><script>{invalid: json}</script></body></html>";
const result = extractFacebookItemData(html);
expect(result).toBeNull();
});
test("should extract item with vehicle data", () => {
const mockVehicleItem = {
id: "789",
__typename: "GroupCommerceProductItem",
marketplace_listing_title: "2006 Honda Civic",
formatted_price: { text: "$5,000" },
listing_price: { amount: "5000.00", currency: "CAD" },
vehicle_make_display_name: "Honda",
vehicle_model_display_name: "Civic",
vehicle_odometer_data: { unit: "KILOMETERS", value: 150000 },
vehicle_transmission_type: "AUTOMATIC",
is_live: true,
};
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
viewer: {
marketplace_product_details_page: {
target: mockVehicleItem,
},
},
},
},
},
},
],
],
};
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
const result = extractFacebookItemData(html);
expect(result).not.toBeNull();
expect(result?.vehicle_make_display_name).toBe("Honda");
expect(result?.vehicle_odometer_data?.value).toBe(150000);
});
});
describe("extractFacebookMarketplaceData", () => {
test("should extract search results from marketplace data", () => {
const mockMarketplaceData = {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Item 1",
listing_price: { amount: "10.00", currency: "CAD" },
},
},
},
{
node: {
listing: {
id: "2",
marketplace_listing_title: "Item 2",
listing_price: { amount: "20.00", currency: "CAD" },
},
},
},
],
},
};
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: mockMarketplaceData,
},
},
},
},
],
],
};
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
const result = extractFacebookMarketplaceData(html);
expect(result).not.toBeNull();
expect(result).toHaveLength(2);
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
"Item 1",
);
});
test("should handle empty search results", () => {
const mockData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: { edges: [] },
},
},
},
},
},
],
],
};
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
const result = extractFacebookMarketplaceData(html);
expect(result).toBeNull();
});
});
});
describe("Data Parsing", () => {
describe("parseFacebookItem", () => {
test("should parse complete item with all fields", () => {
const item = {
id: "123456",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "iPhone 13 Pro",
redacted_description: { text: "Excellent condition" },
formatted_price: { text: "$800.00" },
listing_price: { amount: "800.00", currency: "CAD" },
location_text: { text: "Toronto, ON" },
is_live: true,
creation_time: 1640995200,
marketplace_listing_seller: {
id: "seller1",
name: "John Doe",
},
delivery_types: ["IN_PERSON"],
};
const result = parseFacebookItem(item);
expect(result).not.toBeNull();
expect(result?.title).toBe("iPhone 13 Pro");
expect(result?.description).toBe("Excellent condition");
expect(result?.listingPrice?.amountFormatted).toBe("$800.00");
expect(result?.listingPrice?.cents).toBe(80000);
expect(result?.listingPrice?.currency).toBe("CAD");
expect(result?.address).toBe("Toronto, ON");
expect(result?.listingStatus).toBe("ACTIVE");
expect(result?.seller?.name).toBe("John Doe");
expect(result?.deliveryTypes).toEqual(["IN_PERSON"]);
});
test("should parse FREE items", () => {
const item = {
id: "789",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "Free Sofa",
formatted_price: { text: "FREE" },
listing_price: { amount: "0.00", currency: "CAD" },
is_live: true,
};
const result = parseFacebookItem(item);
expect(result).not.toBeNull();
expect(result?.title).toBe("Free Sofa");
expect(result?.listingPrice?.amountFormatted).toBe("FREE");
expect(result?.listingPrice?.cents).toBe(0);
});
test("should handle missing optional fields", () => {
const item = {
id: "456",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "Minimal Item",
};
const result = parseFacebookItem(item);
expect(result).not.toBeNull();
expect(result?.title).toBe("Minimal Item");
expect(result?.description).toBeUndefined();
expect(result?.seller).toBeUndefined();
});
test("should identify vehicle listings", () => {
const vehicleItem = {
id: "999",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "2012 Mazda 3",
formatted_price: { text: "$8,000" },
listing_price: { amount: "8000.00", currency: "CAD" },
vehicle_make_display_name: "Mazda",
vehicle_model_display_name: "3",
is_live: true,
};
const result = parseFacebookItem(vehicleItem);
expect(result?.listingType).toBe("vehicle");
});
test("should handle different listing statuses", () => {
const soldItem = {
id: "111",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "Sold Item",
is_sold: true,
is_live: false,
};
const pendingItem = {
id: "222",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "Pending Item",
is_pending: true,
is_live: true,
};
const hiddenItem = {
id: "333",
__typename: "GroupCommerceProductItem" as const,
marketplace_listing_title: "Hidden Item",
is_hidden: true,
is_live: false,
};
expect(parseFacebookItem(soldItem)?.listingStatus).toBe("SOLD");
expect(parseFacebookItem(pendingItem)?.listingStatus).toBe("PENDING");
expect(parseFacebookItem(hiddenItem)?.listingStatus).toBe("HIDDEN");
});
test("should return null for items without title", () => {
const invalidItem = {
id: "invalid",
__typename: "GroupCommerceProductItem" as const,
is_live: true,
};
const result = parseFacebookItem(invalidItem);
expect(result).toBeNull();
});
});
describe("parseFacebookAds", () => {
test("should parse search result ads", () => {
const ads = [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Ad 1",
listing_price: {
amount: "50.00",
formatted_amount: "$50.00",
currency: "CAD",
},
location: {
reverse_geocode: { city_page: { display_name: "Toronto" } },
},
creation_time: 1640995200,
is_live: true,
},
},
},
{
node: {
listing: {
id: "2",
marketplace_listing_title: "Ad 2",
listing_price: {
amount: "75.00",
formatted_amount: "$75.00",
currency: "CAD",
},
location: {
reverse_geocode: { city_page: { display_name: "Ottawa" } },
},
creation_time: 1640995300,
is_live: true,
},
},
},
];
const results = parseFacebookAds(ads);
expect(results).toHaveLength(2);
expect(results[0].title).toBe("Ad 1");
expect(results[0].listingPrice?.cents).toBe(5000);
expect(results[0].address).toBe("Toronto");
expect(results[1].title).toBe("Ad 2");
expect(results[1].address).toBe("Ottawa");
});
test("should filter out ads without price", () => {
const ads = [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "With Price",
listing_price: {
amount: "100.00",
formatted_amount: "$100.00",
currency: "CAD",
},
is_live: true,
},
},
},
{
node: {
listing: {
id: "2",
marketplace_listing_title: "No Price",
is_live: true,
},
},
},
];
const results = parseFacebookAds(ads);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("With Price");
});
test("should handle malformed ads gracefully", () => {
const ads = [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Valid Ad",
listing_price: {
amount: "50.00",
formatted_amount: "$50.00",
currency: "CAD",
},
is_live: true,
},
},
},
{
node: {
// Missing listing
},
} as { node: { listing?: unknown } },
];
const results = parseFacebookAds(ads);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Valid Ad");
});
});
});
describe("Utility Functions", () => {
describe("formatCentsToCurrency", () => {
test("should format cents to currency string", () => {
expect(formatCentsToCurrency(100)).toBe("$1.00");
expect(formatCentsToCurrency(1000)).toBe("$10.00");
expect(formatCentsToCurrency(9999)).toBe("$99.99");
expect(formatCentsToCurrency(123456)).toBe("$1,234.56");
});
test("should handle string inputs", () => {
expect(formatCentsToCurrency("100")).toBe("$1.00");
expect(formatCentsToCurrency("1000")).toBe("$10.00");
});
test("should handle zero", () => {
expect(formatCentsToCurrency(0)).toBe("$0.00");
});
test("should handle null and undefined", () => {
expect(formatCentsToCurrency(null)).toBe("");
expect(formatCentsToCurrency(undefined)).toBe("");
});
test("should handle invalid inputs", () => {
expect(formatCentsToCurrency("invalid")).toBe("");
expect(formatCentsToCurrency(Number.NaN)).toBe("");
});
});
describe("formatCookiesForHeader", () => {
const mockCookies = [
{ name: "c_user", value: "123456", domain: ".facebook.com", path: "/" },
{ name: "xs", value: "abcdef", domain: ".facebook.com", path: "/" },
{ name: "session_id", value: "xyz", domain: "other.com", path: "/" },
];
test("should format cookies for header string", () => {
const result = formatCookiesForHeader(mockCookies, "www.facebook.com");
expect(result).toBe("c_user=123456; xs=abcdef");
});
test("should filter expired cookies", () => {
const cookiesWithExpiration = [
...mockCookies,
{
name: "expired",
value: "old",
domain: ".facebook.com",
path: "/",
expirationDate: Date.now() / 1000 - 1000,
},
];
const result = formatCookiesForHeader(
cookiesWithExpiration,
"www.facebook.com",
);
expect(result).not.toContain("expired");
});
test("should handle no matching cookies", () => {
const result = formatCookiesForHeader(mockCookies, "www.google.com");
expect(result).toBe("");
});
test("should handle empty cookie array", () => {
const result = formatCookiesForHeader([], "www.facebook.com");
expect(result).toBe("");
});
});
});
});

View File

@@ -0,0 +1,712 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import fetchFacebookItems, { fetchFacebookItem } from "../src/facebook";
// Mock fetch globally
const originalFetch = global.fetch;
describe("Facebook Marketplace Scraper Integration Tests", () => {
beforeEach(() => {
global.fetch = mock(() => {
throw new Error("fetch should be mocked in individual tests");
});
});
afterEach(() => {
global.fetch = originalFetch;
});
describe("Main Search Function", () => {
const mockCookies = JSON.stringify([
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
]);
test("should successfully fetch search results", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "iPhone 13 Pro",
listing_price: {
amount: "800.00",
formatted_amount: "$800.00",
currency: "CAD",
},
location: {
reverse_geocode: {
city_page: { display_name: "Toronto" },
},
},
creation_time: 1640995200,
is_live: true,
},
},
},
{
node: {
listing: {
id: "2",
marketplace_listing_title: "Samsung Galaxy",
listing_price: {
amount: "600.00",
formatted_amount: "$600.00",
currency: "CAD",
},
location: {
reverse_geocode: {
city_page: { display_name: "Mississauga" },
},
},
creation_time: 1640995300,
is_live: true,
},
},
},
],
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"iPhone",
1,
"toronto",
25,
mockCookies,
);
expect(results).toHaveLength(2);
expect(results[0].title).toBe("iPhone 13 Pro");
expect(results[1].title).toBe("Samsung Galaxy");
});
test("should filter out items without price", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "With Price",
listing_price: {
amount: "100.00",
formatted_amount: "$100.00",
currency: "CAD",
},
is_live: true,
},
},
},
{
node: {
listing: {
id: "2",
marketplace_listing_title: "No Price",
is_live: true,
},
},
},
],
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"test",
1,
"toronto",
25,
mockCookies,
);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("With Price");
});
test("should respect MAX_ITEMS parameter", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: Array.from({ length: 10 }, (_, i) => ({
node: {
listing: {
id: String(i),
marketplace_listing_title: `Item ${i}`,
listing_price: {
amount: `${(i + 1) * 10}.00`,
formatted_amount: `$${(i + 1) * 10}.00`,
currency: "CAD",
},
is_live: true,
},
},
})),
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"test",
1,
"toronto",
5,
mockCookies,
);
expect(results).toHaveLength(5);
});
test("should return empty array for no results", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [],
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"nonexistent query",
1,
"toronto",
25,
mockCookies,
);
expect(results).toEqual([]);
});
test("should handle authentication errors gracefully", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: false,
status: 401,
text: () => Promise.resolve("Unauthorized"),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"test",
1,
"toronto",
25,
mockCookies,
);
expect(results).toEqual([]);
});
test("should handle network errors", async () => {
global.fetch = mock(() => Promise.reject(new Error("Network error")));
await expect(
fetchFacebookItems("test", 1, "toronto", 25, mockCookies),
).rejects.toThrow("Network error");
});
test("should handle rate limiting with retry", async () => {
let attempts = 0;
global.fetch = mock(() => {
attempts++;
if (attempts === 1) {
return Promise.resolve({
ok: false,
status: 429,
headers: {
get: (header: string) => {
if (header === "X-RateLimit-Reset") return "1";
return null;
},
},
text: () => Promise.resolve("Rate limited"),
});
}
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Item 1",
listing_price: {
amount: "100.00",
formatted_amount: "$100.00",
currency: "CAD",
},
is_live: true,
},
},
},
],
},
},
},
},
},
},
],
],
};
return Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
});
});
const results = await fetchFacebookItems(
"test",
1,
"toronto",
25,
mockCookies,
);
expect(attempts).toBe(2);
expect(results).toHaveLength(1);
});
});
describe("Vehicle Listing Integration", () => {
const mockCookies = JSON.stringify([
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
]);
test("should correctly identify and parse vehicle listings", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "2006 Honda Civic",
listing_price: {
amount: "8000.00",
formatted_amount: "$8,000.00",
currency: "CAD",
},
is_live: true,
},
},
},
{
node: {
listing: {
id: "2",
marketplace_listing_title: "iPhone 13",
listing_price: {
amount: "800.00",
formatted_amount: "$800.00",
currency: "CAD",
},
is_live: true,
},
},
},
],
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"cars",
1,
"toronto",
25,
mockCookies,
);
expect(results).toHaveLength(2);
// Both should be classified as "item" type in search results (vehicle detection is for item details)
expect(results[0].title).toBe("2006 Honda Civic");
expect(results[1].title).toBe("iPhone 13");
});
});
describe("Different Categories", () => {
const mockCookies = JSON.stringify([
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
]);
test("should handle electronics listings", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Nintendo Switch",
listing_price: {
amount: "250.00",
formatted_amount: "$250.00",
currency: "CAD",
},
location: {
reverse_geocode: {
city_page: { display_name: "Toronto" },
},
},
marketplace_listing_category_id:
"479353692612078",
condition: "USED",
is_live: true,
},
},
},
],
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"nintendo switch",
1,
"toronto",
25,
mockCookies,
);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Nintendo Switch");
expect(results[0].categoryId).toBe("479353692612078");
});
test("should handle home goods/furniture listings", async () => {
const mockSearchData = {
require: [
[
null,
null,
null,
{
__bbox: {
result: {
data: {
marketplace_search: {
feed_units: {
edges: [
{
node: {
listing: {
id: "1",
marketplace_listing_title: "Dining Table",
listing_price: {
amount: "150.00",
formatted_amount: "$150.00",
currency: "CAD",
},
location: {
reverse_geocode: {
city_page: { display_name: "Mississauga" },
},
},
marketplace_listing_category_id:
"1569171756675761",
condition: "USED",
is_live: true,
},
},
},
],
},
},
},
},
},
},
],
],
};
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"table",
1,
"toronto",
25,
mockCookies,
);
expect(results).toHaveLength(1);
expect(results[0].title).toBe("Dining Table");
expect(results[0].categoryId).toBe("1569171756675761");
});
});
describe("Error Scenarios", () => {
const mockCookies = JSON.stringify([
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
]);
test("should handle malformed HTML responses", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: true,
text: () =>
Promise.resolve(
"<html><body>Invalid HTML without JSON data</body></html>",
),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"test",
1,
"toronto",
25,
mockCookies,
);
expect(results).toEqual([]);
});
test("should handle 404 errors gracefully", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: false,
status: 404,
text: () => Promise.resolve("Not found"),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"test",
1,
"toronto",
25,
mockCookies,
);
expect(results).toEqual([]);
});
test("should handle 500 errors gracefully", async () => {
global.fetch = mock(() =>
Promise.resolve({
ok: false,
status: 500,
text: () => Promise.resolve("Internal Server Error"),
headers: {
get: () => null,
},
}),
);
const results = await fetchFacebookItems(
"test",
1,
"toronto",
25,
mockCookies,
);
expect(results).toEqual([]);
});
});
});

View File

@@ -1,13 +1,13 @@
import { describe, test, expect } from "bun:test";
import { describe, expect, test } from "bun:test";
import {
resolveLocationId,
resolveCategoryId,
buildSearchUrl,
HttpError,
NetworkError,
ParseError,
RateLimitError,
ValidationError
ValidationError,
buildSearchUrl,
resolveCategoryId,
resolveLocationId,
} from "../src/kijiji";
describe("Location and Category Resolution", () => {
@@ -74,8 +74,8 @@ describe("URL Construction", () => {
const url = buildSearchUrl("iphone", {
location: 1700272,
category: 132,
sortBy: 'relevancy',
sortOrder: 'desc',
sortBy: "relevancy",
sortOrder: "desc",
});
expect(url).toContain("b-buy-sell/canada/iphone/k0c132l1700272");
@@ -95,15 +95,15 @@ describe("URL Construction", () => {
test("should handle different sort options", () => {
const dateUrl = buildSearchUrl("iphone", {
sortBy: 'date',
sortOrder: 'asc',
sortBy: "date",
sortOrder: "asc",
});
expect(dateUrl).toContain("sort=DATE");
expect(dateUrl).toContain("order=ASC");
const priceUrl = buildSearchUrl("iphone", {
sortBy: 'price',
sortOrder: 'desc',
sortBy: "price",
sortOrder: "desc",
});
expect(priceUrl).toContain("sort=PRICE");
expect(priceUrl).toContain("order=DESC");
@@ -131,7 +131,11 @@ describe("Error Classes", () => {
test("NetworkError should store URL and cause", () => {
const cause = new Error("Connection failed");
const error = new NetworkError("Network error", "https://example.com", cause);
const error = new NetworkError(
"Network error",
"https://example.com",
cause,
);
expect(error.message).toBe("Network error");
expect(error.url).toBe("https://example.com");
expect(error.cause).toBe(cause);

View File

@@ -1,5 +1,9 @@
import { describe, test, expect, beforeEach, afterEach, mock } from "bun:test";
import { extractApolloState, parseSearch, parseDetailedListing } from "../src/kijiji";
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import {
extractApolloState,
parseDetailedListing,
parseSearch,
} from "../src/kijiji";
// Mock fetch globally
const originalFetch = global.fetch;
@@ -18,29 +22,31 @@ describe("HTML Parsing Integration", () => {
describe("extractApolloState", () => {
test("should extract Apollo state from valid HTML", () => {
const mockHtml = '<html><head><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"__APOLLO_STATE__":{"ROOT_QUERY":{"test":"value"}}}}}</script></head></html>';
const mockHtml =
'<html><head><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"__APOLLO_STATE__":{"ROOT_QUERY":{"test":"value"}}}}}</script></head></html>';
const result = extractApolloState(mockHtml);
expect(result).toEqual({
ROOT_QUERY: { test: "value" }
ROOT_QUERY: { test: "value" },
});
});
test("should return null for HTML without Apollo state", () => {
const mockHtml = '<html><body>No data here</body></html>';
const mockHtml = "<html><body>No data here</body></html>";
const result = extractApolloState(mockHtml);
expect(result).toBeNull();
});
test("should return null for malformed JSON", () => {
const mockHtml = '<html><script id="__NEXT_DATA__" type="application/json">{"invalid": json}</script></html>';
const mockHtml =
'<html><script id="__NEXT_DATA__" type="application/json">{"invalid": json}</script></html>';
const result = extractApolloState(mockHtml);
expect(result).toBeNull();
});
test("should handle missing __NEXT_DATA__ element", () => {
const mockHtml = '<html><body><div>Content</div></body></html>';
const mockHtml = "<html><body><div>Content</div></body></html>";
const result = extractApolloState(mockHtml);
expect(result).toBeNull();
});
@@ -63,10 +69,10 @@ describe("HTML Parsing Integration", () => {
url: "/v-samsung/k0l0",
title: "Samsung Galaxy",
},
"ROOT_QUERY": { test: "value" }
}
}
}
ROOT_QUERY: { test: "value" },
},
},
},
})}
</script>
</html>
@@ -76,11 +82,11 @@ describe("HTML Parsing Integration", () => {
expect(results).toHaveLength(2);
expect(results[0]).toEqual({
name: "iPhone 13 Pro",
listingLink: "https://www.kijiji.ca/v-iphone/k0l0"
listingLink: "https://www.kijiji.ca/v-iphone/k0l0",
});
expect(results[1]).toEqual({
name: "Samsung Galaxy",
listingLink: "https://www.kijiji.ca/v-samsung/k0l0"
listingLink: "https://www.kijiji.ca/v-samsung/k0l0",
});
});
@@ -95,17 +101,19 @@ describe("HTML Parsing Integration", () => {
"Listing:123": {
url: "https://www.kijiji.ca/v-iphone/k0l0",
title: "iPhone 13 Pro",
}
}
}
}
},
},
},
},
})}
</script>
</html>
`;
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
expect(results[0].listingLink).toBe("https://www.kijiji.ca/v-iphone/k0l0");
expect(results[0].listingLink).toBe(
"https://www.kijiji.ca/v-iphone/k0l0",
);
});
test("should filter out invalid listings", () => {
@@ -127,10 +135,10 @@ describe("HTML Parsing Integration", () => {
"Other:789": {
url: "/v-other/k0l0",
title: "Other Item",
}
}
}
}
},
},
},
},
})}
</script>
</html>
@@ -142,7 +150,10 @@ describe("HTML Parsing Integration", () => {
});
test("should return empty array for invalid HTML", () => {
const results = parseSearch("<html><body>Invalid</body></html>", "https://www.kijiji.ca");
const results = parseSearch(
"<html><body>Invalid</body></html>",
"https://www.kijiji.ca",
);
expect(results).toEqual([]);
});
});
@@ -163,7 +174,7 @@ describe("HTML Parsing Integration", () => {
price: {
amount: 80000,
currency: "CAD",
type: "FIXED"
type: "FIXED",
},
type: "OFFER",
status: "ACTIVE",
@@ -176,38 +187,47 @@ describe("HTML Parsing Integration", () => {
name: "Toronto",
coordinates: {
latitude: 43.6532,
longitude: -79.3832
}
longitude: -79.3832,
},
},
imageUrls: [
"https://media.kijiji.ca/api/v1/image1.jpg",
"https://media.kijiji.ca/api/v1/image2.jpg"
"https://media.kijiji.ca/api/v1/image2.jpg",
],
imageCount: 2,
categoryId: 132,
adSource: "ORGANIC",
flags: {
topAd: false,
priceDrop: true
priceDrop: true,
},
posterInfo: {
posterId: "user123",
rating: 4.8
rating: 4.8,
},
attributes: [
{ canonicalName: "forsaleby", canonicalValues: ["ownr"] },
{ canonicalName: "phonecarrier", canonicalValues: ["unlocked"] }
]
}
}
}
}
{
canonicalName: "forsaleby",
canonicalValues: ["ownr"],
},
{
canonicalName: "phonecarrier",
canonicalValues: ["unlocked"],
},
],
},
},
},
},
})}
</script>
</html>
`;
const result = await parseDetailedListing(mockHtml, "https://www.kijiji.ca");
const result = await parseDetailedListing(
mockHtml,
"https://www.kijiji.ca",
);
expect(result).toEqual({
url: "https://www.kijiji.ca/v-iphone-13-pro/k0l0",
title: "iPhone 13 Pro 256GB",
@@ -215,7 +235,7 @@ describe("HTML Parsing Integration", () => {
listingPrice: {
amountFormatted: "$800.00",
cents: 80000,
currency: "CAD"
currency: "CAD",
},
listingType: "OFFER",
listingStatus: "ACTIVE",
@@ -225,30 +245,30 @@ describe("HTML Parsing Integration", () => {
address: "Toronto, ON",
images: [
"https://media.kijiji.ca/api/v1/image1.jpg",
"https://media.kijiji.ca/api/v1/image2.jpg"
"https://media.kijiji.ca/api/v1/image2.jpg",
],
categoryId: 132,
adSource: "ORGANIC",
flags: {
topAd: false,
priceDrop: true
priceDrop: true,
},
attributes: {
forsaleby: ["ownr"],
phonecarrier: ["unlocked"]
phonecarrier: ["unlocked"],
},
location: {
id: 1700273,
name: "Toronto",
coordinates: {
latitude: 43.6532,
longitude: -79.3832
}
longitude: -79.3832,
},
},
sellerInfo: {
posterId: "user123",
rating: 4.8
}
rating: 4.8,
},
});
});
@@ -265,18 +285,21 @@ describe("HTML Parsing Integration", () => {
title: "iPhone for Sale",
price: {
type: "CONTACT",
amount: null
}
}
}
}
}
amount: null,
},
},
},
},
},
})}
</script>
</html>
`;
const result = await parseDetailedListing(mockHtml, "https://www.kijiji.ca");
const result = await parseDetailedListing(
mockHtml,
"https://www.kijiji.ca",
);
expect(result).toBeNull();
});
@@ -291,17 +314,20 @@ describe("HTML Parsing Integration", () => {
"Listing:123": {
url: "/v-iphone/k0l0",
title: "iPhone 13",
price: { amount: 50000 }
}
}
}
}
price: { amount: 50000 },
},
},
},
},
})}
</script>
</html>
`;
const result = await parseDetailedListing(mockHtml, "https://www.kijiji.ca");
const result = await parseDetailedListing(
mockHtml,
"https://www.kijiji.ca",
);
expect(result).toEqual({
url: "https://www.kijiji.ca/v-iphone/k0l0",
title: "iPhone 13",
@@ -309,7 +335,7 @@ describe("HTML Parsing Integration", () => {
listingPrice: {
amountFormatted: "$500.00",
cents: 50000,
currency: undefined
currency: undefined,
},
listingType: undefined,
listingStatus: undefined,
@@ -322,15 +348,15 @@ describe("HTML Parsing Integration", () => {
adSource: "UNKNOWN",
flags: {
topAd: false,
priceDrop: false
priceDrop: false,
},
attributes: {},
location: {
id: 0,
name: "Unknown",
coordinates: undefined
coordinates: undefined,
},
sellerInfo: undefined
sellerInfo: undefined,
});
});
});

View File

@@ -1,5 +1,5 @@
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
import { slugify, formatCentsToCurrency } from "../src/kijiji";
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { formatCentsToCurrency, slugify } from "../src/kijiji";
describe("Utility Functions", () => {
describe("slugify", () => {

View File

@@ -5,8 +5,10 @@ import { expect } from "bun:test";
// This file is loaded before any tests run due to bunfig.toml preload
// Mock fetch globally for tests
global.fetch = global.fetch || (() => {
throw new Error('fetch is not available in test environment');
global.fetch =
global.fetch ||
(() => {
throw new Error("fetch is not available in test environment");
});
// Add any global test utilities here

View File

@@ -7,25 +7,21 @@
"moduleDetection": "force",
"jsx": "react-jsx",
"allowJs": true,
// Bundler mode
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"noEmit": true,
// Best practices
"strict": true,
"skipLibCheck": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedIndexedAccess": true,
"noImplicitAny": true,
// Some stricter flags (disabled by default)
"noUnusedLocals": false,
"noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false,
"paths": {
"@/*": ["./src/*"]
}