Compare commits
20 Commits
daa61c25d8
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e741ead690 | |||
| 6ab9c4c3a5 | |||
| 3919ec0727 | |||
| 1090ad5bfb | |||
| c937a70db7 | |||
| 59fcbf9ed2 | |||
| d8542eb8f7 | |||
| 0a114cf323 | |||
| 5f7de1167e | |||
| 9edafc88c8 | |||
| 5871644e8b | |||
| d5d050013e | |||
| ff56a29171 | |||
| 6a36214528 | |||
| 7af1be3977 | |||
| 844e566b57 | |||
| b3be32835a | |||
| baa34eefdf | |||
| 9011ab4793 | |||
| aae0ce90b8 |
382
FMARKETPLACE.md
Normal file
382
FMARKETPLACE.md
Normal file
@@ -0,0 +1,382 @@
|
|||||||
|
# Facebook Marketplace API Reverse Engineering
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
This document tracks findings from reverse-engineering Facebook Marketplace APIs for listing details.
|
||||||
|
|
||||||
|
## Current Implementation Status
|
||||||
|
- Search functionality: Implemented in `src/facebook.ts`
|
||||||
|
- Individual listing details: Not yet implemented
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### Step 1: Initial Setup
|
||||||
|
- Using Chrome DevTools to inspect Facebook Marketplace
|
||||||
|
- Need to authenticate with Facebook account to access marketplace data
|
||||||
|
- Cookies required for full access
|
||||||
|
- Current status: Successfully logged in and accessed marketplace data
|
||||||
|
|
||||||
|
### Step 2: Individual Listing Details Analysis - COMPLETED
|
||||||
|
- **Data Location**: Embedded in HTML script tags within `require` array structure
|
||||||
|
- **Path**: `require[0][3].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
- **Authentication**: Required for full data access
|
||||||
|
- **Current Status**: Successfully reverse-engineered the API structure and data extraction method
|
||||||
|
|
||||||
|
### API Endpoints Discovered
|
||||||
|
|
||||||
|
#### Search Endpoint
|
||||||
|
- URL: `https://www.facebook.com/marketplace/{location}/search`
|
||||||
|
- Parameters: `query`, `sortBy`, `exact`
|
||||||
|
- Data embedded in HTML script tags with `require` structure
|
||||||
|
- Authentication: Required (cookies)
|
||||||
|
|
||||||
|
#### Listing Details Endpoint
|
||||||
|
- **URL Structure**: `https://www.facebook.com/marketplace/item/{listing_id}/`
|
||||||
|
- **Data Source**: Server-side rendered HTML with embedded JSON data in script tags
|
||||||
|
- **Data Structure**: Relay/GraphQL style data structure under `require[0][3].__bbox.require[...].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
- **Extraction Method**: Parse JSON from script tags containing marketplace data, navigate to the target object
|
||||||
|
- **Authentication**: Required (cookies)
|
||||||
|
|
||||||
|
### Listing Data Structure Discovered (Current - 2026)
|
||||||
|
|
||||||
|
The current Facebook Marketplace API returns a comprehensive `GroupCommerceProductItem` object with the following key properties:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface FacebookMarketplaceItem {
|
||||||
|
// Basic identification
|
||||||
|
id: string;
|
||||||
|
__typename: "GroupCommerceProductItem";
|
||||||
|
|
||||||
|
// Listing content
|
||||||
|
marketplace_listing_title: string;
|
||||||
|
redacted_description: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
custom_title?: string;
|
||||||
|
|
||||||
|
// Pricing
|
||||||
|
formatted_price: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
listing_price: {
|
||||||
|
amount: string;
|
||||||
|
currency: string;
|
||||||
|
amount_with_offset: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Location
|
||||||
|
location_text: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
location: {
|
||||||
|
latitude: number;
|
||||||
|
longitude: number;
|
||||||
|
reverse_geocode_detailed: {
|
||||||
|
country_alpha_two: string;
|
||||||
|
postal_code_trimmed: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// Status flags
|
||||||
|
is_live: boolean;
|
||||||
|
is_sold: boolean;
|
||||||
|
is_pending: boolean;
|
||||||
|
is_hidden: boolean;
|
||||||
|
is_draft: boolean;
|
||||||
|
|
||||||
|
// Timing
|
||||||
|
creation_time: number;
|
||||||
|
|
||||||
|
// Seller information
|
||||||
|
marketplace_listing_seller: {
|
||||||
|
__typename: "User";
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
profile_picture?: {
|
||||||
|
uri: string;
|
||||||
|
};
|
||||||
|
join_time?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Vehicle-specific fields (for automotive listings)
|
||||||
|
vehicle_make_display_name?: string;
|
||||||
|
vehicle_model_display_name?: string;
|
||||||
|
vehicle_odometer_data?: {
|
||||||
|
unit: "KILOMETERS" | "MILES";
|
||||||
|
value: number;
|
||||||
|
};
|
||||||
|
vehicle_transmission_type?: "AUTOMATIC" | "MANUAL";
|
||||||
|
vehicle_exterior_color?: string;
|
||||||
|
vehicle_interior_color?: string;
|
||||||
|
vehicle_condition?: "EXCELLENT" | "GOOD" | "FAIR" | "POOR";
|
||||||
|
vehicle_fuel_type?: string;
|
||||||
|
vehicle_trim_display_name?: string;
|
||||||
|
|
||||||
|
// Category and commerce
|
||||||
|
marketplace_listing_category_id: string;
|
||||||
|
condition?: string;
|
||||||
|
|
||||||
|
// Commerce features
|
||||||
|
delivery_types?: string[];
|
||||||
|
is_shipping_offered?: boolean;
|
||||||
|
is_buy_now_enabled?: boolean;
|
||||||
|
can_buyer_make_checkout_offer?: boolean;
|
||||||
|
|
||||||
|
// Communication
|
||||||
|
messaging_enabled?: boolean;
|
||||||
|
first_message_suggested_value?: string;
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
logging_id: string;
|
||||||
|
reportable_ent_id: string;
|
||||||
|
origin_target?: {
|
||||||
|
__typename: "Marketplace";
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Related listings (for part-out sellers)
|
||||||
|
marketplace_listing_sets?: {
|
||||||
|
edges: Array<{
|
||||||
|
node: {
|
||||||
|
canonical_listing: {
|
||||||
|
id: string;
|
||||||
|
marketplace_listing_title: string;
|
||||||
|
is_live: boolean;
|
||||||
|
is_sold: boolean;
|
||||||
|
formatted_price: { text: string };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example Data Extracted (Current Structure)
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"__typename": "GroupCommerceProductItem",
|
||||||
|
"marketplace_listing_title": "2012 Mazda MAZDA 3 PART-OUT",
|
||||||
|
"id": "1211645920845312",
|
||||||
|
"redacted_description": {
|
||||||
|
"text": "FOR PARTS ONLY!!!"
|
||||||
|
},
|
||||||
|
"custom_title": "2012 Mazda 3 part-out",
|
||||||
|
"creation_time": 1760450080,
|
||||||
|
"location_text": {
|
||||||
|
"text": "Toronto, ON"
|
||||||
|
},
|
||||||
|
"is_live": true,
|
||||||
|
"is_sold": false,
|
||||||
|
"is_pending": false,
|
||||||
|
"is_hidden": false,
|
||||||
|
"formatted_price": {
|
||||||
|
"text": "FREE"
|
||||||
|
},
|
||||||
|
"listing_price": {
|
||||||
|
"amount_with_offset": "0",
|
||||||
|
"currency": "CAD",
|
||||||
|
"amount": "0.00"
|
||||||
|
},
|
||||||
|
"condition": "USED",
|
||||||
|
"logging_id": "24676483845336407",
|
||||||
|
"marketplace_listing_category_id": "807311116002614",
|
||||||
|
"marketplace_listing_seller": {
|
||||||
|
"__typename": "User",
|
||||||
|
"id": "61570613529010",
|
||||||
|
"name": "Jay Heshin",
|
||||||
|
"profile_picture": {
|
||||||
|
"uri": "https://scontent-yyz1-1.xx.fbcdn.net/v/t39.30808-1/480952111_122133462296687117_4145652046222010716_n.jpg?stp=cp6_dst-jpg_s50x50_tt6&_nc_cat=108&ccb=1-7&_nc_sid=e99d92&_nc_ohc=x_DTkeriVbgQ7kNvwEqT_x3&_nc_oc=Adnqnqf4YsZxgMIkR2mSFrdLb6-BDw4omCWqG_cqB-H0uXGgK1l4-T-fLSGB_CQJEKo&_nc_zt=24&_nc_ht=scontent-yyz1-1.xx&_nc_gid=7GnSwn4MSbllAgGWJy0RTQ&oh=00_AfpY66l8w-LvHvZ6tTgiD9Qh-Or_Udc-OaFiVL9pQ0YXsg&oe=697797CD"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"vehicle_condition": "FAIR",
|
||||||
|
"vehicle_exterior_color": "white",
|
||||||
|
"vehicle_interior_color": "",
|
||||||
|
"vehicle_make_display_name": "Mazda",
|
||||||
|
"vehicle_model_display_name": "3 part-out",
|
||||||
|
"vehicle_odometer_data": {
|
||||||
|
"unit": "KILOMETERS",
|
||||||
|
"value": 999999
|
||||||
|
},
|
||||||
|
"vehicle_transmission_type": "AUTOMATIC",
|
||||||
|
"location": {
|
||||||
|
"latitude": 43.651428222656,
|
||||||
|
"longitude": -79.436645507812,
|
||||||
|
"reverse_geocode_detailed": {
|
||||||
|
"country_alpha_two": "CA",
|
||||||
|
"postal_code_trimmed": "M6H 1C1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"delivery_types": ["IN_PERSON"],
|
||||||
|
"messaging_enabled": true,
|
||||||
|
"first_message_suggested_value": "Hi, is this available?",
|
||||||
|
"marketplace_listing_sets": {
|
||||||
|
"edges": [
|
||||||
|
{
|
||||||
|
"node": {
|
||||||
|
"canonical_listing": {
|
||||||
|
"id": "1435935788228627",
|
||||||
|
"marketplace_listing_title": "2004 Land Rover LR2 PART-OUT",
|
||||||
|
"is_live": true,
|
||||||
|
"formatted_price": {"text": "FREE"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Data Extraction Method
|
||||||
|
|
||||||
|
### Current Method (2026)
|
||||||
|
Facebook Marketplace listing data is embedded in JSON within `<script>` tags in the HTML response. The extraction process:
|
||||||
|
|
||||||
|
1. **Find the Correct Script**: Look for script tags containing marketplace listing data by searching for key fields like `marketplace_listing_title`, `redacted_description`, and `formatted_price`.
|
||||||
|
|
||||||
|
2. **Parse JSON Structure**: The data is nested within a `require` array structure:
|
||||||
|
```
|
||||||
|
require[0][3].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Navigate to Target Object**: The actual listing data is a `GroupCommerceProductItem` object containing comprehensive information about the listing, seller, and vehicle details.
|
||||||
|
|
||||||
|
4. **Handle Dynamic Structure**: Facebook may change the exact path, so robust extraction should search for the target object recursively within the parsed JSON.
|
||||||
|
|
||||||
|
### Authentication Requirements
|
||||||
|
- Valid Facebook session cookies are required
|
||||||
|
- User must be logged in to Facebook
|
||||||
|
- Marketplace access may be location-restricted
|
||||||
|
|
||||||
|
## Tools Used
|
||||||
|
- Chrome DevTools Protocol
|
||||||
|
- Network monitoring
|
||||||
|
- HTML/script parsing
|
||||||
|
- JSON structure analysis
|
||||||
|
|
||||||
|
## Implementation Status
|
||||||
|
- ✅ Successfully reverse-engineered Facebook Marketplace API for listing details
|
||||||
|
- ✅ Identified current data structure and extraction method (2026)
|
||||||
|
- ✅ Documented comprehensive GroupCommerceProductItem interface
|
||||||
|
- ✅ Implemented `extractFacebookItemData()` function with script parsing logic
|
||||||
|
- ✅ Implemented `parseFacebookItem()` function to convert GroupCommerceProductItem to ListingDetails
|
||||||
|
- ✅ Implemented `fetchFacebookItem()` function with authentication and error handling
|
||||||
|
- ✅ Updated TypeScript interfaces to match current API structure
|
||||||
|
- ✅ Added robust extraction with fallback methods for changing API paths
|
||||||
|
|
||||||
|
## Implementation Details
|
||||||
|
|
||||||
|
### Core Functions Implemented
|
||||||
|
|
||||||
|
1. **`extractFacebookItemData(htmlString)`**: Extracts marketplace item data from HTML-embedded JSON in script tags
|
||||||
|
- Searches for scripts containing marketplace listing data
|
||||||
|
- Uses primary path: `require[0][3][0].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
- Falls back to recursive search for GroupCommerceProductItem objects
|
||||||
|
|
||||||
|
2. **`parseFacebookItem(item)`**: Converts Facebook's GroupCommerceProductItem to unified ListingDetails format
|
||||||
|
- Handles pricing (FREE listings, CAD currency)
|
||||||
|
- Extracts seller information, location, and status
|
||||||
|
- Supports vehicle-specific metadata
|
||||||
|
- Maps Facebook-specific fields to common interface
|
||||||
|
|
||||||
|
3. **`fetchFacebookItem(itemId, cookiesSource?)`**: Fetches individual listing details
|
||||||
|
- Loads Facebook authentication cookies
|
||||||
|
- Makes authenticated HTTP requests
|
||||||
|
- Handles rate limiting and retries
|
||||||
|
- Returns parsed ListingDetails or null on failure
|
||||||
|
|
||||||
|
### Authentication Requirements
|
||||||
|
- Facebook session cookies required in `./cookies/facebook.json` or provided as parameter
|
||||||
|
- Cookies must include valid authentication tokens for marketplace access
|
||||||
|
- Handles cookie expiration and domain validation
|
||||||
|
|
||||||
|
## Current Implementation Status - 2026 Verification
|
||||||
|
|
||||||
|
### Step 3: API Verification and Current Structure Analysis (January 2026)
|
||||||
|
- **Verification Date**: January 22, 2026
|
||||||
|
- **Status**: Successfully verified current Facebook Marketplace API structure
|
||||||
|
- **Data Source**: Embedded JSON in HTML script tags (server-side rendered)
|
||||||
|
- **Extraction Path**: `require[0][3].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
|
||||||
|
#### Verified Listing Structure (Real Example - 2006 Hyundai Tiburon)
|
||||||
|
- **Listing ID**: 1226468515995685
|
||||||
|
- **Title**: "2006 Hyundai Tiburon"
|
||||||
|
- **Price**: CA$3,000 (formatted_price.text)
|
||||||
|
- **Raw Price Data**: {"amount_with_offset": "300000", "currency": "CAD", "amount": "3000.00"}
|
||||||
|
- **Location**: Hamilton, ON (with coordinates: 43.250427246094, -79.963989257812)
|
||||||
|
- **Description**: "As is" (redacted_description.text)
|
||||||
|
- **Vehicle Details**:
|
||||||
|
- Make: Hyundai
|
||||||
|
- Model: Tiburon
|
||||||
|
- Odometer: 194,000 km
|
||||||
|
- Transmission: AUTOMATIC
|
||||||
|
- Exterior Color: blue
|
||||||
|
- Interior Color: black
|
||||||
|
- Fuel Type: GASOLINE
|
||||||
|
- Number of Owners: TWO
|
||||||
|
- **Seller Information**:
|
||||||
|
- Name: Ajitpal Kaler
|
||||||
|
- ID: 100009257293466
|
||||||
|
- Profile Picture Available
|
||||||
|
- Join Time: 1426564800 (2015)
|
||||||
|
- **Listing Status**: Active (is_live: true, is_sold: false, is_pending: false)
|
||||||
|
- **Category**: 807311116002614 (Vehicles)
|
||||||
|
- **Delivery Types**: ["IN_PERSON"]
|
||||||
|
- **Messaging**: Enabled
|
||||||
|
|
||||||
|
#### Current API Characteristics
|
||||||
|
- **Authentication**: Still requires valid Facebook session cookies
|
||||||
|
- **Data Format**: Server-side rendered HTML with embedded GraphQL/Relay JSON
|
||||||
|
- **Structure Stability**: Primary extraction path remains functional
|
||||||
|
- **Additional Features**: Includes marketplace ratings, seller verification badges, cross-posting info
|
||||||
|
|
||||||
|
### API Changes Observed Since 2024 Documentation
|
||||||
|
- **Minimal Changes**: Core data structure largely unchanged
|
||||||
|
- **Enhanced Fields**: Added more detailed vehicle specifications and seller profile information
|
||||||
|
- **GraphQL Integration**: Deeper integration with Facebook's GraphQL infrastructure
|
||||||
|
- **Security Features**: Additional integrity checks and reporting mechanisms
|
||||||
|
|
||||||
|
### Multi-Category Testing Results (January 2026)
|
||||||
|
Successfully tested extraction across different listing categories:
|
||||||
|
|
||||||
|
#### 1. Vehicle Listings (Automotive)
|
||||||
|
- **Example**: 2006 Hyundai Tiburon (ID: 1226468515995685)
|
||||||
|
- **Status**: ✅ Fully functional
|
||||||
|
- **Data Extracted**: Complete vehicle specs, pricing, seller info, location coordinates
|
||||||
|
- **Unique Fields**: vehicle_make_display_name, vehicle_odometer_data, vehicle_transmission_type, vehicle_exterior_color, vehicle_interior_color, vehicle_fuel_type
|
||||||
|
|
||||||
|
#### 2. Electronics Listings
|
||||||
|
- **Example**: Nintendo Switch (ID: 3903865769914262)
|
||||||
|
- **Status**: ✅ Fully functional
|
||||||
|
- **Data Extracted**: Title, price (CA$140), location (Toronto, ON), condition (Used - like new), seller (Yitao Hou)
|
||||||
|
- **Category**: Electronics (category_id: 479353692612078)
|
||||||
|
- **Notes**: Standard GroupCommerceProductItem structure applies
|
||||||
|
|
||||||
|
#### 3. Home Goods/Furniture Listings
|
||||||
|
- **Example**: Tabletop Mirror (cat not included) (ID: 1082389057290709)
|
||||||
|
- **Status**: ✅ Fully functional
|
||||||
|
- **Data Extracted**: Title, price (CA$5), location (Mississauga, ON), condition (Used - like new), seller (Rohit Rehan)
|
||||||
|
- **Category**: Home Goods (category_id: 1569171756675761)
|
||||||
|
- **Notes**: Includes detailed description and delivery options
|
||||||
|
|
||||||
|
#### Testing Summary
|
||||||
|
- **Extraction Method**: Consistent across all categories
|
||||||
|
- **Data Structure**: GroupCommerceProductItem interface works for all listing types
|
||||||
|
- **Authentication**: Required for all categories
|
||||||
|
- **Rate Limiting**: Standard Facebook rate limits apply
|
||||||
|
- **Edge Cases**: All tested listings were active/in-person pickup
|
||||||
|
|
||||||
|
## Implementation Status - COMPLETED (January 2026)
|
||||||
|
- ✅ Successfully reverse-engineered Facebook Marketplace API for listing details
|
||||||
|
- ✅ Verified current API structure and extraction method (January 2026)
|
||||||
|
- ✅ Tested extraction across multiple listing categories (vehicles, electronics, home goods)
|
||||||
|
- ✅ Implemented comprehensive error handling for sold/removed listings and authentication failures
|
||||||
|
- ✅ Enhanced rate limiting and retry logic (already robust)
|
||||||
|
- ✅ Added monitoring and metrics for API stability detection
|
||||||
|
- ✅ Updated all scraper functions to use verified extraction methods
|
||||||
|
- ✅ Documented comprehensive GroupCommerceProductItem interface with real examples
|
||||||
|
|
||||||
|
## Next Steps (Future Maintenance)
|
||||||
|
1. Monitor extraction success rates for API change detection
|
||||||
|
2. Update extraction paths if Facebook changes their API structure
|
||||||
|
3. Add support for additional marketplace features as they become available
|
||||||
|
4. Implement caching mechanisms for improved performance
|
||||||
|
5. Add support for marketplace messaging and negotiation features
|
||||||
30
biome.json
Normal file
30
biome.json
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
|
||||||
|
"vcs": {
|
||||||
|
"enabled": false,
|
||||||
|
"clientKind": "git",
|
||||||
|
"useIgnoreFile": false
|
||||||
|
},
|
||||||
|
"files": {
|
||||||
|
"ignoreUnknown": false,
|
||||||
|
"ignore": []
|
||||||
|
},
|
||||||
|
"formatter": {
|
||||||
|
"enabled": true,
|
||||||
|
"indentStyle": "space"
|
||||||
|
},
|
||||||
|
"organizeImports": {
|
||||||
|
"enabled": true
|
||||||
|
},
|
||||||
|
"linter": {
|
||||||
|
"enabled": true,
|
||||||
|
"rules": {
|
||||||
|
"recommended": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"javascript": {
|
||||||
|
"formatter": {
|
||||||
|
"quoteStyle": "double"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,12 +14,14 @@
|
|||||||
"-e",
|
"-e",
|
||||||
"/nix/store/lz8ajxhnkkw2llj752bdz41wqr645h9c-google-chrome-dev-146.0.7635.0/bin/google-chrome-unstable",
|
"/nix/store/lz8ajxhnkkw2llj752bdz41wqr645h9c-google-chrome-dev-146.0.7635.0/bin/google-chrome-unstable",
|
||||||
"--ignore-default-chrome-arg='--disable-extensions'"
|
"--ignore-default-chrome-arg='--disable-extensions'"
|
||||||
]
|
],
|
||||||
|
"enabled": false
|
||||||
},
|
},
|
||||||
"bun-docs": {
|
"bun-docs": {
|
||||||
"type": "remote",
|
"type": "remote",
|
||||||
"url": "https://bun.com/docs/mcp",
|
"url": "https://bun.com/docs/mcp",
|
||||||
"timeout": 3000
|
"timeout": 3000,
|
||||||
|
"enabled": false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
183
scripts/parse-facebook-cookies.ts
Normal file
183
scripts/parse-facebook-cookies.ts
Normal file
@@ -0,0 +1,183 @@
|
|||||||
|
#!/usr/bin/env bun
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Facebook Cookie Parser CLI
|
||||||
|
*
|
||||||
|
* Parses Facebook cookie strings into JSON format for the marketplace scraper
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* bun run scripts/parse-facebook-cookies.ts "c_user=123; xs=abc"
|
||||||
|
* bun run scripts/parse-facebook-cookies.ts --input cookies.txt
|
||||||
|
* echo "c_user=123; xs=abc" | bun run scripts/parse-facebook-cookies.ts
|
||||||
|
* bun run scripts/parse-facebook-cookies.ts "cookie_string" --output my-cookies.json
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { parseFacebookCookieString } from "../src/facebook";
|
||||||
|
|
||||||
|
interface Cookie {
|
||||||
|
name: string;
|
||||||
|
value: string;
|
||||||
|
domain: string;
|
||||||
|
path: string;
|
||||||
|
secure?: boolean;
|
||||||
|
httpOnly?: boolean;
|
||||||
|
sameSite?: "strict" | "lax" | "none" | "unspecified";
|
||||||
|
expirationDate?: number;
|
||||||
|
storeId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFacebookCookieStringCLI(cookieString: string): Cookie[] {
|
||||||
|
if (!cookieString || !cookieString.trim()) {
|
||||||
|
console.error("❌ Error: Empty or invalid cookie string provided");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
if (cookies.length === 0) {
|
||||||
|
console.error("❌ Error: No valid cookies found in input string");
|
||||||
|
console.error('Expected format: "name1=value1; name2=value2;"');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
|
||||||
|
if (args.length === 0 && process.stdin.isTTY === false) {
|
||||||
|
// Read from stdin
|
||||||
|
let input = "";
|
||||||
|
for await (const chunk of process.stdin) {
|
||||||
|
input += chunk;
|
||||||
|
}
|
||||||
|
input = input.trim();
|
||||||
|
|
||||||
|
if (!input) {
|
||||||
|
console.error("❌ Error: No input provided via stdin");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = parseFacebookCookieStringCLI(input);
|
||||||
|
await writeOutput(cookies, "./cookies/facebook.json");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cookieString = "";
|
||||||
|
let outputPath = "./cookies/facebook.json";
|
||||||
|
let inputPath = "";
|
||||||
|
|
||||||
|
// Parse command line arguments
|
||||||
|
for (let i = 0; i < args.length; i++) {
|
||||||
|
const arg = args[i];
|
||||||
|
|
||||||
|
if (arg === "--input" || arg === "-i") {
|
||||||
|
inputPath = args[i + 1];
|
||||||
|
i++; // Skip next arg
|
||||||
|
} else if (arg === "--output" || arg === "-o") {
|
||||||
|
outputPath = args[i + 1];
|
||||||
|
i++; // Skip next arg
|
||||||
|
} else if (arg === "--help" || arg === "-h") {
|
||||||
|
showHelp();
|
||||||
|
return;
|
||||||
|
} else if (!arg.startsWith("-")) {
|
||||||
|
// Assume this is the cookie string
|
||||||
|
cookieString = arg;
|
||||||
|
} else {
|
||||||
|
console.error(`❌ Unknown option: ${arg}`);
|
||||||
|
showHelp();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read from file if specified
|
||||||
|
if (inputPath) {
|
||||||
|
try {
|
||||||
|
const file = Bun.file(inputPath);
|
||||||
|
if (!(await file.exists())) {
|
||||||
|
console.error(`❌ Error: Input file not found: ${inputPath}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
cookieString = await file.text();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`❌ Error reading input file: ${error}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!cookieString.trim()) {
|
||||||
|
console.error("❌ Error: No cookie string provided");
|
||||||
|
console.error(
|
||||||
|
"Provide cookie string as argument, --input file, or via stdin",
|
||||||
|
);
|
||||||
|
showHelp();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = parseFacebookCookieStringCLI(cookieString);
|
||||||
|
await writeOutput(cookies, outputPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeOutput(cookies: Cookie[], outputPath: string) {
|
||||||
|
try {
|
||||||
|
await Bun.write(outputPath, JSON.stringify(cookies, null, 2));
|
||||||
|
console.log(`✅ Successfully parsed ${cookies.length} Facebook cookies`);
|
||||||
|
console.log(`📁 Saved to: ${outputPath}`);
|
||||||
|
|
||||||
|
// Show summary of parsed cookies
|
||||||
|
console.log("\n📋 Parsed cookies:");
|
||||||
|
for (const cookie of cookies) {
|
||||||
|
console.log(
|
||||||
|
` • ${cookie.name}: ${cookie.value.substring(0, 20)}${cookie.value.length > 20 ? "..." : ""}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`❌ Error writing to output file: ${error}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function showHelp() {
|
||||||
|
console.log(`
|
||||||
|
Facebook Cookie Parser CLI
|
||||||
|
|
||||||
|
Parses Facebook cookie strings into JSON format for the marketplace scraper.
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
bun run scripts/parse-facebook-cookies.ts [OPTIONS] [COOKIE_STRING]
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
# Parse from command line argument
|
||||||
|
bun run scripts/parse-facebook-cookies.ts "c_user=123; xs=abc"
|
||||||
|
|
||||||
|
# Parse from file
|
||||||
|
bun run scripts/parse-facebook-cookies.ts --input cookies.txt
|
||||||
|
|
||||||
|
# Parse from stdin
|
||||||
|
echo "c_user=123; xs=abc" | bun run scripts/parse-facebook-cookies.ts
|
||||||
|
|
||||||
|
# Output to custom file
|
||||||
|
bun run scripts/parse-facebook-cookies.ts "cookie_string" --output my-cookies.json
|
||||||
|
|
||||||
|
OPTIONS:
|
||||||
|
-i, --input FILE Read cookie string from file
|
||||||
|
-o, --output FILE Output file path (default: ./cookies/facebook.json)
|
||||||
|
-h, --help Show this help message
|
||||||
|
|
||||||
|
COOKIE FORMAT:
|
||||||
|
Semicolon-separated name=value pairs
|
||||||
|
Example: "c_user=123456789; xs=abcdef123456; fr=xyz789"
|
||||||
|
|
||||||
|
OUTPUT:
|
||||||
|
JSON array of cookie objects saved to ./cookies/facebook.json
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the CLI
|
||||||
|
if (import.meta.main) {
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(`❌ Unexpected error: ${error}`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
176
src/ebay.ts
176
src/ebay.ts
@@ -1,6 +1,6 @@
|
|||||||
|
import cliProgress from "cli-progress";
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
import cliProgress from "cli-progress";
|
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
@@ -55,8 +55,10 @@ function formatCentsToCurrency(
|
|||||||
/**
|
/**
|
||||||
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
||||||
*/
|
*/
|
||||||
function parseEbayPrice(priceText: string): { cents: number; currency: string } | null {
|
function parseEbayPrice(
|
||||||
if (!priceText || typeof priceText !== 'string') return null;
|
priceText: string,
|
||||||
|
): { cents: number; currency: string } | null {
|
||||||
|
if (!priceText || typeof priceText !== "string") return null;
|
||||||
|
|
||||||
// Clean up the price text and extract currency and amount
|
// Clean up the price text and extract currency and amount
|
||||||
const cleaned = priceText.trim();
|
const cleaned = priceText.trim();
|
||||||
@@ -65,19 +67,23 @@ function parseEbayPrice(priceText: string): { cents: number; currency: string }
|
|||||||
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
|
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
|
||||||
if (!numberMatches) return null;
|
if (!numberMatches) return null;
|
||||||
|
|
||||||
const amountStr = numberMatches[0].replace(/,/g, '');
|
const amountStr = numberMatches[0].replace(/,/g, "");
|
||||||
const dollars = parseFloat(amountStr);
|
const dollars = Number.parseFloat(amountStr);
|
||||||
if (isNaN(dollars)) return null;
|
if (Number.isNaN(dollars)) return null;
|
||||||
|
|
||||||
const cents = Math.round(dollars * 100);
|
const cents = Math.round(dollars * 100);
|
||||||
|
|
||||||
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
|
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
|
||||||
let currency = 'USD'; // Default
|
let currency = "USD"; // Default
|
||||||
|
|
||||||
if (cleaned.toUpperCase().includes('CAD') || cleaned.includes('CA$') || cleaned.includes('C $')) {
|
if (
|
||||||
currency = 'CAD';
|
cleaned.toUpperCase().includes("CAD") ||
|
||||||
} else if (cleaned.toUpperCase().includes('USD') || cleaned.includes('$')) {
|
cleaned.includes("CA$") ||
|
||||||
currency = 'USD';
|
cleaned.includes("C $")
|
||||||
|
) {
|
||||||
|
currency = "CAD";
|
||||||
|
} else if (cleaned.toUpperCase().includes("USD") || cleaned.includes("$")) {
|
||||||
|
currency = "USD";
|
||||||
}
|
}
|
||||||
|
|
||||||
return { cents, currency };
|
return { cents, currency };
|
||||||
@@ -135,7 +141,9 @@ async function fetchHtml(
|
|||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
// Respect 429 reset if provided
|
// Respect 429 reset if provided
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : NaN;
|
const resetSeconds = rateLimitReset
|
||||||
|
? Number(rateLimitReset)
|
||||||
|
: Number.NaN;
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
const waitMs = Number.isFinite(resetSeconds)
|
||||||
? Math.max(0, resetSeconds * 1000)
|
? Math.max(0, resetSeconds * 1000)
|
||||||
: (attempt + 1) * retryBaseMs;
|
: (attempt + 1) * retryBaseMs;
|
||||||
@@ -176,7 +184,7 @@ function parseEbayListings(
|
|||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
keywords: string[],
|
keywords: string[],
|
||||||
exclusions: string[],
|
exclusions: string[],
|
||||||
strictMode: boolean
|
strictMode: boolean,
|
||||||
): ListingDetails[] {
|
): ListingDetails[] {
|
||||||
const { document } = parseHTML(htmlString);
|
const { document } = parseHTML(htmlString);
|
||||||
const results: ListingDetails[] = [];
|
const results: ListingDetails[] = [];
|
||||||
@@ -184,16 +192,17 @@ function parseEbayListings(
|
|||||||
// Find all listing links by looking for eBay item URLs (/itm/)
|
// Find all listing links by looking for eBay item URLs (/itm/)
|
||||||
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
||||||
|
|
||||||
|
|
||||||
for (const linkElement of linkElements) {
|
for (const linkElement of linkElements) {
|
||||||
try {
|
try {
|
||||||
// Get href attribute
|
// Get href attribute
|
||||||
let href = linkElement.getAttribute('href');
|
let href = linkElement.getAttribute("href");
|
||||||
if (!href) continue;
|
if (!href) continue;
|
||||||
|
|
||||||
// Make href absolute
|
// Make href absolute
|
||||||
if (!href.startsWith('http')) {
|
if (!href.startsWith("http")) {
|
||||||
href = href.startsWith('//') ? `https:${href}` : `https://www.ebay.com${href}`;
|
href = href.startsWith("//")
|
||||||
|
? `https:${href}`
|
||||||
|
: `https://www.ebay.com${href}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find the container - go up several levels to find the item container
|
// Find the container - go up several levels to find the item container
|
||||||
@@ -207,15 +216,23 @@ function parseEbayListings(
|
|||||||
|
|
||||||
// Extract title - look for heading or title-related elements near the link
|
// Extract title - look for heading or title-related elements near the link
|
||||||
// Modern eBay often uses h3, span, or div with text content near the link
|
// Modern eBay often uses h3, span, or div with text content near the link
|
||||||
let titleElement = container.querySelector('h3, [role="heading"], .s-item__title span');
|
let titleElement = container.querySelector(
|
||||||
|
'h3, [role="heading"], .s-item__title span',
|
||||||
|
);
|
||||||
|
|
||||||
// If no direct title element, try finding text content around the link
|
// If no direct title element, try finding text content around the link
|
||||||
if (!titleElement) {
|
if (!titleElement) {
|
||||||
// Look for spans or divs with text near this link
|
// Look for spans or divs with text near this link
|
||||||
const nearbySpans = container.querySelectorAll('span, div');
|
const nearbySpans = container.querySelectorAll("span, div");
|
||||||
for (const span of nearbySpans) {
|
for (const span of nearbySpans) {
|
||||||
const text = span.textContent?.trim();
|
const text = span.textContent?.trim();
|
||||||
if (text && text.length > 10 && text.length < 200 && !text.includes('$') && !text.includes('item')) {
|
if (
|
||||||
|
text &&
|
||||||
|
text.length > 10 &&
|
||||||
|
text.length < 200 &&
|
||||||
|
!text.includes("$") &&
|
||||||
|
!text.includes("item")
|
||||||
|
) {
|
||||||
titleElement = span;
|
titleElement = span;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -228,12 +245,12 @@ function parseEbayListings(
|
|||||||
if (title) {
|
if (title) {
|
||||||
// Remove common eBay UI strings that appear at the end of titles
|
// Remove common eBay UI strings that appear at the end of titles
|
||||||
const uiStrings = [
|
const uiStrings = [
|
||||||
'Opens in a new window',
|
"Opens in a new window",
|
||||||
'Opens in a new tab',
|
"Opens in a new tab",
|
||||||
'Opens in a new window or tab',
|
"Opens in a new window or tab",
|
||||||
'opens in a new window',
|
"opens in a new window",
|
||||||
'opens in a new tab',
|
"opens in a new tab",
|
||||||
'opens in a new window or tab'
|
"opens in a new window or tab",
|
||||||
];
|
];
|
||||||
|
|
||||||
for (const uiString of uiStrings) {
|
for (const uiString of uiStrings) {
|
||||||
@@ -256,18 +273,28 @@ function parseEbayListings(
|
|||||||
if (title === "Shop on eBay" || title.length < 3) continue;
|
if (title === "Shop on eBay" || title.length < 3) continue;
|
||||||
|
|
||||||
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
||||||
let priceElement = container.querySelector('[class*="s-item__price"], .s-item__price, [class*="price"]');
|
let priceElement = container.querySelector(
|
||||||
|
'[class*="s-item__price"], .s-item__price, [class*="price"]',
|
||||||
|
);
|
||||||
|
|
||||||
// If no direct price class, look for spans containing $ (but not titles)
|
// If no direct price class, look for spans containing $ (but not titles)
|
||||||
if (!priceElement) {
|
if (!priceElement) {
|
||||||
const spansAndElements = container.querySelectorAll('span, div, b, em, strong');
|
const spansAndElements = container.querySelectorAll(
|
||||||
|
"span, div, b, em, strong",
|
||||||
|
);
|
||||||
for (const el of spansAndElements) {
|
for (const el of spansAndElements) {
|
||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
|
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
|
||||||
if (text && text.includes('$') && text.length < 100 &&
|
if (
|
||||||
!text.includes('laptop') && !text.includes('computer') && !text.includes('intel') &&
|
text?.includes("$") &&
|
||||||
!text.includes('core') && !text.includes('ram') && !text.includes('ssd') &&
|
text.length < 100 &&
|
||||||
! /\d{4}/.test(text) && // Avoid years like "2024"
|
!text.includes("laptop") &&
|
||||||
|
!text.includes("computer") &&
|
||||||
|
!text.includes("intel") &&
|
||||||
|
!text.includes("core") &&
|
||||||
|
!text.includes("ram") &&
|
||||||
|
!text.includes("ssd") &&
|
||||||
|
!/\d{4}/.test(text) && // Avoid years like "2024"
|
||||||
!text.includes('"') // Avoid measurements
|
!text.includes('"') // Avoid measurements
|
||||||
) {
|
) {
|
||||||
priceElement = el;
|
priceElement = el;
|
||||||
@@ -280,17 +307,26 @@ function parseEbayListings(
|
|||||||
// Prefer sale/current price over original/strikethrough price
|
// Prefer sale/current price over original/strikethrough price
|
||||||
if (priceElement) {
|
if (priceElement) {
|
||||||
// Check if this element or its parent contains multiple price elements
|
// Check if this element or its parent contains multiple price elements
|
||||||
const priceContainer = priceElement.closest('[class*="s-item__price"]') || priceElement.parentElement;
|
const priceContainer =
|
||||||
|
priceElement.closest('[class*="s-item__price"]') ||
|
||||||
|
priceElement.parentElement;
|
||||||
|
|
||||||
if (priceContainer) {
|
if (priceContainer) {
|
||||||
// Look for all price elements within this container, including strikethrough prices
|
// Look for all price elements within this container, including strikethrough prices
|
||||||
const allPriceElements = priceContainer.querySelectorAll('[class*="s-item__price"], span, b, em, strong, s, del, strike');
|
const allPriceElements = priceContainer.querySelectorAll(
|
||||||
|
'[class*="s-item__price"], span, b, em, strong, s, del, strike',
|
||||||
|
);
|
||||||
|
|
||||||
// Filter to only elements that actually contain prices (not labels)
|
// Filter to only elements that actually contain prices (not labels)
|
||||||
const actualPrices: HTMLElement[] = [];
|
const actualPrices: HTMLElement[] = [];
|
||||||
for (const el of allPriceElements) {
|
for (const el of allPriceElements) {
|
||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
if (text && /^\s*[\$£€¥]/u.test(text) && text.length < 50 && !/\d{4}/.test(text)) {
|
if (
|
||||||
|
text &&
|
||||||
|
/^\s*[\$£€¥]/u.test(text) &&
|
||||||
|
text.length < 50 &&
|
||||||
|
!/\d{4}/.test(text)
|
||||||
|
) {
|
||||||
actualPrices.push(el);
|
actualPrices.push(el);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -298,11 +334,18 @@ function parseEbayListings(
|
|||||||
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
|
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
|
||||||
if (actualPrices.length > 1) {
|
if (actualPrices.length > 1) {
|
||||||
// First, look for prices that are NOT struck through
|
// First, look for prices that are NOT struck through
|
||||||
const nonStrikethroughPrices = actualPrices.filter(el => {
|
const nonStrikethroughPrices = actualPrices.filter((el) => {
|
||||||
const tagName = el.tagName.toLowerCase();
|
const tagName = el.tagName.toLowerCase();
|
||||||
const styles = el.classList.contains('s-strikethrough') || el.classList.contains('u-flStrike') ||
|
const styles =
|
||||||
el.closest('s, del, strike');
|
el.classList.contains("s-strikethrough") ||
|
||||||
return tagName !== 's' && tagName !== 'del' && tagName !== 'strike' && !styles;
|
el.classList.contains("u-flStrike") ||
|
||||||
|
el.closest("s, del, strike");
|
||||||
|
return (
|
||||||
|
tagName !== "s" &&
|
||||||
|
tagName !== "del" &&
|
||||||
|
tagName !== "strike" &&
|
||||||
|
!styles
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
if (nonStrikethroughPrices.length > 0) {
|
if (nonStrikethroughPrices.length > 0) {
|
||||||
@@ -317,7 +360,7 @@ function parseEbayListings(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let priceText = priceElement?.textContent?.trim();
|
const priceText = priceElement?.textContent?.trim();
|
||||||
|
|
||||||
if (!priceText) continue;
|
if (!priceText) continue;
|
||||||
|
|
||||||
@@ -326,12 +369,21 @@ function parseEbayListings(
|
|||||||
if (!priceInfo) continue;
|
if (!priceInfo) continue;
|
||||||
|
|
||||||
// Apply exclusion filters
|
// Apply exclusion filters
|
||||||
if (exclusions.some(exclusion => title.toLowerCase().includes(exclusion.toLowerCase()))) {
|
if (
|
||||||
|
exclusions.some((exclusion) =>
|
||||||
|
title.toLowerCase().includes(exclusion.toLowerCase()),
|
||||||
|
)
|
||||||
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply strict mode filter (title must contain at least one keyword)
|
// Apply strict mode filter (title must contain at least one keyword)
|
||||||
if (strictMode && !keywords.some(keyword => title!.toLowerCase().includes(keyword.toLowerCase()))) {
|
if (
|
||||||
|
strictMode &&
|
||||||
|
!keywords.some((keyword) =>
|
||||||
|
title?.toLowerCase().includes(keyword.toLowerCase()),
|
||||||
|
)
|
||||||
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -351,7 +403,6 @@ function parseEbayListings(
|
|||||||
results.push(listing);
|
results.push(listing);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn(`Error parsing eBay listing: ${err}`);
|
console.warn(`Error parsing eBay listing: ${err}`);
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -376,7 +427,7 @@ export default async function fetchEbayItems(
|
|||||||
maxPrice = Number.MAX_SAFE_INTEGER,
|
maxPrice = Number.MAX_SAFE_INTEGER,
|
||||||
strictMode = false,
|
strictMode = false,
|
||||||
exclusions = [],
|
exclusions = [],
|
||||||
keywords = [SEARCH_QUERY] // Default to search query if no keywords provided
|
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
||||||
} = opts;
|
} = opts;
|
||||||
|
|
||||||
// Build eBay search URL - use Canadian site and tracking parameters like real browser
|
// Build eBay search URL - use Canadian site and tracking parameters like real browser
|
||||||
@@ -389,18 +440,19 @@ export default async function fetchEbayItems(
|
|||||||
try {
|
try {
|
||||||
// Use custom headers modeled after real browser requests to bypass bot detection
|
// Use custom headers modeled after real browser requests to bypass bot detection
|
||||||
const headers: Record<string, string> = {
|
const headers: Record<string, string> = {
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0',
|
"User-Agent":
|
||||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
|
||||||
'Accept-Language': 'en-US,en;q=0.5',
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
'Accept-Encoding': 'gzip, deflate, br',
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
'Referer': 'https://www.ebay.ca/',
|
"Accept-Encoding": "gzip, deflate, br",
|
||||||
'Connection': 'keep-alive',
|
Referer: "https://www.ebay.ca/",
|
||||||
'Upgrade-Insecure-Requests': '1',
|
Connection: "keep-alive",
|
||||||
'Sec-Fetch-Dest': 'document',
|
"Upgrade-Insecure-Requests": "1",
|
||||||
'Sec-Fetch-Mode': 'navigate',
|
"Sec-Fetch-Dest": "document",
|
||||||
'Sec-Fetch-Site': 'same-origin',
|
"Sec-Fetch-Mode": "navigate",
|
||||||
'Sec-Fetch-User': '?1',
|
"Sec-Fetch-Site": "same-origin",
|
||||||
'Priority': 'u=0, i'
|
"Sec-Fetch-User": "?1",
|
||||||
|
Priority: "u=0, i",
|
||||||
};
|
};
|
||||||
|
|
||||||
const res = await fetch(searchUrl, {
|
const res = await fetch(searchUrl, {
|
||||||
@@ -420,19 +472,23 @@ export default async function fetchEbayItems(
|
|||||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
||||||
await delay(DELAY_MS);
|
await delay(DELAY_MS);
|
||||||
|
|
||||||
console.log(`\nParsing eBay listings...`);
|
console.log("\nParsing eBay listings...");
|
||||||
|
|
||||||
const listings = parseEbayListings(searchHtml, keywords, exclusions, strictMode);
|
const listings = parseEbayListings(
|
||||||
|
searchHtml,
|
||||||
|
keywords,
|
||||||
|
exclusions,
|
||||||
|
strictMode,
|
||||||
|
);
|
||||||
|
|
||||||
// Filter by price range (additional safety check)
|
// Filter by price range (additional safety check)
|
||||||
const filteredListings = listings.filter(listing => {
|
const filteredListings = listings.filter((listing) => {
|
||||||
const cents = listing.listingPrice?.cents;
|
const cents = listing.listingPrice?.cents;
|
||||||
return cents && cents >= minPrice && cents <= maxPrice;
|
return cents && cents >= minPrice && cents <= maxPrice;
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||||
return filteredListings;
|
return filteredListings;
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
console.error(
|
console.error(
|
||||||
|
|||||||
743
src/facebook.ts
743
src/facebook.ts
@@ -1,6 +1,6 @@
|
|||||||
|
import cliProgress from "cli-progress";
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
import cliProgress from "cli-progress";
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Facebook Marketplace Scraper
|
* Facebook Marketplace Scraper
|
||||||
@@ -24,7 +24,7 @@ interface Cookie {
|
|||||||
sameSite?: "strict" | "lax" | "none" | "unspecified";
|
sameSite?: "strict" | "lax" | "none" | "unspecified";
|
||||||
session?: boolean;
|
session?: boolean;
|
||||||
expirationDate?: number;
|
expirationDate?: number;
|
||||||
partitionKey?: any;
|
partitionKey?: Record<string, unknown>;
|
||||||
storeId?: string;
|
storeId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,6 +68,112 @@ interface FacebookRequireData {
|
|||||||
[k: string]: unknown;
|
[k: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FacebookMarketplaceItem {
|
||||||
|
// Basic identification
|
||||||
|
id: string;
|
||||||
|
__typename: "GroupCommerceProductItem";
|
||||||
|
|
||||||
|
// Listing content
|
||||||
|
marketplace_listing_title: string;
|
||||||
|
redacted_description?: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
custom_title?: string;
|
||||||
|
|
||||||
|
// Pricing
|
||||||
|
formatted_price?: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
listing_price?: {
|
||||||
|
amount: string;
|
||||||
|
currency: string;
|
||||||
|
amount_with_offset: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Location
|
||||||
|
location_text?: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
location?: {
|
||||||
|
latitude: number;
|
||||||
|
longitude: number;
|
||||||
|
reverse_geocode_detailed?: {
|
||||||
|
country_alpha_two: string;
|
||||||
|
postal_code_trimmed: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// Status flags
|
||||||
|
is_live?: boolean;
|
||||||
|
is_sold?: boolean;
|
||||||
|
is_pending?: boolean;
|
||||||
|
is_hidden?: boolean;
|
||||||
|
is_draft?: boolean;
|
||||||
|
|
||||||
|
// Timing
|
||||||
|
creation_time?: number;
|
||||||
|
|
||||||
|
// Seller information
|
||||||
|
marketplace_listing_seller?: {
|
||||||
|
__typename: "User";
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
profile_picture?: {
|
||||||
|
uri: string;
|
||||||
|
};
|
||||||
|
join_time?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Vehicle-specific fields (for automotive listings)
|
||||||
|
vehicle_make_display_name?: string;
|
||||||
|
vehicle_model_display_name?: string;
|
||||||
|
vehicle_odometer_data?: {
|
||||||
|
unit: "KILOMETERS" | "MILES";
|
||||||
|
value: number;
|
||||||
|
};
|
||||||
|
vehicle_transmission_type?: "AUTOMATIC" | "MANUAL";
|
||||||
|
vehicle_exterior_color?: string;
|
||||||
|
vehicle_interior_color?: string;
|
||||||
|
vehicle_condition?: "EXCELLENT" | "GOOD" | "FAIR" | "POOR";
|
||||||
|
vehicle_fuel_type?: string;
|
||||||
|
vehicle_trim_display_name?: string;
|
||||||
|
|
||||||
|
// Category and commerce
|
||||||
|
marketplace_listing_category_id?: string;
|
||||||
|
condition?: string;
|
||||||
|
|
||||||
|
// Commerce features
|
||||||
|
delivery_types?: string[];
|
||||||
|
is_shipping_offered?: boolean;
|
||||||
|
is_buy_now_enabled?: boolean;
|
||||||
|
can_buyer_make_checkout_offer?: boolean;
|
||||||
|
|
||||||
|
// Communication
|
||||||
|
messaging_enabled?: boolean;
|
||||||
|
first_message_suggested_value?: string;
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
logging_id?: string;
|
||||||
|
reportable_ent_id?: string;
|
||||||
|
|
||||||
|
// Related listings (for part-out sellers)
|
||||||
|
marketplace_listing_sets?: {
|
||||||
|
edges: Array<{
|
||||||
|
node: {
|
||||||
|
canonical_listing: {
|
||||||
|
id: string;
|
||||||
|
marketplace_listing_title: string;
|
||||||
|
is_live: boolean;
|
||||||
|
is_sold: boolean;
|
||||||
|
formatted_price: { text: string };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
[k: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
type ListingDetails = {
|
type ListingDetails = {
|
||||||
url: string;
|
url: string;
|
||||||
title: string;
|
title: string;
|
||||||
@@ -107,7 +213,10 @@ async function delay(ms: number): Promise<void> {
|
|||||||
/**
|
/**
|
||||||
* Load Facebook cookies from file or string
|
* Load Facebook cookies from file or string
|
||||||
*/
|
*/
|
||||||
async function loadFacebookCookies(cookiesSource?: string): Promise<Cookie[]> {
|
async function loadFacebookCookies(
|
||||||
|
cookiesSource?: string,
|
||||||
|
cookiePath = "./cookies/facebook.json",
|
||||||
|
): Promise<Cookie[]> {
|
||||||
// First try to load from provided string parameter
|
// First try to load from provided string parameter
|
||||||
if (cookiesSource) {
|
if (cookiesSource) {
|
||||||
try {
|
try {
|
||||||
@@ -125,9 +234,9 @@ async function loadFacebookCookies(cookiesSource?: string): Promise<Cookie[]> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to load from ./cookies/facebook.json
|
// Try to load from specified path
|
||||||
try {
|
try {
|
||||||
const cookiesPath = "./cookies/facebook.json";
|
const cookiesPath = cookiePath;
|
||||||
const file = Bun.file(cookiesPath);
|
const file = Bun.file(cookiesPath);
|
||||||
if (await file.exists()) {
|
if (await file.exists()) {
|
||||||
const content = await file.text();
|
const content = await file.text();
|
||||||
@@ -148,6 +257,89 @@ async function loadFacebookCookies(cookiesSource?: string): Promise<Cookie[]> {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse Facebook cookie string into Cookie array format
|
||||||
|
*/
|
||||||
|
function parseFacebookCookieString(cookieString: string): Cookie[] {
|
||||||
|
if (!cookieString || !cookieString.trim()) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return cookieString
|
||||||
|
.split(";")
|
||||||
|
.map((pair) => pair.trim())
|
||||||
|
.filter((pair) => pair.includes("="))
|
||||||
|
.map((pair) => {
|
||||||
|
const [name, value] = pair.split("=", 2);
|
||||||
|
const trimmedName = name.trim();
|
||||||
|
const trimmedValue = value.trim();
|
||||||
|
|
||||||
|
// Skip empty names or values
|
||||||
|
if (!trimmedName || !trimmedValue) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: trimmedName,
|
||||||
|
value: decodeURIComponent(trimmedValue),
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
secure: true,
|
||||||
|
httpOnly: false,
|
||||||
|
sameSite: "lax" as const,
|
||||||
|
expirationDate: undefined, // Session cookies
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((cookie): cookie is Cookie => cookie !== null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure Facebook cookies are available, parsing from env var if needed
|
||||||
|
*/
|
||||||
|
async function ensureFacebookCookies(
|
||||||
|
cookiePath = "./cookies/facebook.json",
|
||||||
|
): Promise<Cookie[]> {
|
||||||
|
// First try to load existing cookies
|
||||||
|
try {
|
||||||
|
const existing = await loadFacebookCookies(undefined, cookiePath);
|
||||||
|
if (existing.length > 0) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// File doesn't exist or is invalid, continue to check env var
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to parse from environment variable
|
||||||
|
const cookieString = process.env.FACEBOOK_COOKIE;
|
||||||
|
if (!cookieString || !cookieString.trim()) {
|
||||||
|
throw new Error(
|
||||||
|
"No valid Facebook cookies found. Either:\n" +
|
||||||
|
" 1. Set FACEBOOK_COOKIE environment variable with cookie string, or\n" +
|
||||||
|
" 2. Create ./cookies/facebook.json manually with cookie array",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the cookie string
|
||||||
|
const cookies = parseFacebookCookieString(cookieString);
|
||||||
|
if (cookies.length === 0) {
|
||||||
|
throw new Error(
|
||||||
|
"FACEBOOK_COOKIE environment variable contains no valid cookies. " +
|
||||||
|
'Expected format: "name1=value1; name2=value2;"',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save to file for future use
|
||||||
|
try {
|
||||||
|
await Bun.write(cookiePath, JSON.stringify(cookies, null, 2));
|
||||||
|
console.log(`✅ Saved ${cookies.length} Facebook cookies to ${cookiePath}`);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`! Could not save cookies to ${cookiePath}: ${error}`);
|
||||||
|
// Continue anyway, we have the cookies in memory
|
||||||
|
}
|
||||||
|
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Format cookies array into Cookie header string
|
* Format cookies array into Cookie header string
|
||||||
*/
|
*/
|
||||||
@@ -161,10 +353,9 @@ function formatCookiesForHeader(cookies: Cookie[], domain: string): string {
|
|||||||
domain.endsWith(cookie.domain.slice(1)) ||
|
domain.endsWith(cookie.domain.slice(1)) ||
|
||||||
domain === cookie.domain.slice(1)
|
domain === cookie.domain.slice(1)
|
||||||
);
|
);
|
||||||
} else {
|
}
|
||||||
// Host-only cookie
|
// Host-only cookie
|
||||||
return cookie.domain === domain;
|
return cookie.domain === domain;
|
||||||
}
|
|
||||||
})
|
})
|
||||||
.filter((cookie) => {
|
.filter((cookie) => {
|
||||||
// Check expiration
|
// Check expiration
|
||||||
@@ -230,7 +421,7 @@ async function fetchHtml(
|
|||||||
|
|
||||||
// Add cookies if provided
|
// Add cookies if provided
|
||||||
if (opts?.cookies) {
|
if (opts?.cookies) {
|
||||||
headers["cookie"] = opts.cookies;
|
headers.cookie = opts.cookies;
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = await fetch(url, {
|
const res = await fetch(url, {
|
||||||
@@ -245,7 +436,9 @@ async function fetchHtml(
|
|||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
// Respect 429 reset if provided
|
// Respect 429 reset if provided
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : NaN;
|
const resetSeconds = rateLimitReset
|
||||||
|
? Number(rateLimitReset)
|
||||||
|
: Number.NaN;
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
const waitMs = Number.isFinite(resetSeconds)
|
||||||
? Math.max(0, resetSeconds * 1000)
|
? Math.max(0, resetSeconds * 1000)
|
||||||
: (attempt + 1) * retryBaseMs;
|
: (attempt + 1) * retryBaseMs;
|
||||||
@@ -300,7 +493,7 @@ function extractFacebookMarketplaceData(
|
|||||||
let marketplaceData: FacebookMarketplaceSearch | null = null;
|
let marketplaceData: FacebookMarketplaceSearch | null = null;
|
||||||
|
|
||||||
// Find the script containing the require data with marketplace_search
|
// Find the script containing the require data with marketplace_search
|
||||||
for (const script of scripts as unknown as HTMLScriptElement[]) {
|
for (const script of Array.from(scripts) as HTMLScriptElement[]) {
|
||||||
const scriptText = script.textContent;
|
const scriptText = script.textContent;
|
||||||
if (!scriptText) continue;
|
if (!scriptText) continue;
|
||||||
|
|
||||||
@@ -312,58 +505,114 @@ function extractFacebookMarketplaceData(
|
|||||||
// Try multiple navigation paths to find marketplace_search
|
// Try multiple navigation paths to find marketplace_search
|
||||||
const paths = [
|
const paths = [
|
||||||
// Original path from example
|
// Original path from example
|
||||||
() => parsed.require[0][3][0]['__bbox']['require'][0][3][1]['__bbox']['result']['data']['marketplace_search'],
|
() =>
|
||||||
|
parsed.require[0][3][0].__bbox.require[0][3][1].__bbox.result.data
|
||||||
|
.marketplace_search,
|
||||||
// Alternative path structure
|
// Alternative path structure
|
||||||
() => parsed.require[0][3][1]?.__bbox?.result?.data?.marketplace_search,
|
() =>
|
||||||
|
parsed.require[0][3][1]?.__bbox?.result?.data?.marketplace_search,
|
||||||
// Another variation
|
// Another variation
|
||||||
() => parsed.require[0][3][0]['__bbox']['result']['data']['marketplace_search'],
|
() => parsed.require[0][3][0].__bbox.result.data.marketplace_search,
|
||||||
// Direct access for some responses
|
// Direct access for some responses
|
||||||
() => {
|
() => {
|
||||||
for (const item of parsed.require) {
|
for (const item of parsed.require) {
|
||||||
if (item && item.length >= 4 && item[3]) {
|
if (item && item.length >= 4 && item[3]) {
|
||||||
const bbox = item[3]?.['__bbox']?.result?.data?.marketplace_search;
|
const bbox = item[3]?.__bbox?.result?.data?.marketplace_search;
|
||||||
if (bbox) return bbox;
|
if (bbox) return bbox;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
for (const getData of paths) {
|
for (const getData of paths) {
|
||||||
try {
|
try {
|
||||||
const result = getData();
|
const result = getData();
|
||||||
if (result && isRecord(result) && result.feed_units?.edges) {
|
if (
|
||||||
|
result &&
|
||||||
|
isRecord(result) &&
|
||||||
|
result.feed_units?.edges?.length > 0
|
||||||
|
) {
|
||||||
marketplaceData = result as FacebookMarketplaceSearch;
|
marketplaceData = result as FacebookMarketplaceSearch;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {}
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (marketplaceData) break;
|
if (marketplaceData) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Also check for direct marketplace_search in the parsed data
|
// Also check for direct marketplace_search in the parsed data
|
||||||
if (parsed.marketplace_search && isRecord(parsed.marketplace_search) && parsed.marketplace_search.feed_units?.edges) {
|
if (parsed.marketplace_search && isRecord(parsed.marketplace_search)) {
|
||||||
marketplaceData = parsed.marketplace_search as FacebookMarketplaceSearch;
|
const searchData =
|
||||||
|
parsed.marketplace_search as FacebookMarketplaceSearch;
|
||||||
|
if (searchData.feed_units?.edges?.length > 0) {
|
||||||
|
marketplaceData = searchData;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore parsing errors for other scripts
|
// Ignore parsing errors for other scripts
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!marketplaceData?.feed_units?.edges) {
|
if (!marketplaceData?.feed_units?.edges?.length) {
|
||||||
console.warn("No marketplace data found in HTML response");
|
console.warn("No marketplace data found in HTML response");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`Successfully parsed ${marketplaceData.feed_units.edges.length} Facebook marketplace listings`);
|
console.log(
|
||||||
|
`Successfully parsed ${marketplaceData.feed_units.edges.length} Facebook marketplace listings`,
|
||||||
|
);
|
||||||
return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node }));
|
return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Monitor API extraction success/failure for detecting changes
|
||||||
|
*/
|
||||||
|
const extractionStats = {
|
||||||
|
totalExtractions: 0,
|
||||||
|
successfulExtractions: 0,
|
||||||
|
failedExtractions: 0,
|
||||||
|
lastApiChangeDetected: null as Date | null,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log extraction metrics for monitoring API stability
|
||||||
|
*/
|
||||||
|
function logExtractionMetrics(success: boolean, itemId?: string) {
|
||||||
|
extractionStats.totalExtractions++;
|
||||||
|
if (success) {
|
||||||
|
extractionStats.successfulExtractions++;
|
||||||
|
} else {
|
||||||
|
extractionStats.failedExtractions++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log warning if extraction success rate drops below 80%
|
||||||
|
const successRate =
|
||||||
|
extractionStats.successfulExtractions / extractionStats.totalExtractions;
|
||||||
|
if (
|
||||||
|
extractionStats.totalExtractions > 10 &&
|
||||||
|
successRate < 0.8 &&
|
||||||
|
!extractionStats.lastApiChangeDetected
|
||||||
|
) {
|
||||||
|
console.warn(
|
||||||
|
"! Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
|
||||||
|
);
|
||||||
|
extractionStats.lastApiChangeDetected = new Date();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (success) {
|
||||||
|
console.log(
|
||||||
|
`📊 Facebook API extraction stats: ${extractionStats.successfulExtractions}/${extractionStats.totalExtractions} successful`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
console.warn(
|
||||||
|
`❌ Facebook API extraction failed for item ${itemId || "unknown"}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Turns cents to localized currency string.
|
* Turns cents to localized currency string.
|
||||||
*/
|
*/
|
||||||
@@ -376,6 +625,8 @@ function formatCentsToCurrency(
|
|||||||
if (Number.isNaN(cents)) return "";
|
if (Number.isNaN(cents)) return "";
|
||||||
const dollars = cents / 100;
|
const dollars = cents / 100;
|
||||||
const formatter = new Intl.NumberFormat(locale, {
|
const formatter = new Intl.NumberFormat(locale, {
|
||||||
|
style: "currency",
|
||||||
|
currency: "USD",
|
||||||
minimumFractionDigits: 2,
|
minimumFractionDigits: 2,
|
||||||
maximumFractionDigits: 2,
|
maximumFractionDigits: 2,
|
||||||
useGrouping: true,
|
useGrouping: true,
|
||||||
@@ -383,6 +634,145 @@ function formatCentsToCurrency(
|
|||||||
return formatter.format(dollars);
|
return formatter.format(dollars);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
Extract marketplace item details from Facebook item page HTML
|
||||||
|
Updated for 2026 Facebook Marketplace API structure with multiple extraction paths
|
||||||
|
*/
|
||||||
|
function extractFacebookItemData(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
): FacebookMarketplaceItem | null {
|
||||||
|
const { document } = parseHTML(htmlString);
|
||||||
|
const scripts = document.querySelectorAll("script");
|
||||||
|
|
||||||
|
for (const script of scripts) {
|
||||||
|
const scriptText = script.textContent;
|
||||||
|
if (!scriptText) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(scriptText);
|
||||||
|
|
||||||
|
// Check for the 2026 require structure with marketplace product details
|
||||||
|
if (parsed.require && Array.isArray(parsed.require)) {
|
||||||
|
// Try multiple extraction paths discovered from reverse engineering
|
||||||
|
const extractionPaths = [
|
||||||
|
// Path 1: Primary path from current API structure
|
||||||
|
() =>
|
||||||
|
parsed.require[0][3].__bbox.result.data.viewer
|
||||||
|
.marketplace_product_details_page.target,
|
||||||
|
// Path 2: Alternative path with nested require
|
||||||
|
() =>
|
||||||
|
parsed.require[0][3][0].__bbox.require[3][3][1].__bbox.result.data
|
||||||
|
.viewer.marketplace_product_details_page.target,
|
||||||
|
// Path 3: Variation without the [0] index
|
||||||
|
() =>
|
||||||
|
parsed.require[0][3].__bbox.require[3][3][1].__bbox.result.data
|
||||||
|
.viewer.marketplace_product_details_page.target,
|
||||||
|
// Path 4-5: Additional fallback paths for edge cases
|
||||||
|
() =>
|
||||||
|
parsed.require[0][3][1]?.__bbox?.result?.data?.viewer
|
||||||
|
?.marketplace_product_details_page?.target,
|
||||||
|
() =>
|
||||||
|
parsed.require[0][3][2]?.__bbox?.result?.data?.viewer
|
||||||
|
?.marketplace_product_details_page?.target,
|
||||||
|
];
|
||||||
|
|
||||||
|
let pathIndex = 0;
|
||||||
|
for (const getPath of extractionPaths) {
|
||||||
|
try {
|
||||||
|
const targetData = getPath();
|
||||||
|
if (
|
||||||
|
targetData &&
|
||||||
|
typeof targetData === "object" &&
|
||||||
|
targetData.id &&
|
||||||
|
targetData.marketplace_listing_title &&
|
||||||
|
targetData.__typename === "GroupCommerceProductItem"
|
||||||
|
) {
|
||||||
|
console.log(
|
||||||
|
`Successfully extracted Facebook item data using extraction path ${pathIndex + 1}`,
|
||||||
|
);
|
||||||
|
return targetData as FacebookMarketplaceItem;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Path not found or invalid, try next path
|
||||||
|
}
|
||||||
|
pathIndex++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: Search recursively for marketplace data in the parsed structure
|
||||||
|
const findMarketplaceData = (
|
||||||
|
obj: unknown,
|
||||||
|
depth = 0,
|
||||||
|
maxDepth = 10,
|
||||||
|
): FacebookMarketplaceItem | null => {
|
||||||
|
if (depth > maxDepth) return null; // Prevent infinite recursion
|
||||||
|
if (isRecord(obj)) {
|
||||||
|
// Check if this object matches the expected marketplace item structure
|
||||||
|
if (
|
||||||
|
obj.marketplace_listing_title &&
|
||||||
|
obj.id &&
|
||||||
|
obj.__typename === "GroupCommerceProductItem" &&
|
||||||
|
obj.redacted_description
|
||||||
|
) {
|
||||||
|
return obj as FacebookMarketplaceItem;
|
||||||
|
}
|
||||||
|
// Recursively search nested objects and arrays
|
||||||
|
for (const key in obj) {
|
||||||
|
const value = obj[key];
|
||||||
|
if (isRecord(value) || Array.isArray(value)) {
|
||||||
|
const result = findMarketplaceData(value, depth + 1, maxDepth);
|
||||||
|
if (result) return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (Array.isArray(obj)) {
|
||||||
|
// Search through arrays
|
||||||
|
for (const item of obj) {
|
||||||
|
const result = findMarketplaceData(item, depth + 1, maxDepth);
|
||||||
|
if (result) return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Search through the entire require structure
|
||||||
|
const recursiveResult = findMarketplaceData(parsed.require);
|
||||||
|
if (recursiveResult) {
|
||||||
|
console.log(
|
||||||
|
"Successfully extracted Facebook item data using recursive search",
|
||||||
|
);
|
||||||
|
return recursiveResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Additional search in other potential locations
|
||||||
|
if (
|
||||||
|
parsed.__bbox?.result?.data?.viewer?.marketplace_product_details_page
|
||||||
|
?.target
|
||||||
|
) {
|
||||||
|
const bboxData =
|
||||||
|
parsed.__bbox.result.data.viewer.marketplace_product_details_page
|
||||||
|
.target;
|
||||||
|
if (
|
||||||
|
bboxData &&
|
||||||
|
typeof bboxData === "object" &&
|
||||||
|
bboxData.id &&
|
||||||
|
bboxData.marketplace_listing_title &&
|
||||||
|
bboxData.__typename === "GroupCommerceProductItem"
|
||||||
|
) {
|
||||||
|
console.log(
|
||||||
|
"Successfully extracted Facebook item data from __bbox structure",
|
||||||
|
);
|
||||||
|
return bboxData as FacebookMarketplaceItem;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Log parsing errors for debugging but continue to next script
|
||||||
|
console.debug(`Failed to parse script for Facebook item data: ${error}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Parse Facebook marketplace search results into ListingDetails[]
|
Parse Facebook marketplace search results into ListingDetails[]
|
||||||
*/
|
*/
|
||||||
@@ -406,7 +796,8 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
// - formatted_amount: human-readable price (like "CA$1")
|
// - formatted_amount: human-readable price (like "CA$1")
|
||||||
let cents: number;
|
let cents: number;
|
||||||
if (priceObj.amount != null) {
|
if (priceObj.amount != null) {
|
||||||
const dollars = typeof priceObj.amount === 'string'
|
const dollars =
|
||||||
|
typeof priceObj.amount === "string"
|
||||||
? Number.parseFloat(priceObj.amount)
|
? Number.parseFloat(priceObj.amount)
|
||||||
: priceObj.amount;
|
: priceObj.amount;
|
||||||
cents = Math.round(dollars * 100);
|
cents = Math.round(dollars * 100);
|
||||||
@@ -420,7 +811,7 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
if (priceObj.formatted_amount) {
|
if (priceObj.formatted_amount) {
|
||||||
const match = priceObj.formatted_amount.match(/[\d,]+\.?\d*/);
|
const match = priceObj.formatted_amount.match(/[\d,]+\.?\d*/);
|
||||||
if (match) {
|
if (match) {
|
||||||
const dollars = Number.parseFloat(match[0].replace(',', ''));
|
const dollars = Number.parseFloat(match[0].replace(",", ""));
|
||||||
if (!Number.isNaN(dollars)) {
|
if (!Number.isNaN(dollars)) {
|
||||||
cents = Math.round(dollars * 100);
|
cents = Math.round(dollars * 100);
|
||||||
} else {
|
} else {
|
||||||
@@ -465,19 +856,24 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
|
|
||||||
// Extract image and video URLs
|
// Extract image and video URLs
|
||||||
const imageUrl = listing.primary_listing_photo?.image?.uri;
|
const imageUrl = listing.primary_listing_photo?.image?.uri;
|
||||||
const videoUrl = listing.listing_video ? `https://www.facebook.com/${listing.listing_video.id}/` : undefined;
|
const videoUrl = listing.listing_video
|
||||||
|
? `https://www.facebook.com/${listing.listing_video.id}/`
|
||||||
|
: undefined;
|
||||||
|
|
||||||
// Extract seller information
|
// Extract seller information
|
||||||
const seller = listing.marketplace_listing_seller ? {
|
const seller = listing.marketplace_listing_seller
|
||||||
|
? {
|
||||||
name: listing.marketplace_listing_seller.name,
|
name: listing.marketplace_listing_seller.name,
|
||||||
id: listing.marketplace_listing_seller.id
|
id: listing.marketplace_listing_seller.id,
|
||||||
} : undefined;
|
}
|
||||||
|
: undefined;
|
||||||
|
|
||||||
const listingDetails: ListingDetails = {
|
const listingDetails: ListingDetails = {
|
||||||
url,
|
url,
|
||||||
title,
|
title,
|
||||||
listingPrice: {
|
listingPrice: {
|
||||||
amountFormatted: priceObj.formatted_amount || formatCentsToCurrency(cents),
|
amountFormatted:
|
||||||
|
priceObj.formatted_amount || formatCentsToCurrency(cents),
|
||||||
cents,
|
cents,
|
||||||
currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD
|
currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD
|
||||||
},
|
},
|
||||||
@@ -493,15 +889,121 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
};
|
};
|
||||||
|
|
||||||
results.push(listingDetails);
|
results.push(listingDetails);
|
||||||
} catch {
|
} catch {}
|
||||||
// Skip malformed ads
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
Parse Facebook marketplace item details into ListingDetails format
|
||||||
|
Updated for 2026 GroupCommerceProductItem structure
|
||||||
|
*/
|
||||||
|
function parseFacebookItem(
|
||||||
|
item: FacebookMarketplaceItem,
|
||||||
|
): ListingDetails | null {
|
||||||
|
try {
|
||||||
|
const title = item.marketplace_listing_title || item.custom_title;
|
||||||
|
if (!title) return null;
|
||||||
|
|
||||||
|
const url = `https://www.facebook.com/marketplace/item/${item.id}`;
|
||||||
|
|
||||||
|
// Extract price information
|
||||||
|
let cents = 0;
|
||||||
|
let currency = "CAD"; // Default
|
||||||
|
let amountFormatted = item.formatted_price?.text || "FREE";
|
||||||
|
|
||||||
|
if (item.listing_price) {
|
||||||
|
currency = item.listing_price.currency || "CAD";
|
||||||
|
if (item.listing_price.amount && item.listing_price.amount !== "0.00") {
|
||||||
|
const amount = Number.parseFloat(item.listing_price.amount);
|
||||||
|
if (!Number.isNaN(amount)) {
|
||||||
|
cents = Math.round(amount * 100);
|
||||||
|
amountFormatted =
|
||||||
|
item.formatted_price?.text || formatCentsToCurrency(cents);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract description
|
||||||
|
const description = item.redacted_description?.text;
|
||||||
|
|
||||||
|
// Extract location
|
||||||
|
const address = item.location_text?.text || null;
|
||||||
|
|
||||||
|
// Extract seller information
|
||||||
|
const seller = item.marketplace_listing_seller
|
||||||
|
? {
|
||||||
|
name: item.marketplace_listing_seller.name,
|
||||||
|
id: item.marketplace_listing_seller.id,
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Determine listing status
|
||||||
|
let listingStatus: string | undefined;
|
||||||
|
if (item.is_sold) {
|
||||||
|
listingStatus = "SOLD";
|
||||||
|
} else if (item.is_pending) {
|
||||||
|
listingStatus = "PENDING";
|
||||||
|
} else if (item.is_live) {
|
||||||
|
listingStatus = "ACTIVE";
|
||||||
|
} else if (item.is_hidden) {
|
||||||
|
listingStatus = "HIDDEN";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format creation date
|
||||||
|
const creationDate = item.creation_time
|
||||||
|
? new Date(item.creation_time * 1000).toISOString()
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Determine listing type based on category or vehicle data
|
||||||
|
let listingType = "item";
|
||||||
|
if (item.vehicle_make_display_name || item.vehicle_odometer_data) {
|
||||||
|
listingType = "vehicle";
|
||||||
|
} else if (item.marketplace_listing_category_id) {
|
||||||
|
// Could map category IDs to types, but keeping simple for now
|
||||||
|
listingType = "item";
|
||||||
|
}
|
||||||
|
|
||||||
|
const listingDetails: ListingDetails = {
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
listingPrice: {
|
||||||
|
amountFormatted,
|
||||||
|
cents,
|
||||||
|
currency,
|
||||||
|
},
|
||||||
|
address,
|
||||||
|
creationDate,
|
||||||
|
listingType,
|
||||||
|
listingStatus,
|
||||||
|
categoryId: item.marketplace_listing_category_id,
|
||||||
|
seller,
|
||||||
|
deliveryTypes: item.delivery_types,
|
||||||
|
};
|
||||||
|
|
||||||
|
return listingDetails;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`Failed to parse Facebook item ${item.id}:`, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Exports for Testing -----------------------------
|
||||||
|
// Export internal functions for comprehensive testing
|
||||||
|
export {
|
||||||
|
extractFacebookItemData,
|
||||||
|
extractFacebookMarketplaceData,
|
||||||
|
parseFacebookItem,
|
||||||
|
parseFacebookAds,
|
||||||
|
formatCentsToCurrency,
|
||||||
|
loadFacebookCookies,
|
||||||
|
formatCookiesForHeader,
|
||||||
|
parseFacebookCookieString,
|
||||||
|
ensureFacebookCookies,
|
||||||
|
};
|
||||||
|
|
||||||
// ----------------------------- Main -----------------------------
|
// ----------------------------- Main -----------------------------
|
||||||
|
|
||||||
export default async function fetchFacebookItems(
|
export default async function fetchFacebookItems(
|
||||||
@@ -510,9 +1012,18 @@ export default async function fetchFacebookItems(
|
|||||||
LOCATION = "toronto",
|
LOCATION = "toronto",
|
||||||
MAX_ITEMS = 25,
|
MAX_ITEMS = 25,
|
||||||
cookiesSource?: string,
|
cookiesSource?: string,
|
||||||
|
cookiePath?: string,
|
||||||
) {
|
) {
|
||||||
// Load Facebook cookies - required for Facebook Marketplace access
|
// Load Facebook cookies - required for Facebook Marketplace access
|
||||||
const cookies = await loadFacebookCookies(cookiesSource);
|
let cookies: Cookie[];
|
||||||
|
if (cookiesSource) {
|
||||||
|
// Use provided cookie source (backward compatibility)
|
||||||
|
cookies = await loadFacebookCookies(cookiesSource);
|
||||||
|
} else {
|
||||||
|
// Auto-load from file or parse from env var
|
||||||
|
cookies = await ensureFacebookCookies(cookiePath);
|
||||||
|
}
|
||||||
|
|
||||||
if (cookies.length === 0) {
|
if (cookies.length === 0) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Facebook cookies are required for marketplace access. " +
|
"Facebook cookies are required for marketplace access. " +
|
||||||
@@ -546,8 +1057,7 @@ export default async function fetchFacebookItems(
|
|||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(
|
console.log(
|
||||||
"\n" +
|
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
`Facebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -581,7 +1091,7 @@ export default async function fetchFacebookItems(
|
|||||||
cliProgress.Presets.shades_classic,
|
cliProgress.Presets.shades_classic,
|
||||||
);
|
);
|
||||||
const totalProgress = ads.length;
|
const totalProgress = ads.length;
|
||||||
let currentProgress = 0;
|
const currentProgress = 0;
|
||||||
progressBar.start(totalProgress, currentProgress);
|
progressBar.start(totalProgress, currentProgress);
|
||||||
|
|
||||||
const items = parseFacebookAds(ads);
|
const items = parseFacebookAds(ads);
|
||||||
@@ -597,3 +1107,158 @@ export default async function fetchFacebookItems(
|
|||||||
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
|
console.log(`\nParsed ${pricedItems.length} Facebook marketplace listings.`);
|
||||||
return pricedItems.slice(0, MAX_ITEMS); // Limit results
|
return pricedItems.slice(0, MAX_ITEMS); // Limit results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch individual Facebook marketplace item details with enhanced error handling
|
||||||
|
*/
|
||||||
|
export async function fetchFacebookItem(
|
||||||
|
itemId: string,
|
||||||
|
cookiesSource?: string,
|
||||||
|
cookiePath?: string,
|
||||||
|
): Promise<ListingDetails | null> {
|
||||||
|
// Load Facebook cookies - required for Facebook Marketplace access
|
||||||
|
let cookies: Cookie[];
|
||||||
|
if (cookiesSource) {
|
||||||
|
// Use provided cookie source (backward compatibility)
|
||||||
|
cookies = await loadFacebookCookies(cookiesSource);
|
||||||
|
} else {
|
||||||
|
// Auto-load from file or parse from env var
|
||||||
|
cookies = await ensureFacebookCookies(cookiePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cookies.length === 0) {
|
||||||
|
throw new Error(
|
||||||
|
"Facebook cookies are required for marketplace access. " +
|
||||||
|
"Please provide cookies via 'cookies' parameter or create ./cookies/facebook.json file with valid Facebook session cookies.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format cookies for HTTP header
|
||||||
|
const domain = "www.facebook.com";
|
||||||
|
const cookiesHeader = formatCookiesForHeader(cookies, domain);
|
||||||
|
if (!cookiesHeader) {
|
||||||
|
throw new Error(
|
||||||
|
"No valid Facebook cookies found. Please check that cookies are not expired and apply to facebook.com domain.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const itemUrl = `https://www.facebook.com/marketplace/item/${itemId}/`;
|
||||||
|
|
||||||
|
console.log(`Fetching Facebook marketplace item: ${itemUrl}`);
|
||||||
|
|
||||||
|
let itemHtml: string;
|
||||||
|
try {
|
||||||
|
itemHtml = await fetchHtml(itemUrl, 1000, {
|
||||||
|
onRateInfo: (remaining, reset) => {
|
||||||
|
if (remaining && reset) {
|
||||||
|
console.log(
|
||||||
|
`\nFacebook - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cookies: cookiesHeader,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof HttpError) {
|
||||||
|
console.warn(
|
||||||
|
`\nFacebook marketplace item access failed (${err.status}): ${err.message}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Enhanced error handling based on status codes
|
||||||
|
switch (err.status) {
|
||||||
|
case 400:
|
||||||
|
case 401:
|
||||||
|
case 403:
|
||||||
|
console.warn(
|
||||||
|
"Authentication error: Invalid or expired cookies. Please update ./cookies/facebook.json with fresh session cookies.",
|
||||||
|
);
|
||||||
|
console.warn(
|
||||||
|
"Try logging out and back into Facebook, then export fresh cookies.",
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case 404:
|
||||||
|
console.warn(
|
||||||
|
"Listing not found: The marketplace item may have been removed, sold, or the URL is invalid.",
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case 429:
|
||||||
|
console.warn(
|
||||||
|
"Rate limited: Too many requests. Facebook is blocking access temporarily.",
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
case 500:
|
||||||
|
case 502:
|
||||||
|
case 503:
|
||||||
|
console.warn(
|
||||||
|
"Facebook server error: Marketplace may be temporarily unavailable.",
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
console.warn(`Unexpected error status: ${err.status}`);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
const itemData = extractFacebookItemData(itemHtml);
|
||||||
|
if (!itemData) {
|
||||||
|
logExtractionMetrics(false, itemId);
|
||||||
|
// Enhanced checking for specific failure scenarios
|
||||||
|
if (
|
||||||
|
itemHtml.includes("This listing is no longer available") ||
|
||||||
|
itemHtml.includes("listing has been removed") ||
|
||||||
|
itemHtml.includes("This item has been sold")
|
||||||
|
) {
|
||||||
|
console.warn(
|
||||||
|
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
itemHtml.includes("log in to Facebook") ||
|
||||||
|
itemHtml.includes("You must log in") ||
|
||||||
|
itemHtml.includes("authentication required")
|
||||||
|
) {
|
||||||
|
console.warn(
|
||||||
|
`Authentication failed for item ${itemId}. Cookies may be expired.`,
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.warn(
|
||||||
|
`No item data found in Facebook marketplace page for item ${itemId}. This may indicate:`,
|
||||||
|
);
|
||||||
|
console.warn(" - The listing was removed or sold");
|
||||||
|
console.warn(" - Authentication issues");
|
||||||
|
console.warn(" - Facebook changed their API structure");
|
||||||
|
console.warn(" - Network or parsing issues");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
logExtractionMetrics(true, itemId);
|
||||||
|
console.log(`Successfully extracted data for item ${itemId}`);
|
||||||
|
|
||||||
|
const parsedItem = parseFacebookItem(itemData);
|
||||||
|
if (!parsedItem) {
|
||||||
|
console.warn(`Failed to parse item ${itemId}: Invalid data structure`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for sold/removed status in the parsed data with proper precedence
|
||||||
|
if (itemData.is_sold) {
|
||||||
|
console.warn(`Item ${itemId} is marked as sold in the marketplace.`);
|
||||||
|
// Still return the data but mark it as sold
|
||||||
|
parsedItem.listingStatus = "SOLD";
|
||||||
|
} else if (!itemData.is_live) {
|
||||||
|
console.warn(`Item ${itemId} is not live/active in the marketplace.`);
|
||||||
|
parsedItem.listingStatus = itemData.is_hidden
|
||||||
|
? "HIDDEN"
|
||||||
|
: itemData.is_pending
|
||||||
|
? "PENDING"
|
||||||
|
: "INACTIVE";
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsedItem;
|
||||||
|
}
|
||||||
|
|||||||
107
src/index.ts
107
src/index.ts
@@ -1,6 +1,6 @@
|
|||||||
import fetchKijijiItems from "@/kijiji";
|
|
||||||
import fetchFacebookItems from "@/facebook";
|
|
||||||
import fetchEbayItems from "@/ebay";
|
import fetchEbayItems from "@/ebay";
|
||||||
|
import fetchFacebookItems from "@/facebook";
|
||||||
|
import fetchKijijiItems from "@/kijiji";
|
||||||
|
|
||||||
const PORT = process.env.PORT || 4005;
|
const PORT = process.env.PORT || 4005;
|
||||||
|
|
||||||
@@ -26,17 +26,77 @@ const server = Bun.serve({
|
|||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
|
|
||||||
const items = await fetchKijijiItems(SEARCH_QUERY, 1, undefined, {}, {
|
// Parse optional parameters with enhanced defaults
|
||||||
includeImages: true,
|
const location = reqUrl.searchParams.get("location");
|
||||||
sellerDataDepth: 'detailed',
|
const category = reqUrl.searchParams.get("category");
|
||||||
includeClientSideData: false,
|
const maxPagesParam = reqUrl.searchParams.get("maxPages");
|
||||||
});
|
const maxPages = maxPagesParam ? Number.parseInt(maxPagesParam, 10) : 5; // Default: 5 pages
|
||||||
|
const sortBy = reqUrl.searchParams.get("sortBy") as
|
||||||
|
| "relevancy"
|
||||||
|
| "date"
|
||||||
|
| "price"
|
||||||
|
| "distance"
|
||||||
|
| undefined;
|
||||||
|
const sortOrder = reqUrl.searchParams.get("sortOrder") as
|
||||||
|
| "asc"
|
||||||
|
| "desc"
|
||||||
|
| undefined;
|
||||||
|
|
||||||
|
// Build search options
|
||||||
|
const locationValue = location
|
||||||
|
? /^\d+$/.test(location)
|
||||||
|
? Number(location)
|
||||||
|
: location
|
||||||
|
: 1700272;
|
||||||
|
const categoryValue = category
|
||||||
|
? /^\d+$/.test(category)
|
||||||
|
? Number(category)
|
||||||
|
: category
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
const searchOptions: import("@/kijiji").SearchOptions = {
|
||||||
|
location: locationValue,
|
||||||
|
category: categoryValue,
|
||||||
|
keywords: SEARCH_QUERY,
|
||||||
|
sortBy: sortBy || "relevancy",
|
||||||
|
sortOrder: sortOrder || "desc",
|
||||||
|
maxPages,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build listing fetch options with enhanced defaults
|
||||||
|
const listingOptions: import("@/kijiji").ListingFetchOptions = {
|
||||||
|
includeImages: true, // Always include full image arrays
|
||||||
|
sellerDataDepth: "detailed", // Default: detailed seller info
|
||||||
|
includeClientSideData: false, // GraphQL reviews disabled by default
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const items = await fetchKijijiItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
1,
|
||||||
|
undefined,
|
||||||
|
searchOptions,
|
||||||
|
listingOptions,
|
||||||
|
);
|
||||||
if (!items || items.length === 0)
|
if (!items || items.length === 0)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Kijiji scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message: `Scraping failed: ${errorMessage}`,
|
||||||
|
query: SEARCH_QUERY,
|
||||||
|
options: { searchOptions, listingOptions },
|
||||||
|
},
|
||||||
|
{ status: 500 },
|
||||||
|
);
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
"/api/facebook": async (req: Request) => {
|
"/api/facebook": async (req: Request) => {
|
||||||
@@ -57,7 +117,14 @@ const server = Bun.serve({
|
|||||||
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const items = await fetchFacebookItems(SEARCH_QUERY, 5, LOCATION, 25, COOKIES_SOURCE);
|
const items = await fetchFacebookItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
5,
|
||||||
|
LOCATION,
|
||||||
|
25,
|
||||||
|
COOKIES_SOURCE,
|
||||||
|
"./cookies/facebook.json",
|
||||||
|
);
|
||||||
if (!items || items.length === 0)
|
if (!items || items.length === 0)
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ message: "Search didn't return any results!" },
|
{ message: "Search didn't return any results!" },
|
||||||
@@ -66,11 +133,9 @@ const server = Bun.serve({
|
|||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Facebook scraping error:", error);
|
console.error("Facebook scraping error:", error);
|
||||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
|
const errorMessage =
|
||||||
return Response.json(
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
{ message: errorMessage },
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -99,9 +164,13 @@ const server = Bun.serve({
|
|||||||
: undefined;
|
: undefined;
|
||||||
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
||||||
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
||||||
const exclusions = exclusionsParam ? exclusionsParam.split(",").map(s => s.trim()) : [];
|
const exclusions = exclusionsParam
|
||||||
|
? exclusionsParam.split(",").map((s) => s.trim())
|
||||||
|
: [];
|
||||||
const keywordsParam = reqUrl.searchParams.get("keywords");
|
const keywordsParam = reqUrl.searchParams.get("keywords");
|
||||||
const keywords = keywordsParam ? keywordsParam.split(",").map(s => s.trim()) : [SEARCH_QUERY];
|
const keywords = keywordsParam
|
||||||
|
? keywordsParam.split(",").map((s) => s.trim())
|
||||||
|
: [SEARCH_QUERY];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const items = await fetchEbayItems(SEARCH_QUERY, 5, {
|
const items = await fetchEbayItems(SEARCH_QUERY, 5, {
|
||||||
@@ -119,11 +188,9 @@ const server = Bun.serve({
|
|||||||
return Response.json(items, { status: 200 });
|
return Response.json(items, { status: 200 });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("eBay scraping error:", error);
|
console.error("eBay scraping error:", error);
|
||||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
|
const errorMessage =
|
||||||
return Response.json(
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
{ message: errorMessage },
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|||||||
294
src/kijiji.ts
294
src/kijiji.ts
@@ -1,7 +1,7 @@
|
|||||||
|
import cliProgress from "cli-progress";
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
import unidecode from "unidecode";
|
import unidecode from "unidecode";
|
||||||
import cliProgress from "cli-progress";
|
|
||||||
|
|
||||||
// const unidecode = require("unidecode");
|
// const unidecode = require("unidecode");
|
||||||
|
|
||||||
@@ -99,8 +99,8 @@ interface SearchOptions {
|
|||||||
location?: number | string; // Location ID or name
|
location?: number | string; // Location ID or name
|
||||||
category?: number | string; // Category ID or name
|
category?: number | string; // Category ID or name
|
||||||
keywords?: string;
|
keywords?: string;
|
||||||
sortBy?: 'relevancy' | 'date' | 'price' | 'distance';
|
sortBy?: "relevancy" | "date" | "price" | "distance";
|
||||||
sortOrder?: 'desc' | 'asc';
|
sortOrder?: "desc" | "asc";
|
||||||
maxPages?: number; // Default: 5
|
maxPages?: number; // Default: 5
|
||||||
priceMin?: number;
|
priceMin?: number;
|
||||||
priceMax?: number;
|
priceMax?: number;
|
||||||
@@ -108,7 +108,7 @@ interface SearchOptions {
|
|||||||
|
|
||||||
interface ListingFetchOptions {
|
interface ListingFetchOptions {
|
||||||
includeImages?: boolean; // Default: true
|
includeImages?: boolean; // Default: true
|
||||||
sellerDataDepth?: 'basic' | 'detailed' | 'full'; // Default: 'detailed'
|
sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed'
|
||||||
includeClientSideData?: boolean; // Default: false
|
includeClientSideData?: boolean; // Default: false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,65 +116,65 @@ interface ListingFetchOptions {
|
|||||||
|
|
||||||
// Location mappings from KIJIJI.md
|
// Location mappings from KIJIJI.md
|
||||||
const LOCATION_MAPPINGS: Record<string, number> = {
|
const LOCATION_MAPPINGS: Record<string, number> = {
|
||||||
'canada': 0,
|
canada: 0,
|
||||||
'ontario': 9004,
|
ontario: 9004,
|
||||||
'toronto': 1700273,
|
toronto: 1700273,
|
||||||
'gta': 1700272,
|
gta: 1700272,
|
||||||
'oshawa': 1700275,
|
oshawa: 1700275,
|
||||||
'quebec': 9001,
|
quebec: 9001,
|
||||||
'nova scotia': 9002,
|
"nova scotia": 9002,
|
||||||
'alberta': 9003,
|
alberta: 9003,
|
||||||
'new brunswick': 9005,
|
"new brunswick": 9005,
|
||||||
'manitoba': 9006,
|
manitoba: 9006,
|
||||||
'british columbia': 9007,
|
"british columbia": 9007,
|
||||||
'newfoundland': 9008,
|
newfoundland: 9008,
|
||||||
'saskatchewan': 9009,
|
saskatchewan: 9009,
|
||||||
'territories': 9010,
|
territories: 9010,
|
||||||
'pei': 9011,
|
pei: 9011,
|
||||||
'prince edward island': 9011,
|
"prince edward island": 9011,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Category mappings from KIJIJI.md (Buy & Sell main categories)
|
// Category mappings from KIJIJI.md (Buy & Sell main categories)
|
||||||
const CATEGORY_MAPPINGS: Record<string, number> = {
|
const CATEGORY_MAPPINGS: Record<string, number> = {
|
||||||
'all': 0,
|
all: 0,
|
||||||
'buy-sell': 10,
|
"buy-sell": 10,
|
||||||
'arts-collectibles': 12,
|
"arts-collectibles": 12,
|
||||||
'audio': 767,
|
audio: 767,
|
||||||
'baby-items': 253,
|
"baby-items": 253,
|
||||||
'bags-luggage': 931,
|
"bags-luggage": 931,
|
||||||
'bikes': 644,
|
bikes: 644,
|
||||||
'books': 109,
|
books: 109,
|
||||||
'cameras': 103,
|
cameras: 103,
|
||||||
'cds': 104,
|
cds: 104,
|
||||||
'clothing': 274,
|
clothing: 274,
|
||||||
'computers': 16,
|
computers: 16,
|
||||||
'computer-accessories': 128,
|
"computer-accessories": 128,
|
||||||
'electronics': 29659001,
|
electronics: 29659001,
|
||||||
'free-stuff': 17220001,
|
"free-stuff": 17220001,
|
||||||
'furniture': 235,
|
furniture: 235,
|
||||||
'garage-sales': 638,
|
"garage-sales": 638,
|
||||||
'health-special-needs': 140,
|
"health-special-needs": 140,
|
||||||
'hobbies-crafts': 139,
|
"hobbies-crafts": 139,
|
||||||
'home-appliances': 107,
|
"home-appliances": 107,
|
||||||
'home-indoor': 717,
|
"home-indoor": 717,
|
||||||
'home-outdoor': 727,
|
"home-outdoor": 727,
|
||||||
'jewellery': 133,
|
jewellery: 133,
|
||||||
'musical-instruments': 17,
|
"musical-instruments": 17,
|
||||||
'phones': 132,
|
phones: 132,
|
||||||
'sporting-goods': 111,
|
"sporting-goods": 111,
|
||||||
'tools': 110,
|
tools: 110,
|
||||||
'toys-games': 108,
|
"toys-games": 108,
|
||||||
'tvs-video': 15093001,
|
"tvs-video": 15093001,
|
||||||
'video-games': 141,
|
"video-games": 141,
|
||||||
'other': 26,
|
other: 26,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Sort parameter mappings
|
// Sort parameter mappings
|
||||||
const SORT_MAPPINGS: Record<string, string> = {
|
const SORT_MAPPINGS: Record<string, string> = {
|
||||||
'relevancy': 'MATCH',
|
relevancy: "MATCH",
|
||||||
'date': 'DATE',
|
date: "DATE",
|
||||||
'price': 'PRICE',
|
price: "PRICE",
|
||||||
'distance': 'DISTANCE',
|
distance: "DISTANCE",
|
||||||
};
|
};
|
||||||
|
|
||||||
// ----------------------------- Exports for Testing -----------------------------
|
// ----------------------------- Exports for Testing -----------------------------
|
||||||
@@ -193,9 +193,9 @@ const SEPS = new Set([" ", "–", "—", "/", ":", ";", ",", ".", "-"]);
|
|||||||
* Resolve location ID from name or return numeric ID
|
* Resolve location ID from name or return numeric ID
|
||||||
*/
|
*/
|
||||||
function resolveLocationId(location?: number | string): number {
|
function resolveLocationId(location?: number | string): number {
|
||||||
if (typeof location === 'number') return location;
|
if (typeof location === "number") return location;
|
||||||
if (typeof location === 'string') {
|
if (typeof location === "string") {
|
||||||
const normalized = location.toLowerCase().replace(/\s+/g, '-');
|
const normalized = location.toLowerCase().replace(/\s+/g, "-");
|
||||||
return LOCATION_MAPPINGS[normalized] ?? 0; // Default to Canada (0)
|
return LOCATION_MAPPINGS[normalized] ?? 0; // Default to Canada (0)
|
||||||
}
|
}
|
||||||
return 0; // Default to Canada
|
return 0; // Default to Canada
|
||||||
@@ -205,9 +205,9 @@ function resolveLocationId(location?: number | string): number {
|
|||||||
* Resolve category ID from name or return numeric ID
|
* Resolve category ID from name or return numeric ID
|
||||||
*/
|
*/
|
||||||
function resolveCategoryId(category?: number | string): number {
|
function resolveCategoryId(category?: number | string): number {
|
||||||
if (typeof category === 'number') return category;
|
if (typeof category === "number") return category;
|
||||||
if (typeof category === 'string') {
|
if (typeof category === "string") {
|
||||||
const normalized = category.toLowerCase().replace(/\s+/g, '-');
|
const normalized = category.toLowerCase().replace(/\s+/g, "-");
|
||||||
return CATEGORY_MAPPINGS[normalized] ?? 0; // Default to all categories
|
return CATEGORY_MAPPINGS[normalized] ?? 0; // Default to all categories
|
||||||
}
|
}
|
||||||
return 0; // Default to all categories
|
return 0; // Default to all categories
|
||||||
@@ -219,19 +219,22 @@ function resolveCategoryId(category?: number | string): number {
|
|||||||
function buildSearchUrl(
|
function buildSearchUrl(
|
||||||
keywords: string,
|
keywords: string,
|
||||||
options: SearchOptions & { page?: number },
|
options: SearchOptions & { page?: number },
|
||||||
BASE_URL = "https://www.kijiji.ca"
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
): string {
|
): string {
|
||||||
const locationId = resolveLocationId(options.location);
|
const locationId = resolveLocationId(options.location);
|
||||||
const categoryId = resolveCategoryId(options.category);
|
const categoryId = resolveCategoryId(options.category);
|
||||||
|
|
||||||
const categorySlug = categoryId === 0 ? 'buy-sell' : 'buy-sell'; // Could be enhanced
|
const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell"; // Could be enhanced
|
||||||
const locationSlug = locationId === 0 ? 'canada' : 'canada'; // Could be enhanced
|
const locationSlug = locationId === 0 ? "canada" : "canada"; // Could be enhanced
|
||||||
|
|
||||||
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
|
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
|
||||||
|
|
||||||
const sortParam = options.sortBy ? `&sort=${SORT_MAPPINGS[options.sortBy]}` : '';
|
const sortParam = options.sortBy
|
||||||
const sortOrder = options.sortOrder === 'asc' ? 'ASC' : 'DESC';
|
? `&sort=${SORT_MAPPINGS[options.sortBy]}`
|
||||||
const pageParam = options.page && options.page > 1 ? `&page=${options.page}` : '';
|
: "";
|
||||||
|
const sortOrder = options.sortOrder === "asc" ? "ASC" : "DESC";
|
||||||
|
const pageParam =
|
||||||
|
options.page && options.page > 1 ? `&page=${options.page}` : "";
|
||||||
|
|
||||||
url += `?sort=relevancyDesc&view=list${sortParam}&order=${sortOrder}${pageParam}`;
|
url += `?sort=relevancyDesc&view=list${sortParam}&order=${sortOrder}${pageParam}`;
|
||||||
|
|
||||||
@@ -278,8 +281,8 @@ export function formatCentsToCurrency(
|
|||||||
if (Number.isNaN(cents)) return "";
|
if (Number.isNaN(cents)) return "";
|
||||||
const dollars = cents / 100;
|
const dollars = cents / 100;
|
||||||
const formatter = new Intl.NumberFormat(locale, {
|
const formatter = new Intl.NumberFormat(locale, {
|
||||||
style: 'currency',
|
style: "currency",
|
||||||
currency: 'USD',
|
currency: "USD",
|
||||||
minimumFractionDigits: 2,
|
minimumFractionDigits: 2,
|
||||||
maximumFractionDigits: 2,
|
maximumFractionDigits: 2,
|
||||||
});
|
});
|
||||||
@@ -394,7 +397,9 @@ async function fetchHtml(
|
|||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
// Handle rate limiting
|
// Handle rate limiting
|
||||||
if (res.status === 429) {
|
if (res.status === 429) {
|
||||||
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : Number.NaN;
|
const resetSeconds = rateLimitReset
|
||||||
|
? Number(rateLimitReset)
|
||||||
|
: Number.NaN;
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
const waitMs = Number.isFinite(resetSeconds)
|
||||||
? Math.max(0, resetSeconds * 1000)
|
? Math.max(0, resetSeconds * 1000)
|
||||||
: calculateBackoffDelay(attempt, retryBaseMs);
|
: calculateBackoffDelay(attempt, retryBaseMs);
|
||||||
@@ -428,14 +433,13 @@ async function fetchHtml(
|
|||||||
// Respect per-request delay to maintain rate limiting
|
// Respect per-request delay to maintain rate limiting
|
||||||
await delay(DELAY_MS);
|
await delay(DELAY_MS);
|
||||||
return html;
|
return html;
|
||||||
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Handle different error types
|
// Handle different error types
|
||||||
if (err instanceof RateLimitError || err instanceof HttpError) {
|
if (err instanceof RateLimitError || err instanceof HttpError) {
|
||||||
throw err; // Re-throw known errors
|
throw err; // Re-throw known errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if (err instanceof Error && err.name === 'AbortError') {
|
if (err instanceof Error && err.name === "AbortError") {
|
||||||
if (attempt < maxRetries) {
|
if (attempt < maxRetries) {
|
||||||
await delay(calculateBackoffDelay(attempt, retryBaseMs));
|
await delay(calculateBackoffDelay(attempt, retryBaseMs));
|
||||||
continue;
|
continue;
|
||||||
@@ -451,7 +455,7 @@ async function fetchHtml(
|
|||||||
throw new NetworkError(
|
throw new NetworkError(
|
||||||
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
url,
|
url,
|
||||||
err instanceof Error ? err : undefined
|
err instanceof Error ? err : undefined,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -463,7 +467,7 @@ async function fetchHtml(
|
|||||||
* Calculate exponential backoff delay with jitter
|
* Calculate exponential backoff delay with jitter
|
||||||
*/
|
*/
|
||||||
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
||||||
const exponentialDelay = baseMs * (2 ** attempt);
|
const exponentialDelay = baseMs * 2 ** attempt;
|
||||||
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
||||||
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
||||||
}
|
}
|
||||||
@@ -476,16 +480,16 @@ function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
|||||||
async function fetchGraphQLData(
|
async function fetchGraphQLData(
|
||||||
query: string,
|
query: string,
|
||||||
variables: Record<string, unknown>,
|
variables: Record<string, unknown>,
|
||||||
BASE_URL = "https://www.kijiji.ca"
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
): Promise<unknown> {
|
): Promise<unknown> {
|
||||||
const endpoint = `${BASE_URL}/anvil/api`;
|
const endpoint = `${BASE_URL}/anvil/api`;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(endpoint, {
|
const response = await fetch(endpoint, {
|
||||||
method: 'POST',
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
"Content-Type": "application/json",
|
||||||
'apollo-require-preflight': 'true',
|
"apollo-require-preflight": "true",
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
query,
|
query,
|
||||||
@@ -497,14 +501,17 @@ async function fetchGraphQLData(
|
|||||||
throw new HttpError(
|
throw new HttpError(
|
||||||
`GraphQL request failed with status ${response.status}`,
|
`GraphQL request failed with status ${response.status}`,
|
||||||
response.status,
|
response.status,
|
||||||
endpoint
|
endpoint,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
|
||||||
if (result.errors) {
|
if (result.errors) {
|
||||||
throw new ParseError(`GraphQL errors: ${JSON.stringify(result.errors)}`, result.errors);
|
throw new ParseError(
|
||||||
|
`GraphQL errors: ${JSON.stringify(result.errors)}`,
|
||||||
|
result.errors,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.data;
|
return result.data;
|
||||||
@@ -515,7 +522,7 @@ async function fetchGraphQLData(
|
|||||||
throw new NetworkError(
|
throw new NetworkError(
|
||||||
`Failed to fetch GraphQL data: ${err instanceof Error ? err.message : String(err)}`,
|
`Failed to fetch GraphQL data: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
endpoint,
|
endpoint,
|
||||||
err instanceof Error ? err : undefined
|
err instanceof Error ? err : undefined,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -567,12 +574,25 @@ const GRAPHQL_QUERIES = {
|
|||||||
*/
|
*/
|
||||||
async function fetchSellerDetails(
|
async function fetchSellerDetails(
|
||||||
posterId: string,
|
posterId: string,
|
||||||
BASE_URL = "https://www.kijiji.ca"
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
): Promise<{ reviewCount?: number; reviewScore?: number; memberSince?: string; accountType?: string }> {
|
): Promise<{
|
||||||
|
reviewCount?: number;
|
||||||
|
reviewScore?: number;
|
||||||
|
memberSince?: string;
|
||||||
|
accountType?: string;
|
||||||
|
}> {
|
||||||
try {
|
try {
|
||||||
const [reviewData, profileData] = await Promise.all([
|
const [reviewData, profileData] = await Promise.all([
|
||||||
fetchGraphQLData(GRAPHQL_QUERIES.getReviewSummary, { userId: posterId }, BASE_URL),
|
fetchGraphQLData(
|
||||||
fetchGraphQLData(GRAPHQL_QUERIES.getProfileMetrics, { profileId: posterId }, BASE_URL),
|
GRAPHQL_QUERIES.getReviewSummary,
|
||||||
|
{ userId: posterId },
|
||||||
|
BASE_URL,
|
||||||
|
),
|
||||||
|
fetchGraphQLData(
|
||||||
|
GRAPHQL_QUERIES.getProfileMetrics,
|
||||||
|
{ profileId: posterId },
|
||||||
|
BASE_URL,
|
||||||
|
),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const reviewResponse = reviewData as GraphQLReviewResponse;
|
const reviewResponse = reviewData as GraphQLReviewResponse;
|
||||||
@@ -586,7 +606,10 @@ async function fetchSellerDetails(
|
|||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Silently fail for GraphQL errors - not critical for basic functionality
|
// Silently fail for GraphQL errors - not critical for basic functionality
|
||||||
console.warn(`Failed to fetch seller details for ${posterId}:`, err instanceof Error ? err.message : String(err));
|
console.warn(
|
||||||
|
`Failed to fetch seller details for ${posterId}:`,
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
);
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -694,7 +717,8 @@ function parseListing(
|
|||||||
listingPrice: amountFormatted
|
listingPrice: amountFormatted
|
||||||
? {
|
? {
|
||||||
amountFormatted,
|
amountFormatted,
|
||||||
cents: cents !== undefined && Number.isFinite(cents) ? cents : undefined,
|
cents:
|
||||||
|
cents !== undefined && Number.isFinite(cents) ? cents : undefined,
|
||||||
currency: price?.currency,
|
currency: price?.currency,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
@@ -702,7 +726,10 @@ function parseListing(
|
|||||||
listingStatus: status,
|
listingStatus: status,
|
||||||
creationDate: activationDate,
|
creationDate: activationDate,
|
||||||
endDate,
|
endDate,
|
||||||
numberOfViews: numberOfViews !== undefined && Number.isFinite(numberOfViews) ? numberOfViews : undefined,
|
numberOfViews:
|
||||||
|
numberOfViews !== undefined && Number.isFinite(numberOfViews)
|
||||||
|
? numberOfViews
|
||||||
|
: undefined,
|
||||||
address: location?.address ?? null,
|
address: location?.address ?? null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -713,7 +740,7 @@ function parseListing(
|
|||||||
async function parseDetailedListing(
|
async function parseDetailedListing(
|
||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
BASE_URL: string,
|
BASE_URL: string,
|
||||||
options: ListingFetchOptions = {}
|
options: ListingFetchOptions = {},
|
||||||
): Promise<DetailedListing | null> {
|
): Promise<DetailedListing | null> {
|
||||||
const apolloState = extractApolloState(htmlString);
|
const apolloState = extractApolloState(htmlString);
|
||||||
if (!apolloState) return null;
|
if (!apolloState) return null;
|
||||||
@@ -766,8 +793,9 @@ async function parseDetailedListing(
|
|||||||
if (!amountFormatted || cents === undefined) return null;
|
if (!amountFormatted || cents === undefined) return null;
|
||||||
|
|
||||||
// Extract images if requested
|
// Extract images if requested
|
||||||
const images = options.includeImages !== false && Array.isArray(imageUrls)
|
const images =
|
||||||
? imageUrls.filter((url): url is string => typeof url === 'string')
|
options.includeImages !== false && Array.isArray(imageUrls)
|
||||||
|
? imageUrls.filter((url): url is string => typeof url === "string")
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
// Extract attributes as key-value pairs
|
// Extract attributes as key-value pairs
|
||||||
@@ -781,26 +809,35 @@ async function parseDetailedListing(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Extract seller info based on depth setting
|
// Extract seller info based on depth setting
|
||||||
let sellerInfo: DetailedListing['sellerInfo'];
|
let sellerInfo: DetailedListing["sellerInfo"];
|
||||||
const depth = options.sellerDataDepth ?? 'detailed';
|
const depth = options.sellerDataDepth ?? "detailed";
|
||||||
|
|
||||||
if (posterInfo?.posterId) {
|
if (posterInfo?.posterId) {
|
||||||
sellerInfo = {
|
sellerInfo = {
|
||||||
posterId: posterInfo.posterId,
|
posterId: posterInfo.posterId,
|
||||||
rating: typeof posterInfo.rating === 'number' ? posterInfo.rating : undefined,
|
rating:
|
||||||
|
typeof posterInfo.rating === "number" ? posterInfo.rating : undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Add more detailed info if requested and client-side data is enabled
|
// Add more detailed info if requested and client-side data is enabled
|
||||||
if ((depth === 'detailed' || depth === 'full') && options.includeClientSideData) {
|
if (
|
||||||
|
(depth === "detailed" || depth === "full") &&
|
||||||
|
options.includeClientSideData
|
||||||
|
) {
|
||||||
try {
|
try {
|
||||||
const additionalData = await fetchSellerDetails(posterInfo.posterId, BASE_URL);
|
const additionalData = await fetchSellerDetails(
|
||||||
|
posterInfo.posterId,
|
||||||
|
BASE_URL,
|
||||||
|
);
|
||||||
sellerInfo = {
|
sellerInfo = {
|
||||||
...sellerInfo,
|
...sellerInfo,
|
||||||
...additionalData,
|
...additionalData,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Silently fail - GraphQL data is optional
|
// Silently fail - GraphQL data is optional
|
||||||
console.warn(`Failed to fetch additional seller data for ${posterInfo.posterId}`);
|
console.warn(
|
||||||
|
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -818,23 +855,28 @@ async function parseDetailedListing(
|
|||||||
listingStatus: status,
|
listingStatus: status,
|
||||||
creationDate: activationDate,
|
creationDate: activationDate,
|
||||||
endDate,
|
endDate,
|
||||||
numberOfViews: numberOfViews !== undefined && Number.isFinite(numberOfViews) ? numberOfViews : undefined,
|
numberOfViews:
|
||||||
|
numberOfViews !== undefined && Number.isFinite(numberOfViews)
|
||||||
|
? numberOfViews
|
||||||
|
: undefined,
|
||||||
address: location?.address ?? null,
|
address: location?.address ?? null,
|
||||||
images,
|
images,
|
||||||
categoryId: typeof categoryId === 'number' ? categoryId : 0,
|
categoryId: typeof categoryId === "number" ? categoryId : 0,
|
||||||
adSource: typeof adSource === 'string' ? adSource : 'UNKNOWN',
|
adSource: typeof adSource === "string" ? adSource : "UNKNOWN",
|
||||||
flags: {
|
flags: {
|
||||||
topAd: flags?.topAd === true,
|
topAd: flags?.topAd === true,
|
||||||
priceDrop: flags?.priceDrop === true,
|
priceDrop: flags?.priceDrop === true,
|
||||||
},
|
},
|
||||||
attributes: attributeMap,
|
attributes: attributeMap,
|
||||||
location: {
|
location: {
|
||||||
id: typeof location?.id === 'number' ? location.id : 0,
|
id: typeof location?.id === "number" ? location.id : 0,
|
||||||
name: typeof location?.name === 'string' ? location.name : 'Unknown',
|
name: typeof location?.name === "string" ? location.name : "Unknown",
|
||||||
coordinates: location?.coordinates ? {
|
coordinates: location?.coordinates
|
||||||
|
? {
|
||||||
latitude: location.coordinates.latitude,
|
latitude: location.coordinates.latitude,
|
||||||
longitude: location.coordinates.longitude,
|
longitude: location.coordinates.longitude,
|
||||||
} : undefined,
|
}
|
||||||
|
: undefined,
|
||||||
},
|
},
|
||||||
sellerInfo,
|
sellerInfo,
|
||||||
};
|
};
|
||||||
@@ -856,8 +898,8 @@ export default async function fetchKijijiItems(
|
|||||||
location: searchOptions.location ?? 1700272, // Default to GTA
|
location: searchOptions.location ?? 1700272, // Default to GTA
|
||||||
category: searchOptions.category ?? 0, // Default to all categories
|
category: searchOptions.category ?? 0, // Default to all categories
|
||||||
keywords: searchOptions.keywords ?? SEARCH_QUERY,
|
keywords: searchOptions.keywords ?? SEARCH_QUERY,
|
||||||
sortBy: searchOptions.sortBy ?? 'relevancy',
|
sortBy: searchOptions.sortBy ?? "relevancy",
|
||||||
sortOrder: searchOptions.sortOrder ?? 'desc',
|
sortOrder: searchOptions.sortOrder ?? "desc",
|
||||||
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
|
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
|
||||||
priceMin: searchOptions.priceMin,
|
priceMin: searchOptions.priceMin,
|
||||||
priceMax: searchOptions.priceMax,
|
priceMax: searchOptions.priceMax,
|
||||||
@@ -865,7 +907,7 @@ export default async function fetchKijijiItems(
|
|||||||
|
|
||||||
const finalListingOptions: Required<ListingFetchOptions> = {
|
const finalListingOptions: Required<ListingFetchOptions> = {
|
||||||
includeImages: listingOptions.includeImages ?? true,
|
includeImages: listingOptions.includeImages ?? true,
|
||||||
sellerDataDepth: listingOptions.sellerDataDepth ?? 'detailed',
|
sellerDataDepth: listingOptions.sellerDataDepth ?? "detailed",
|
||||||
includeClientSideData: listingOptions.includeClientSideData ?? false,
|
includeClientSideData: listingOptions.includeClientSideData ?? false,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -874,24 +916,32 @@ export default async function fetchKijijiItems(
|
|||||||
|
|
||||||
// Fetch multiple pages
|
// Fetch multiple pages
|
||||||
for (let page = 1; page <= finalSearchOptions.maxPages; page++) {
|
for (let page = 1; page <= finalSearchOptions.maxPages; page++) {
|
||||||
const searchUrl = buildSearchUrl(finalSearchOptions.keywords, {
|
const searchUrl = buildSearchUrl(
|
||||||
|
finalSearchOptions.keywords,
|
||||||
|
{
|
||||||
...finalSearchOptions,
|
...finalSearchOptions,
|
||||||
// Add page parameter for pagination
|
// Add page parameter for pagination
|
||||||
...(page > 1 && { page }),
|
...(page > 1 && { page }),
|
||||||
}, BASE_URL);
|
},
|
||||||
|
BASE_URL,
|
||||||
|
);
|
||||||
|
|
||||||
console.log(`Fetching search page ${page}: ${searchUrl}`);
|
console.log(`Fetching search page ${page}: ${searchUrl}`);
|
||||||
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`);
|
console.log(
|
||||||
|
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const searchResults = parseSearch(searchHtml, BASE_URL);
|
const searchResults = parseSearch(searchHtml, BASE_URL);
|
||||||
if (searchResults.length === 0) {
|
if (searchResults.length === 0) {
|
||||||
console.log(`No more results found on page ${page}. Stopping pagination.`);
|
console.log(
|
||||||
|
`No more results found on page ${page}. Stopping pagination.`,
|
||||||
|
);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -904,7 +954,9 @@ export default async function fetchKijijiItems(
|
|||||||
seenUrls.add(link);
|
seenUrls.add(link);
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`);
|
console.log(
|
||||||
|
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
|
||||||
|
);
|
||||||
|
|
||||||
// Fetch details for this page's listings
|
// Fetch details for this page's listings
|
||||||
const progressBar = new cliProgress.SingleBar(
|
const progressBar = new cliProgress.SingleBar(
|
||||||
@@ -920,19 +972,29 @@ export default async function fetchKijijiItems(
|
|||||||
const html = await fetchHtml(link, DELAY_MS, {
|
const html = await fetchHtml(link, DELAY_MS, {
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`);
|
console.log(
|
||||||
|
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const parsed = await parseDetailedListing(html, BASE_URL, finalListingOptions);
|
const parsed = await parseDetailedListing(
|
||||||
|
html,
|
||||||
|
BASE_URL,
|
||||||
|
finalListingOptions,
|
||||||
|
);
|
||||||
if (parsed) {
|
if (parsed) {
|
||||||
allListings.push(parsed);
|
allListings.push(parsed);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof HttpError) {
|
if (err instanceof HttpError) {
|
||||||
console.error(`\nFailed to fetch ${link}\n - ${err.status} ${err.message}`);
|
console.error(
|
||||||
|
`\nFailed to fetch ${link}\n - ${err.status} ${err.message}`,
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
console.error(`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`);
|
console.error(
|
||||||
|
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
currentProgress++;
|
currentProgress++;
|
||||||
|
|||||||
834
test/facebook-core.test.ts
Normal file
834
test/facebook-core.test.ts
Normal file
@@ -0,0 +1,834 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import {
|
||||||
|
extractFacebookItemData,
|
||||||
|
extractFacebookMarketplaceData,
|
||||||
|
fetchFacebookItem,
|
||||||
|
formatCentsToCurrency,
|
||||||
|
formatCookiesForHeader,
|
||||||
|
loadFacebookCookies,
|
||||||
|
parseFacebookAds,
|
||||||
|
parseFacebookCookieString,
|
||||||
|
parseFacebookItem,
|
||||||
|
} from "../src/facebook";
|
||||||
|
|
||||||
|
// Mock fetch globally
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Cookie Parsing", () => {
|
||||||
|
describe("parseFacebookCookieString", () => {
|
||||||
|
test("should parse valid cookie string", () => {
|
||||||
|
const cookieString = "c_user=123456789; xs=abcdef123456; fr=xyz789";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result).toHaveLength(3);
|
||||||
|
expect(result[0]).toEqual({
|
||||||
|
name: "c_user",
|
||||||
|
value: "123456789",
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
secure: true,
|
||||||
|
httpOnly: false,
|
||||||
|
sameSite: "lax",
|
||||||
|
expirationDate: undefined,
|
||||||
|
});
|
||||||
|
expect(result[1]).toEqual({
|
||||||
|
name: "xs",
|
||||||
|
value: "abcdef123456",
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
secure: true,
|
||||||
|
httpOnly: false,
|
||||||
|
sameSite: "lax",
|
||||||
|
expirationDate: undefined,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle URL-encoded values", () => {
|
||||||
|
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result[0].value).toBe("123+456");
|
||||||
|
expect(result[1].value).toBe("abc=def");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out malformed cookies", () => {
|
||||||
|
const cookieString = "c_user=123; invalid; xs=abc; =empty";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result.map((c) => c.name)).toEqual(["c_user", "xs"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty input", () => {
|
||||||
|
expect(parseFacebookCookieString("")).toEqual([]);
|
||||||
|
expect(parseFacebookCookieString(" ")).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle extra whitespace", () => {
|
||||||
|
const cookieString = " c_user = 123 ; xs=abc ";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result[0].name).toBe("c_user");
|
||||||
|
expect(result[0].value).toBe("123");
|
||||||
|
expect(result[1].name).toBe("xs");
|
||||||
|
expect(result[1].value).toBe("abc");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Facebook Item Fetching", () => {
|
||||||
|
describe("fetchFacebookItem", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should handle authentication errors", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 401,
|
||||||
|
text: () => Promise.resolve("Authentication required"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("123", mockCookies);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle item not found", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
text: () => Promise.resolve("Not found"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("nonexistent", mockCookies);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle rate limiting", async () => {
|
||||||
|
let attempts = 0;
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
attempts++;
|
||||||
|
if (attempts === 1) {
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 429,
|
||||||
|
headers: {
|
||||||
|
get: (header: string) => {
|
||||||
|
if (header === "X-RateLimit-Reset") return "1";
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
text: () => Promise.resolve("Rate limited"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: {
|
||||||
|
id: "123",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Test Item",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("123", mockCookies);
|
||||||
|
expect(attempts).toBe(2);
|
||||||
|
// Should eventually succeed after retry
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle sold items", async () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: {
|
||||||
|
id: "456",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Sold Item",
|
||||||
|
is_sold: true,
|
||||||
|
is_live: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("456", mockCookies);
|
||||||
|
expect(result?.listingStatus).toBe("SOLD");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing authentication cookies", async () => {
|
||||||
|
// Use a test-specific cookie file that doesn't exist
|
||||||
|
const testCookiePath = "./cookies/facebook-test.json";
|
||||||
|
|
||||||
|
// Test with no cookies available (test file doesn't exist)
|
||||||
|
await expect(
|
||||||
|
fetchFacebookItem("123", undefined, testCookiePath),
|
||||||
|
).rejects.toThrow("No valid Facebook cookies found");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle successful item extraction", async () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: {
|
||||||
|
id: "789",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Working Item",
|
||||||
|
formatted_price: { text: "$299.00" },
|
||||||
|
listing_price: {
|
||||||
|
amount: "299.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
creation_time: 1640995200,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("789", mockCookies);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("Working Item");
|
||||||
|
expect(result?.listingPrice?.amountFormatted).toBe("$299.00");
|
||||||
|
expect(result?.listingStatus).toBe("ACTIVE");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle server errors", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 500,
|
||||||
|
text: () => Promise.resolve("Internal Server Error"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("error", mockCookies);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Data Extraction", () => {
|
||||||
|
describe("extractFacebookItemData", () => {
|
||||||
|
test("should extract item data from standard require structure", () => {
|
||||||
|
const mockItemData = {
|
||||||
|
id: "123456",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Test Item",
|
||||||
|
formatted_price: { text: "$100.00" },
|
||||||
|
listing_price: { amount: "100.00", currency: "CAD" },
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: mockItemData,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.id).toBe("123456");
|
||||||
|
expect(result?.marketplace_listing_title).toBe("Test Item");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing item data", () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle malformed HTML", () => {
|
||||||
|
const result = extractFacebookItemData(
|
||||||
|
"<html><body>Invalid HTML</body></html>",
|
||||||
|
);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle invalid JSON in script tags", () => {
|
||||||
|
const html =
|
||||||
|
"<html><body><script>{invalid: json}</script></body></html>";
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should extract item with vehicle data", () => {
|
||||||
|
const mockVehicleItem = {
|
||||||
|
id: "789",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "2006 Honda Civic",
|
||||||
|
formatted_price: { text: "$5,000" },
|
||||||
|
listing_price: { amount: "5000.00", currency: "CAD" },
|
||||||
|
vehicle_make_display_name: "Honda",
|
||||||
|
vehicle_model_display_name: "Civic",
|
||||||
|
vehicle_odometer_data: { unit: "KILOMETERS", value: 150000 },
|
||||||
|
vehicle_transmission_type: "AUTOMATIC",
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: mockVehicleItem,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.vehicle_make_display_name).toBe("Honda");
|
||||||
|
expect(result?.vehicle_odometer_data?.value).toBe(150000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("extractFacebookMarketplaceData", () => {
|
||||||
|
test("should extract search results from marketplace data", () => {
|
||||||
|
const mockMarketplaceData = {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Item 1",
|
||||||
|
listing_price: { amount: "10.00", currency: "CAD" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "Item 2",
|
||||||
|
listing_price: { amount: "20.00", currency: "CAD" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: mockMarketplaceData,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookMarketplaceData(html);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
|
||||||
|
"Item 1",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty search results", () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: { edges: [] },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookMarketplaceData(html);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Data Parsing", () => {
|
||||||
|
describe("parseFacebookItem", () => {
|
||||||
|
test("should parse complete item with all fields", () => {
|
||||||
|
const item = {
|
||||||
|
id: "123456",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "iPhone 13 Pro",
|
||||||
|
redacted_description: { text: "Excellent condition" },
|
||||||
|
formatted_price: { text: "$800.00" },
|
||||||
|
listing_price: { amount: "800.00", currency: "CAD" },
|
||||||
|
location_text: { text: "Toronto, ON" },
|
||||||
|
is_live: true,
|
||||||
|
creation_time: 1640995200,
|
||||||
|
marketplace_listing_seller: {
|
||||||
|
id: "seller1",
|
||||||
|
name: "John Doe",
|
||||||
|
},
|
||||||
|
delivery_types: ["IN_PERSON"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(item);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("iPhone 13 Pro");
|
||||||
|
expect(result?.description).toBe("Excellent condition");
|
||||||
|
expect(result?.listingPrice?.amountFormatted).toBe("$800.00");
|
||||||
|
expect(result?.listingPrice?.cents).toBe(80000);
|
||||||
|
expect(result?.listingPrice?.currency).toBe("CAD");
|
||||||
|
expect(result?.address).toBe("Toronto, ON");
|
||||||
|
expect(result?.listingStatus).toBe("ACTIVE");
|
||||||
|
expect(result?.seller?.name).toBe("John Doe");
|
||||||
|
expect(result?.deliveryTypes).toEqual(["IN_PERSON"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should parse FREE items", () => {
|
||||||
|
const item = {
|
||||||
|
id: "789",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Free Sofa",
|
||||||
|
formatted_price: { text: "FREE" },
|
||||||
|
listing_price: { amount: "0.00", currency: "CAD" },
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(item);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("Free Sofa");
|
||||||
|
expect(result?.listingPrice?.amountFormatted).toBe("FREE");
|
||||||
|
expect(result?.listingPrice?.cents).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing optional fields", () => {
|
||||||
|
const item = {
|
||||||
|
id: "456",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Minimal Item",
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(item);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("Minimal Item");
|
||||||
|
expect(result?.description).toBeUndefined();
|
||||||
|
expect(result?.seller).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should identify vehicle listings", () => {
|
||||||
|
const vehicleItem = {
|
||||||
|
id: "999",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "2012 Mazda 3",
|
||||||
|
formatted_price: { text: "$8,000" },
|
||||||
|
listing_price: { amount: "8000.00", currency: "CAD" },
|
||||||
|
vehicle_make_display_name: "Mazda",
|
||||||
|
vehicle_model_display_name: "3",
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(vehicleItem);
|
||||||
|
expect(result?.listingType).toBe("vehicle");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle different listing statuses", () => {
|
||||||
|
const soldItem = {
|
||||||
|
id: "111",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Sold Item",
|
||||||
|
is_sold: true,
|
||||||
|
is_live: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const pendingItem = {
|
||||||
|
id: "222",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Pending Item",
|
||||||
|
is_pending: true,
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const hiddenItem = {
|
||||||
|
id: "333",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Hidden Item",
|
||||||
|
is_hidden: true,
|
||||||
|
is_live: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(parseFacebookItem(soldItem)?.listingStatus).toBe("SOLD");
|
||||||
|
expect(parseFacebookItem(pendingItem)?.listingStatus).toBe("PENDING");
|
||||||
|
expect(parseFacebookItem(hiddenItem)?.listingStatus).toBe("HIDDEN");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return null for items without title", () => {
|
||||||
|
const invalidItem = {
|
||||||
|
id: "invalid",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(invalidItem);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseFacebookAds", () => {
|
||||||
|
test("should parse search result ads", () => {
|
||||||
|
const ads = [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Ad 1",
|
||||||
|
listing_price: {
|
||||||
|
amount: "50.00",
|
||||||
|
formatted_amount: "$50.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: { city_page: { display_name: "Toronto" } },
|
||||||
|
},
|
||||||
|
creation_time: 1640995200,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "Ad 2",
|
||||||
|
listing_price: {
|
||||||
|
amount: "75.00",
|
||||||
|
formatted_amount: "$75.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: { city_page: { display_name: "Ottawa" } },
|
||||||
|
},
|
||||||
|
creation_time: 1640995300,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const results = parseFacebookAds(ads);
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
expect(results[0].title).toBe("Ad 1");
|
||||||
|
expect(results[0].listingPrice?.cents).toBe(5000);
|
||||||
|
expect(results[0].address).toBe("Toronto");
|
||||||
|
expect(results[1].title).toBe("Ad 2");
|
||||||
|
expect(results[1].address).toBe("Ottawa");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out ads without price", () => {
|
||||||
|
const ads = [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "With Price",
|
||||||
|
listing_price: {
|
||||||
|
amount: "100.00",
|
||||||
|
formatted_amount: "$100.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "No Price",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const results = parseFacebookAds(ads);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("With Price");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle malformed ads gracefully", () => {
|
||||||
|
const ads = [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Valid Ad",
|
||||||
|
listing_price: {
|
||||||
|
amount: "50.00",
|
||||||
|
formatted_amount: "$50.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
// Missing listing
|
||||||
|
},
|
||||||
|
} as { node: { listing?: unknown } },
|
||||||
|
];
|
||||||
|
|
||||||
|
const results = parseFacebookAds(ads);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("Valid Ad");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Utility Functions", () => {
|
||||||
|
describe("formatCentsToCurrency", () => {
|
||||||
|
test("should format cents to currency string", () => {
|
||||||
|
expect(formatCentsToCurrency(100)).toBe("$1.00");
|
||||||
|
expect(formatCentsToCurrency(1000)).toBe("$10.00");
|
||||||
|
expect(formatCentsToCurrency(9999)).toBe("$99.99");
|
||||||
|
expect(formatCentsToCurrency(123456)).toBe("$1,234.56");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle string inputs", () => {
|
||||||
|
expect(formatCentsToCurrency("100")).toBe("$1.00");
|
||||||
|
expect(formatCentsToCurrency("1000")).toBe("$10.00");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle zero", () => {
|
||||||
|
expect(formatCentsToCurrency(0)).toBe("$0.00");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle null and undefined", () => {
|
||||||
|
expect(formatCentsToCurrency(null)).toBe("");
|
||||||
|
expect(formatCentsToCurrency(undefined)).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle invalid inputs", () => {
|
||||||
|
expect(formatCentsToCurrency("invalid")).toBe("");
|
||||||
|
expect(formatCentsToCurrency(Number.NaN)).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("formatCookiesForHeader", () => {
|
||||||
|
const mockCookies = [
|
||||||
|
{ name: "c_user", value: "123456", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abcdef", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "session_id", value: "xyz", domain: "other.com", path: "/" },
|
||||||
|
];
|
||||||
|
|
||||||
|
test("should format cookies for header string", () => {
|
||||||
|
const result = formatCookiesForHeader(mockCookies, "www.facebook.com");
|
||||||
|
expect(result).toBe("c_user=123456; xs=abcdef");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter expired cookies", () => {
|
||||||
|
const cookiesWithExpiration = [
|
||||||
|
...mockCookies,
|
||||||
|
{
|
||||||
|
name: "expired",
|
||||||
|
value: "old",
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
expirationDate: Date.now() / 1000 - 1000,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const result = formatCookiesForHeader(
|
||||||
|
cookiesWithExpiration,
|
||||||
|
"www.facebook.com",
|
||||||
|
);
|
||||||
|
expect(result).not.toContain("expired");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle no matching cookies", () => {
|
||||||
|
const result = formatCookiesForHeader(mockCookies, "www.google.com");
|
||||||
|
expect(result).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty cookie array", () => {
|
||||||
|
const result = formatCookiesForHeader([], "www.facebook.com");
|
||||||
|
expect(result).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
712
test/facebook-integration.test.ts
Normal file
712
test/facebook-integration.test.ts
Normal file
@@ -0,0 +1,712 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import fetchFacebookItems, { fetchFacebookItem } from "../src/facebook";
|
||||||
|
|
||||||
|
// Mock fetch globally
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Main Search Function", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should successfully fetch search results", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "iPhone 13 Pro",
|
||||||
|
listing_price: {
|
||||||
|
amount: "800.00",
|
||||||
|
formatted_amount: "$800.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Toronto" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
creation_time: 1640995200,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "Samsung Galaxy",
|
||||||
|
listing_price: {
|
||||||
|
amount: "600.00",
|
||||||
|
formatted_amount: "$600.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Mississauga" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
creation_time: 1640995300,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"iPhone",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
expect(results[0].title).toBe("iPhone 13 Pro");
|
||||||
|
expect(results[1].title).toBe("Samsung Galaxy");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out items without price", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "With Price",
|
||||||
|
listing_price: {
|
||||||
|
amount: "100.00",
|
||||||
|
formatted_amount: "$100.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "No Price",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("With Price");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should respect MAX_ITEMS parameter", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: Array.from({ length: 10 }, (_, i) => ({
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: String(i),
|
||||||
|
marketplace_listing_title: `Item ${i}`,
|
||||||
|
listing_price: {
|
||||||
|
amount: `${(i + 1) * 10}.00`,
|
||||||
|
formatted_amount: `$${(i + 1) * 10}.00`,
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
5,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return empty array for no results", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"nonexistent query",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle authentication errors gracefully", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 401,
|
||||||
|
text: () => Promise.resolve("Unauthorized"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle network errors", async () => {
|
||||||
|
global.fetch = mock(() => Promise.reject(new Error("Network error")));
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
fetchFacebookItems("test", 1, "toronto", 25, mockCookies),
|
||||||
|
).rejects.toThrow("Network error");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle rate limiting with retry", async () => {
|
||||||
|
let attempts = 0;
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
attempts++;
|
||||||
|
if (attempts === 1) {
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 429,
|
||||||
|
headers: {
|
||||||
|
get: (header: string) => {
|
||||||
|
if (header === "X-RateLimit-Reset") return "1";
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
text: () => Promise.resolve("Rate limited"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Item 1",
|
||||||
|
listing_price: {
|
||||||
|
amount: "100.00",
|
||||||
|
formatted_amount: "$100.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(attempts).toBe(2);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Vehicle Listing Integration", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should correctly identify and parse vehicle listings", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "2006 Honda Civic",
|
||||||
|
listing_price: {
|
||||||
|
amount: "8000.00",
|
||||||
|
formatted_amount: "$8,000.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "iPhone 13",
|
||||||
|
listing_price: {
|
||||||
|
amount: "800.00",
|
||||||
|
formatted_amount: "$800.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"cars",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
// Both should be classified as "item" type in search results (vehicle detection is for item details)
|
||||||
|
expect(results[0].title).toBe("2006 Honda Civic");
|
||||||
|
expect(results[1].title).toBe("iPhone 13");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Different Categories", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should handle electronics listings", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Nintendo Switch",
|
||||||
|
listing_price: {
|
||||||
|
amount: "250.00",
|
||||||
|
formatted_amount: "$250.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Toronto" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
marketplace_listing_category_id:
|
||||||
|
"479353692612078",
|
||||||
|
condition: "USED",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"nintendo switch",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("Nintendo Switch");
|
||||||
|
expect(results[0].categoryId).toBe("479353692612078");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle home goods/furniture listings", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Dining Table",
|
||||||
|
listing_price: {
|
||||||
|
amount: "150.00",
|
||||||
|
formatted_amount: "$150.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Mississauga" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
marketplace_listing_category_id:
|
||||||
|
"1569171756675761",
|
||||||
|
condition: "USED",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"table",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("Dining Table");
|
||||||
|
expect(results[0].categoryId).toBe("1569171756675761");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Error Scenarios", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should handle malformed HTML responses", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
"<html><body>Invalid HTML without JSON data</body></html>",
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle 404 errors gracefully", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
text: () => Promise.resolve("Not found"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle 500 errors gracefully", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 500,
|
||||||
|
text: () => Promise.resolve("Internal Server Error"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
import { describe, test, expect } from "bun:test";
|
import { describe, expect, test } from "bun:test";
|
||||||
import {
|
import {
|
||||||
resolveLocationId,
|
|
||||||
resolveCategoryId,
|
|
||||||
buildSearchUrl,
|
|
||||||
HttpError,
|
HttpError,
|
||||||
NetworkError,
|
NetworkError,
|
||||||
ParseError,
|
ParseError,
|
||||||
RateLimitError,
|
RateLimitError,
|
||||||
ValidationError
|
ValidationError,
|
||||||
|
buildSearchUrl,
|
||||||
|
resolveCategoryId,
|
||||||
|
resolveLocationId,
|
||||||
} from "../src/kijiji";
|
} from "../src/kijiji";
|
||||||
|
|
||||||
describe("Location and Category Resolution", () => {
|
describe("Location and Category Resolution", () => {
|
||||||
@@ -74,8 +74,8 @@ describe("URL Construction", () => {
|
|||||||
const url = buildSearchUrl("iphone", {
|
const url = buildSearchUrl("iphone", {
|
||||||
location: 1700272,
|
location: 1700272,
|
||||||
category: 132,
|
category: 132,
|
||||||
sortBy: 'relevancy',
|
sortBy: "relevancy",
|
||||||
sortOrder: 'desc',
|
sortOrder: "desc",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(url).toContain("b-buy-sell/canada/iphone/k0c132l1700272");
|
expect(url).toContain("b-buy-sell/canada/iphone/k0c132l1700272");
|
||||||
@@ -95,15 +95,15 @@ describe("URL Construction", () => {
|
|||||||
|
|
||||||
test("should handle different sort options", () => {
|
test("should handle different sort options", () => {
|
||||||
const dateUrl = buildSearchUrl("iphone", {
|
const dateUrl = buildSearchUrl("iphone", {
|
||||||
sortBy: 'date',
|
sortBy: "date",
|
||||||
sortOrder: 'asc',
|
sortOrder: "asc",
|
||||||
});
|
});
|
||||||
expect(dateUrl).toContain("sort=DATE");
|
expect(dateUrl).toContain("sort=DATE");
|
||||||
expect(dateUrl).toContain("order=ASC");
|
expect(dateUrl).toContain("order=ASC");
|
||||||
|
|
||||||
const priceUrl = buildSearchUrl("iphone", {
|
const priceUrl = buildSearchUrl("iphone", {
|
||||||
sortBy: 'price',
|
sortBy: "price",
|
||||||
sortOrder: 'desc',
|
sortOrder: "desc",
|
||||||
});
|
});
|
||||||
expect(priceUrl).toContain("sort=PRICE");
|
expect(priceUrl).toContain("sort=PRICE");
|
||||||
expect(priceUrl).toContain("order=DESC");
|
expect(priceUrl).toContain("order=DESC");
|
||||||
@@ -131,7 +131,11 @@ describe("Error Classes", () => {
|
|||||||
|
|
||||||
test("NetworkError should store URL and cause", () => {
|
test("NetworkError should store URL and cause", () => {
|
||||||
const cause = new Error("Connection failed");
|
const cause = new Error("Connection failed");
|
||||||
const error = new NetworkError("Network error", "https://example.com", cause);
|
const error = new NetworkError(
|
||||||
|
"Network error",
|
||||||
|
"https://example.com",
|
||||||
|
cause,
|
||||||
|
);
|
||||||
expect(error.message).toBe("Network error");
|
expect(error.message).toBe("Network error");
|
||||||
expect(error.url).toBe("https://example.com");
|
expect(error.url).toBe("https://example.com");
|
||||||
expect(error.cause).toBe(cause);
|
expect(error.cause).toBe(cause);
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
import { describe, test, expect, beforeEach, afterEach, mock } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
import { extractApolloState, parseSearch, parseDetailedListing } from "../src/kijiji";
|
import {
|
||||||
|
extractApolloState,
|
||||||
|
parseDetailedListing,
|
||||||
|
parseSearch,
|
||||||
|
} from "../src/kijiji";
|
||||||
|
|
||||||
// Mock fetch globally
|
// Mock fetch globally
|
||||||
const originalFetch = global.fetch;
|
const originalFetch = global.fetch;
|
||||||
@@ -18,29 +22,31 @@ describe("HTML Parsing Integration", () => {
|
|||||||
|
|
||||||
describe("extractApolloState", () => {
|
describe("extractApolloState", () => {
|
||||||
test("should extract Apollo state from valid HTML", () => {
|
test("should extract Apollo state from valid HTML", () => {
|
||||||
const mockHtml = '<html><head><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"__APOLLO_STATE__":{"ROOT_QUERY":{"test":"value"}}}}}</script></head></html>';
|
const mockHtml =
|
||||||
|
'<html><head><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"__APOLLO_STATE__":{"ROOT_QUERY":{"test":"value"}}}}}</script></head></html>';
|
||||||
|
|
||||||
const result = extractApolloState(mockHtml);
|
const result = extractApolloState(mockHtml);
|
||||||
expect(result).toEqual({
|
expect(result).toEqual({
|
||||||
ROOT_QUERY: { test: "value" }
|
ROOT_QUERY: { test: "value" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should return null for HTML without Apollo state", () => {
|
test("should return null for HTML without Apollo state", () => {
|
||||||
const mockHtml = '<html><body>No data here</body></html>';
|
const mockHtml = "<html><body>No data here</body></html>";
|
||||||
const result = extractApolloState(mockHtml);
|
const result = extractApolloState(mockHtml);
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should return null for malformed JSON", () => {
|
test("should return null for malformed JSON", () => {
|
||||||
const mockHtml = '<html><script id="__NEXT_DATA__" type="application/json">{"invalid": json}</script></html>';
|
const mockHtml =
|
||||||
|
'<html><script id="__NEXT_DATA__" type="application/json">{"invalid": json}</script></html>';
|
||||||
|
|
||||||
const result = extractApolloState(mockHtml);
|
const result = extractApolloState(mockHtml);
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should handle missing __NEXT_DATA__ element", () => {
|
test("should handle missing __NEXT_DATA__ element", () => {
|
||||||
const mockHtml = '<html><body><div>Content</div></body></html>';
|
const mockHtml = "<html><body><div>Content</div></body></html>";
|
||||||
const result = extractApolloState(mockHtml);
|
const result = extractApolloState(mockHtml);
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
});
|
});
|
||||||
@@ -63,10 +69,10 @@ describe("HTML Parsing Integration", () => {
|
|||||||
url: "/v-samsung/k0l0",
|
url: "/v-samsung/k0l0",
|
||||||
title: "Samsung Galaxy",
|
title: "Samsung Galaxy",
|
||||||
},
|
},
|
||||||
"ROOT_QUERY": { test: "value" }
|
ROOT_QUERY: { test: "value" },
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
})}
|
})}
|
||||||
</script>
|
</script>
|
||||||
</html>
|
</html>
|
||||||
@@ -76,11 +82,11 @@ describe("HTML Parsing Integration", () => {
|
|||||||
expect(results).toHaveLength(2);
|
expect(results).toHaveLength(2);
|
||||||
expect(results[0]).toEqual({
|
expect(results[0]).toEqual({
|
||||||
name: "iPhone 13 Pro",
|
name: "iPhone 13 Pro",
|
||||||
listingLink: "https://www.kijiji.ca/v-iphone/k0l0"
|
listingLink: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
});
|
});
|
||||||
expect(results[1]).toEqual({
|
expect(results[1]).toEqual({
|
||||||
name: "Samsung Galaxy",
|
name: "Samsung Galaxy",
|
||||||
listingLink: "https://www.kijiji.ca/v-samsung/k0l0"
|
listingLink: "https://www.kijiji.ca/v-samsung/k0l0",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -95,17 +101,19 @@ describe("HTML Parsing Integration", () => {
|
|||||||
"Listing:123": {
|
"Listing:123": {
|
||||||
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
title: "iPhone 13 Pro",
|
title: "iPhone 13 Pro",
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
})}
|
})}
|
||||||
</script>
|
</script>
|
||||||
</html>
|
</html>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||||
expect(results[0].listingLink).toBe("https://www.kijiji.ca/v-iphone/k0l0");
|
expect(results[0].listingLink).toBe(
|
||||||
|
"https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("should filter out invalid listings", () => {
|
test("should filter out invalid listings", () => {
|
||||||
@@ -127,10 +135,10 @@ describe("HTML Parsing Integration", () => {
|
|||||||
"Other:789": {
|
"Other:789": {
|
||||||
url: "/v-other/k0l0",
|
url: "/v-other/k0l0",
|
||||||
title: "Other Item",
|
title: "Other Item",
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
})}
|
})}
|
||||||
</script>
|
</script>
|
||||||
</html>
|
</html>
|
||||||
@@ -142,7 +150,10 @@ describe("HTML Parsing Integration", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("should return empty array for invalid HTML", () => {
|
test("should return empty array for invalid HTML", () => {
|
||||||
const results = parseSearch("<html><body>Invalid</body></html>", "https://www.kijiji.ca");
|
const results = parseSearch(
|
||||||
|
"<html><body>Invalid</body></html>",
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
expect(results).toEqual([]);
|
expect(results).toEqual([]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -163,7 +174,7 @@ describe("HTML Parsing Integration", () => {
|
|||||||
price: {
|
price: {
|
||||||
amount: 80000,
|
amount: 80000,
|
||||||
currency: "CAD",
|
currency: "CAD",
|
||||||
type: "FIXED"
|
type: "FIXED",
|
||||||
},
|
},
|
||||||
type: "OFFER",
|
type: "OFFER",
|
||||||
status: "ACTIVE",
|
status: "ACTIVE",
|
||||||
@@ -176,38 +187,47 @@ describe("HTML Parsing Integration", () => {
|
|||||||
name: "Toronto",
|
name: "Toronto",
|
||||||
coordinates: {
|
coordinates: {
|
||||||
latitude: 43.6532,
|
latitude: 43.6532,
|
||||||
longitude: -79.3832
|
longitude: -79.3832,
|
||||||
}
|
},
|
||||||
},
|
},
|
||||||
imageUrls: [
|
imageUrls: [
|
||||||
"https://media.kijiji.ca/api/v1/image1.jpg",
|
"https://media.kijiji.ca/api/v1/image1.jpg",
|
||||||
"https://media.kijiji.ca/api/v1/image2.jpg"
|
"https://media.kijiji.ca/api/v1/image2.jpg",
|
||||||
],
|
],
|
||||||
imageCount: 2,
|
imageCount: 2,
|
||||||
categoryId: 132,
|
categoryId: 132,
|
||||||
adSource: "ORGANIC",
|
adSource: "ORGANIC",
|
||||||
flags: {
|
flags: {
|
||||||
topAd: false,
|
topAd: false,
|
||||||
priceDrop: true
|
priceDrop: true,
|
||||||
},
|
},
|
||||||
posterInfo: {
|
posterInfo: {
|
||||||
posterId: "user123",
|
posterId: "user123",
|
||||||
rating: 4.8
|
rating: 4.8,
|
||||||
},
|
},
|
||||||
attributes: [
|
attributes: [
|
||||||
{ canonicalName: "forsaleby", canonicalValues: ["ownr"] },
|
{
|
||||||
{ canonicalName: "phonecarrier", canonicalValues: ["unlocked"] }
|
canonicalName: "forsaleby",
|
||||||
]
|
canonicalValues: ["ownr"],
|
||||||
}
|
},
|
||||||
}
|
{
|
||||||
}
|
canonicalName: "phonecarrier",
|
||||||
}
|
canonicalValues: ["unlocked"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
})}
|
})}
|
||||||
</script>
|
</script>
|
||||||
</html>
|
</html>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const result = await parseDetailedListing(mockHtml, "https://www.kijiji.ca");
|
const result = await parseDetailedListing(
|
||||||
|
mockHtml,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
expect(result).toEqual({
|
expect(result).toEqual({
|
||||||
url: "https://www.kijiji.ca/v-iphone-13-pro/k0l0",
|
url: "https://www.kijiji.ca/v-iphone-13-pro/k0l0",
|
||||||
title: "iPhone 13 Pro 256GB",
|
title: "iPhone 13 Pro 256GB",
|
||||||
@@ -215,7 +235,7 @@ describe("HTML Parsing Integration", () => {
|
|||||||
listingPrice: {
|
listingPrice: {
|
||||||
amountFormatted: "$800.00",
|
amountFormatted: "$800.00",
|
||||||
cents: 80000,
|
cents: 80000,
|
||||||
currency: "CAD"
|
currency: "CAD",
|
||||||
},
|
},
|
||||||
listingType: "OFFER",
|
listingType: "OFFER",
|
||||||
listingStatus: "ACTIVE",
|
listingStatus: "ACTIVE",
|
||||||
@@ -225,30 +245,30 @@ describe("HTML Parsing Integration", () => {
|
|||||||
address: "Toronto, ON",
|
address: "Toronto, ON",
|
||||||
images: [
|
images: [
|
||||||
"https://media.kijiji.ca/api/v1/image1.jpg",
|
"https://media.kijiji.ca/api/v1/image1.jpg",
|
||||||
"https://media.kijiji.ca/api/v1/image2.jpg"
|
"https://media.kijiji.ca/api/v1/image2.jpg",
|
||||||
],
|
],
|
||||||
categoryId: 132,
|
categoryId: 132,
|
||||||
adSource: "ORGANIC",
|
adSource: "ORGANIC",
|
||||||
flags: {
|
flags: {
|
||||||
topAd: false,
|
topAd: false,
|
||||||
priceDrop: true
|
priceDrop: true,
|
||||||
},
|
},
|
||||||
attributes: {
|
attributes: {
|
||||||
forsaleby: ["ownr"],
|
forsaleby: ["ownr"],
|
||||||
phonecarrier: ["unlocked"]
|
phonecarrier: ["unlocked"],
|
||||||
},
|
},
|
||||||
location: {
|
location: {
|
||||||
id: 1700273,
|
id: 1700273,
|
||||||
name: "Toronto",
|
name: "Toronto",
|
||||||
coordinates: {
|
coordinates: {
|
||||||
latitude: 43.6532,
|
latitude: 43.6532,
|
||||||
longitude: -79.3832
|
longitude: -79.3832,
|
||||||
}
|
},
|
||||||
},
|
},
|
||||||
sellerInfo: {
|
sellerInfo: {
|
||||||
posterId: "user123",
|
posterId: "user123",
|
||||||
rating: 4.8
|
rating: 4.8,
|
||||||
}
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -265,18 +285,21 @@ describe("HTML Parsing Integration", () => {
|
|||||||
title: "iPhone for Sale",
|
title: "iPhone for Sale",
|
||||||
price: {
|
price: {
|
||||||
type: "CONTACT",
|
type: "CONTACT",
|
||||||
amount: null
|
amount: null,
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
})}
|
})}
|
||||||
</script>
|
</script>
|
||||||
</html>
|
</html>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const result = await parseDetailedListing(mockHtml, "https://www.kijiji.ca");
|
const result = await parseDetailedListing(
|
||||||
|
mockHtml,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -291,17 +314,20 @@ describe("HTML Parsing Integration", () => {
|
|||||||
"Listing:123": {
|
"Listing:123": {
|
||||||
url: "/v-iphone/k0l0",
|
url: "/v-iphone/k0l0",
|
||||||
title: "iPhone 13",
|
title: "iPhone 13",
|
||||||
price: { amount: 50000 }
|
price: { amount: 50000 },
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
})}
|
})}
|
||||||
</script>
|
</script>
|
||||||
</html>
|
</html>
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const result = await parseDetailedListing(mockHtml, "https://www.kijiji.ca");
|
const result = await parseDetailedListing(
|
||||||
|
mockHtml,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
expect(result).toEqual({
|
expect(result).toEqual({
|
||||||
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
title: "iPhone 13",
|
title: "iPhone 13",
|
||||||
@@ -309,7 +335,7 @@ describe("HTML Parsing Integration", () => {
|
|||||||
listingPrice: {
|
listingPrice: {
|
||||||
amountFormatted: "$500.00",
|
amountFormatted: "$500.00",
|
||||||
cents: 50000,
|
cents: 50000,
|
||||||
currency: undefined
|
currency: undefined,
|
||||||
},
|
},
|
||||||
listingType: undefined,
|
listingType: undefined,
|
||||||
listingStatus: undefined,
|
listingStatus: undefined,
|
||||||
@@ -322,15 +348,15 @@ describe("HTML Parsing Integration", () => {
|
|||||||
adSource: "UNKNOWN",
|
adSource: "UNKNOWN",
|
||||||
flags: {
|
flags: {
|
||||||
topAd: false,
|
topAd: false,
|
||||||
priceDrop: false
|
priceDrop: false,
|
||||||
},
|
},
|
||||||
attributes: {},
|
attributes: {},
|
||||||
location: {
|
location: {
|
||||||
id: 0,
|
id: 0,
|
||||||
name: "Unknown",
|
name: "Unknown",
|
||||||
coordinates: undefined
|
coordinates: undefined,
|
||||||
},
|
},
|
||||||
sellerInfo: undefined
|
sellerInfo: undefined,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||||
import { slugify, formatCentsToCurrency } from "../src/kijiji";
|
import { formatCentsToCurrency, slugify } from "../src/kijiji";
|
||||||
|
|
||||||
describe("Utility Functions", () => {
|
describe("Utility Functions", () => {
|
||||||
describe("slugify", () => {
|
describe("slugify", () => {
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ import { expect } from "bun:test";
|
|||||||
// This file is loaded before any tests run due to bunfig.toml preload
|
// This file is loaded before any tests run due to bunfig.toml preload
|
||||||
|
|
||||||
// Mock fetch globally for tests
|
// Mock fetch globally for tests
|
||||||
global.fetch = global.fetch || (() => {
|
global.fetch =
|
||||||
throw new Error('fetch is not available in test environment');
|
global.fetch ||
|
||||||
});
|
(() => {
|
||||||
|
throw new Error("fetch is not available in test environment");
|
||||||
|
});
|
||||||
|
|
||||||
// Add any global test utilities here
|
// Add any global test utilities here
|
||||||
@@ -7,25 +7,21 @@
|
|||||||
"moduleDetection": "force",
|
"moduleDetection": "force",
|
||||||
"jsx": "react-jsx",
|
"jsx": "react-jsx",
|
||||||
"allowJs": true,
|
"allowJs": true,
|
||||||
|
|
||||||
// Bundler mode
|
// Bundler mode
|
||||||
"moduleResolution": "bundler",
|
"moduleResolution": "bundler",
|
||||||
"allowImportingTsExtensions": true,
|
"allowImportingTsExtensions": true,
|
||||||
"verbatimModuleSyntax": true,
|
"verbatimModuleSyntax": true,
|
||||||
"noEmit": true,
|
"noEmit": true,
|
||||||
|
|
||||||
// Best practices
|
// Best practices
|
||||||
"strict": true,
|
"strict": true,
|
||||||
"skipLibCheck": true,
|
"skipLibCheck": true,
|
||||||
"noFallthroughCasesInSwitch": true,
|
"noFallthroughCasesInSwitch": true,
|
||||||
"noUncheckedIndexedAccess": true,
|
"noUncheckedIndexedAccess": true,
|
||||||
"noImplicitAny": true,
|
"noImplicitAny": true,
|
||||||
|
|
||||||
// Some stricter flags (disabled by default)
|
// Some stricter flags (disabled by default)
|
||||||
"noUnusedLocals": false,
|
"noUnusedLocals": false,
|
||||||
"noUnusedParameters": false,
|
"noUnusedParameters": false,
|
||||||
"noPropertyAccessFromIndexSignature": false,
|
"noPropertyAccessFromIndexSignature": false,
|
||||||
|
|
||||||
"paths": {
|
"paths": {
|
||||||
"@/*": ["./src/*"]
|
"@/*": ["./src/*"]
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user