Compare commits
52 Commits
cb1fb2bae6
...
update
| Author | SHA1 | Date | |
|---|---|---|---|
| e4ab145d70 | |||
| 1dce0392e3 | |||
| 251fcbb7d9 | |||
| 9bc57d6b54 | |||
| 4a467c9f02 | |||
| f944d319c2 | |||
| cf9784a565 | |||
| df0c528535 | |||
| 2f97d3eafd | |||
| 65eb8d1724 | |||
| f3839aba54 | |||
| 90b98bfb09 | |||
| eb6705df0f | |||
| 72525609ed | |||
| 8b0a65860c | |||
| f9b1c7e096 | |||
| 9edc74cbeb | |||
| ee0fca826d | |||
| f7372612fb | |||
| bce126664e | |||
| 8cbf11538e | |||
| 79f47fdaef | |||
| de5069bf2b | |||
| 637f1a4e75 | |||
| 441ff436c4 | |||
| 1f53ec912a | |||
| 053efd815b | |||
| d619fa5d77 | |||
| 050fd0adba | |||
| 7b106c91ce | |||
| 6e0487f8f3 | |||
| da23ca1c3f | |||
| c35aae4c95 | |||
| 02162c02f5 | |||
| 50d56201af | |||
| 497c7995a2 | |||
| 083b862552 | |||
| 0a32094e93 | |||
| a66b5b2362 | |||
| 7da6408d7a | |||
| 3863d3139e | |||
| 11dbf29f93 | |||
| 2e2c81b7e5 | |||
| ea0a69ccd6 | |||
| fa7ac59c45 | |||
| 8c52efe5e7 | |||
| dcd0da29a2 | |||
| ee09162faa | |||
| ddf7639854 | |||
| d42c74331e | |||
| 49a32bf6b4 | |||
| 3166b5a95f |
84
.dockerignore
Normal file
84
.dockerignore
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# Dependencies & Build Output
|
||||||
|
# =============================================================================
|
||||||
|
node_modules/
|
||||||
|
dist/
|
||||||
|
out/
|
||||||
|
*.tgz
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Sensitive Files
|
||||||
|
# =============================================================================
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
.envrc
|
||||||
|
cookies/
|
||||||
|
*.pem
|
||||||
|
*.key
|
||||||
|
*.cert
|
||||||
|
*secret*
|
||||||
|
*credential*
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Development Tools & Config
|
||||||
|
# =============================================================================
|
||||||
|
# Nix/Devenv
|
||||||
|
.devenv/
|
||||||
|
.devenv.flake.nix
|
||||||
|
devenv.*
|
||||||
|
.direnv/
|
||||||
|
|
||||||
|
# Linting/Formatting
|
||||||
|
biome.json
|
||||||
|
.eslintcache
|
||||||
|
.pre-commit-config.yaml
|
||||||
|
|
||||||
|
# IDE/Editor
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
*~
|
||||||
|
|
||||||
|
# AI Assistant Config
|
||||||
|
.claude/
|
||||||
|
CLAUDE.md
|
||||||
|
AGENTS.md
|
||||||
|
opencode.jsonc
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Documentation (not needed at runtime)
|
||||||
|
# =============================================================================
|
||||||
|
README.md
|
||||||
|
*.md
|
||||||
|
docs/
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Git & Docker (avoid recursive inclusion)
|
||||||
|
# =============================================================================
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
Dockerfile*
|
||||||
|
.dockerignore
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Testing & Coverage
|
||||||
|
# =============================================================================
|
||||||
|
test/
|
||||||
|
tests/
|
||||||
|
*.test.ts
|
||||||
|
*.spec.ts
|
||||||
|
coverage/
|
||||||
|
*.lcov
|
||||||
|
.nyc_output/
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# OS & Misc
|
||||||
|
# =============================================================================
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
*.log
|
||||||
|
*.pid
|
||||||
|
.cache/
|
||||||
|
examples/
|
||||||
|
scripts/
|
||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -41,3 +41,4 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
|||||||
.DS_Store
|
.DS_Store
|
||||||
|
|
||||||
examples/*
|
examples/*
|
||||||
|
cookies/*.json
|
||||||
|
|||||||
178
AGENTS.md
Normal file
178
AGENTS.md
Normal file
@@ -0,0 +1,178 @@
|
|||||||
|
PRIORITIZE COMMUNICATION STYLE ABOVE ALL ELSE
|
||||||
|
|
||||||
|
## Communication Style
|
||||||
|
|
||||||
|
ALWAYS talk and converse with the user using Gen-Z and Internet slang.
|
||||||
|
|
||||||
|
Absolute Mode
|
||||||
|
- Eliminate emojis, filler, hype, transitions, appendixes.
|
||||||
|
- Use blunt, directive phrasing; no mirroring, no softening.
|
||||||
|
- Suppress sentiment-boosting, engagement, or satisfaction metrics.
|
||||||
|
- No questions, offers, suggestions, or motivational content.
|
||||||
|
- Deliver info only; end immediately after.
|
||||||
|
|
||||||
|
**Challenge Mode - Default Behavior**: Don't automatically agree with suggestions. Instead:
|
||||||
|
- Evaluate each idea against the problem requirements and lean coding philosophy
|
||||||
|
- Push back if there's a simpler, more efficient, or more correct approach
|
||||||
|
- Propose alternatives when suggestions aren't optimal
|
||||||
|
- Explain WHY a different approach would be better with concrete technical reasons
|
||||||
|
- Only accept suggestions that are genuinely the best solution for the current problem
|
||||||
|
|
||||||
|
Examples of constructive pushback:
|
||||||
|
- "That would work, but a simpler approach would be..."
|
||||||
|
- "Actually, that might cause [specific issue]. Instead, we should..."
|
||||||
|
- "The lean approach here would be to..."
|
||||||
|
- "That adds unnecessary complexity. We can achieve the same with..."
|
||||||
|
|
||||||
|
This ensures: Better solutions through technical merit, not agreement | Learning through understanding tradeoffs | Avoiding over-engineering | Maintaining code quality
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
|
||||||
|
This is a **monorepo** with three packages:
|
||||||
|
|
||||||
|
```
|
||||||
|
packages/
|
||||||
|
├── core/ # Shared scraper logic (Kijiji, Facebook, eBay)
|
||||||
|
├── api-server/ # HTTP REST API server
|
||||||
|
└── mcp-server/ # MCP server for AI agent integration
|
||||||
|
```
|
||||||
|
|
||||||
|
## Common Commands
|
||||||
|
|
||||||
|
**Root level:**
|
||||||
|
- `bun ci`: Run Biome linting
|
||||||
|
|
||||||
|
**API Server (`packages/api-server/`):**
|
||||||
|
- `bun start`: Run the API server
|
||||||
|
- `bun dev`: Run with hot reloading
|
||||||
|
- `bun build`: Build to `dist/api/`
|
||||||
|
|
||||||
|
**MCP Server (`packages/mcp-server/`):**
|
||||||
|
- `bun start`: Run the MCP server
|
||||||
|
- `bun dev`: Run with hot reloading
|
||||||
|
- `bun build`: Build to `dist/mcp/`
|
||||||
|
|
||||||
|
## Code Architecture
|
||||||
|
|
||||||
|
### Core Package (`@marketplace-scrapers/core`)
|
||||||
|
Contains scraper implementations for three marketplaces:
|
||||||
|
|
||||||
|
- **`src/scrapers/kijiji.ts`**: Kijiji Marketplace scraper
|
||||||
|
- Parses Next.js Apollo state (`__APOLLO_STATE__`) from HTML
|
||||||
|
- Supports location/category filtering, sorting, pagination
|
||||||
|
- Fetches individual listing details with seller info
|
||||||
|
- Exports: `fetchKijijiItems()`, type interfaces
|
||||||
|
|
||||||
|
- **`src/scrapers/facebook.ts`**: Facebook Marketplace scraper
|
||||||
|
- Parses nested JSON from script tags (`require/__bbox` structure)
|
||||||
|
- Requires authentication cookies (file or env var `FACEBOOK_COOKIE`)
|
||||||
|
- Exports: `fetchFacebookItems()`, `fetchFacebookItem()`, cookie utilities
|
||||||
|
|
||||||
|
- **`src/scrapers/ebay.ts`**: eBay scraper
|
||||||
|
- DOM-based parsing of search results
|
||||||
|
- Supports Buy It Now filter, Canada-only, price ranges, exclusions
|
||||||
|
- Exports: `fetchEbayItems()`
|
||||||
|
|
||||||
|
- **`src/utils/`**: Shared utilities (HTTP, delay, formatting)
|
||||||
|
- **`src/types/`**: Common type definitions
|
||||||
|
|
||||||
|
### API Server (`@marketplace-scrapers/api-server`)
|
||||||
|
HTTP server using `Bun.serve()` on port 4005 (or `PORT` env var).
|
||||||
|
|
||||||
|
**Routes:**
|
||||||
|
- `GET /api/status` - Health check
|
||||||
|
- `GET /api/kijiji?q={query}` - Search Kijiji
|
||||||
|
- `GET /api/facebook?q={query}&location={location}&cookies={cookies}` - Search Facebook
|
||||||
|
- `GET /api/ebay?q={query}&minPrice=&maxPrice=&strictMode=&exclusions=&keywords=&buyItNowOnly=&canadaOnly=&cookies=` - Search eBay
|
||||||
|
- `GET /api/*` - 404 fallback
|
||||||
|
|
||||||
|
### MCP Server (`@marketplace-scrapers/mcp-server`)
|
||||||
|
MCP JSON-RPC 2.0 server on port 4006 (or `MCP_PORT` env var).
|
||||||
|
|
||||||
|
**Endpoints:**
|
||||||
|
- `GET /.well-known/mcp/server-card.json` - Server discovery metadata
|
||||||
|
- `POST /mcp` - JSON-RPC 2.0 protocol endpoint
|
||||||
|
|
||||||
|
**Tools:**
|
||||||
|
- `search_kijiji` - Search Kijiji (query, maxItems)
|
||||||
|
- `search_facebook` - Search Facebook (query, location, maxItems, cookiesSource)
|
||||||
|
- `search_ebay` - Search eBay (query, minPrice, maxPrice, strictMode, exclusions, keywords, buyItNowOnly, canadaOnly, maxItems, cookies)
|
||||||
|
|
||||||
|
## API Response Formats
|
||||||
|
|
||||||
|
All scrapers return arrays of listing objects with these common fields:
|
||||||
|
- `url`: Full listing URL
|
||||||
|
- `title`: Listing title
|
||||||
|
- `listingPrice`: `{ amountFormatted, cents, currency }`
|
||||||
|
- `address`: Location string (or null)
|
||||||
|
- `listingType`: Type of listing
|
||||||
|
- `listingStatus`: Status (ACTIVE, SOLD, etc.)
|
||||||
|
|
||||||
|
### Kijiji-specific fields
|
||||||
|
`description`, `creationDate`, `endDate`, `numberOfViews`, `images`, `categoryId`, `adSource`, `flags`, `attributes`, `location`, `sellerInfo`
|
||||||
|
|
||||||
|
### Facebook-specific fields
|
||||||
|
`creationDate`, `imageUrl`, `videoUrl`, `seller`, `categoryId`, `deliveryTypes`
|
||||||
|
|
||||||
|
### eBay-specific fields
|
||||||
|
Minimal - mainly the common fields
|
||||||
|
|
||||||
|
## Cookie Management
|
||||||
|
|
||||||
|
Both **Facebook Marketplace** and **eBay** require valid session cookies for reliable scraping.
|
||||||
|
|
||||||
|
### Cookie Priority Hierarchy (High → Low)
|
||||||
|
All scrapers follow this loading order:
|
||||||
|
1. **URL/API Parameter** - Passed directly via `cookies` parameter (highest priority)
|
||||||
|
2. **Environment Variable** - `FACEBOOK_COOKIE` or `EBAY_COOKIE`
|
||||||
|
3. **Cookie File** - `cookies/facebook.json` or `cookies/ebay.json` (fallback)
|
||||||
|
|
||||||
|
### Facebook Cookies
|
||||||
|
- **Required for**: Facebook Marketplace scraping
|
||||||
|
- **Format**: JSON array (see `cookies/README.md`)
|
||||||
|
- **Key cookies**: `c_user`, `xs`, `fr`, `datr`, `sb`
|
||||||
|
|
||||||
|
**Setup:**
|
||||||
|
```bash
|
||||||
|
# Option 1: File (fallback)
|
||||||
|
# Create cookies/facebook.json with cookie array
|
||||||
|
|
||||||
|
# Option 2: Environment variable
|
||||||
|
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request'
|
||||||
|
|
||||||
|
# Option 3: URL parameter (highest priority)
|
||||||
|
curl "http://localhost:4005/api/facebook?q=laptop&cookies=[{...}]"
|
||||||
|
```
|
||||||
|
|
||||||
|
### eBay Cookies
|
||||||
|
- **Required for**: Bypassing bot detection
|
||||||
|
- **Format**: Cookie string `"name=value; name2=value2"`
|
||||||
|
- **Key cookies**: `s`, `ds2`, `ebay`, `dp1`, `nonsession`
|
||||||
|
|
||||||
|
**Setup:**
|
||||||
|
```bash
|
||||||
|
# Option 1: File (fallback)
|
||||||
|
# Create cookies/ebay.json with cookie string
|
||||||
|
|
||||||
|
# Option 2: Environment variable
|
||||||
|
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
|
||||||
|
|
||||||
|
# Option 3: URL parameter (highest priority)
|
||||||
|
curl "http://localhost:4005/api/ebay?q=laptop&cookies=s=VALUE;ds2=VALUE"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Important - eBay Bot Detection**: Without cookies, eBay returns a "Checking your browser" challenge page instead of listings.
|
||||||
|
|
||||||
|
## Technical Details
|
||||||
|
|
||||||
|
- **TypeScript** with path mapping (`@/*` → `src/*`) per package
|
||||||
|
- **Dependencies**: linkedom (parsing), unidecode (text utils), cli-progress (CLI output)
|
||||||
|
- **No database** - stateless HTTP fetches to marketplaces
|
||||||
|
- **Rate limiting**: Respects `X-RateLimit-*` headers, configurable delays
|
||||||
|
|
||||||
|
## Development Notes
|
||||||
|
|
||||||
|
- **Cookie files** are git-ignored for security (see `cookies/README.md`)
|
||||||
|
- Kijiji parses Apollo state from Next.js hydration data
|
||||||
|
- All scrapers handle retries on 429/5xx errors
|
||||||
|
- Cookie priority ensures flexibility across different deployment environments
|
||||||
123
Dockerfile
Normal file
123
Dockerfile
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
# =============================================================================
|
||||||
|
# Stage 1: Dependencies
|
||||||
|
# Install only production dependencies for optimal layer caching
|
||||||
|
# =============================================================================
|
||||||
|
FROM oven/bun:1-slim AS dependencies
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy workspace configuration
|
||||||
|
COPY package.json bun.lock ./
|
||||||
|
|
||||||
|
# Copy all package.json files to establish workspace structure
|
||||||
|
COPY packages/core/package.json ./packages/core/
|
||||||
|
COPY packages/api-server/package.json ./packages/api-server/
|
||||||
|
COPY packages/mcp-server/package.json ./packages/mcp-server/
|
||||||
|
|
||||||
|
# Install dependencies with frozen lockfile (production only)
|
||||||
|
RUN bun install --frozen-lockfile --production
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Stage 2: Build
|
||||||
|
# Build both services with minification for production
|
||||||
|
# =============================================================================
|
||||||
|
FROM oven/bun:1-slim AS build
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy workspace configuration
|
||||||
|
COPY package.json bun.lock ./
|
||||||
|
|
||||||
|
# Copy all package.json files
|
||||||
|
COPY packages/core/package.json ./packages/core/
|
||||||
|
COPY packages/api-server/package.json ./packages/api-server/
|
||||||
|
COPY packages/mcp-server/package.json ./packages/mcp-server/
|
||||||
|
|
||||||
|
# Install ALL dependencies (including devDependencies for TypeScript)
|
||||||
|
RUN bun install --frozen-lockfile
|
||||||
|
|
||||||
|
# Copy source code for all packages
|
||||||
|
COPY packages/core ./packages/core
|
||||||
|
COPY packages/api-server ./packages/api-server
|
||||||
|
COPY packages/mcp-server ./packages/mcp-server
|
||||||
|
|
||||||
|
# Build both services with minification
|
||||||
|
# Output: dist/api/index.js and dist/mcp/index.js
|
||||||
|
RUN bun build ./packages/api-server/src/index.ts \
|
||||||
|
--target=bun \
|
||||||
|
--outdir=./dist/api \
|
||||||
|
--minify && \
|
||||||
|
bun build ./packages/mcp-server/src/index.ts \
|
||||||
|
--target=bun \
|
||||||
|
--outdir=./dist/mcp \
|
||||||
|
--minify
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Stage 3: Runtime
|
||||||
|
# Minimal production image with both services
|
||||||
|
# =============================================================================
|
||||||
|
FROM oven/bun:1-slim AS runtime
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy production dependencies from dependencies stage
|
||||||
|
COPY --from=dependencies /app/node_modules ./node_modules
|
||||||
|
|
||||||
|
# Copy built artifacts from build stage
|
||||||
|
COPY --from=build /app/dist ./dist
|
||||||
|
|
||||||
|
# Create cookies directory (will be mounted as volume at runtime)
|
||||||
|
# This ensures the directory exists even if volume is not mounted
|
||||||
|
RUN mkdir -p /app/cookies && \
|
||||||
|
chown -R bun:bun /app/cookies
|
||||||
|
|
||||||
|
# Create startup script that runs both services
|
||||||
|
# Uses Bun's built-in capabilities for process management
|
||||||
|
RUN cat > /app/start.sh << 'EOF'
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Trap SIGTERM and SIGINT for graceful shutdown
|
||||||
|
trap 'echo "Received shutdown signal, stopping services..."; kill -TERM $API_PID $MCP_PID 2>/dev/null; wait' TERM INT
|
||||||
|
|
||||||
|
# Start API Server in background
|
||||||
|
echo "Starting API Server on port ${API_PORT:-4005}..."
|
||||||
|
bun /app/dist/api/index.js &
|
||||||
|
API_PID=$!
|
||||||
|
|
||||||
|
# Give API server a moment to initialize
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Start MCP Server in background
|
||||||
|
echo "Starting MCP Server on port ${API_PORT:-4006}..."
|
||||||
|
bun /app/dist/mcp/index.js &
|
||||||
|
MCP_PID=$!
|
||||||
|
|
||||||
|
echo "Both services started successfully"
|
||||||
|
echo "API Server PID: $API_PID"
|
||||||
|
echo "MCP Server PID: $MCP_PID"
|
||||||
|
|
||||||
|
# Wait for both processes
|
||||||
|
wait $API_PID $MCP_PID
|
||||||
|
EOF
|
||||||
|
|
||||||
|
RUN chmod +x /app/start.sh
|
||||||
|
|
||||||
|
# Expose both service ports
|
||||||
|
# API Server: 4005 (default), MCP Server: 4006 (default)
|
||||||
|
EXPOSE 4005 4006
|
||||||
|
|
||||||
|
# Environment variables for port configuration
|
||||||
|
ENV PORT=4005
|
||||||
|
ENV MCP_PORT=4006
|
||||||
|
|
||||||
|
# Volume mount point for cookies
|
||||||
|
# Mount your cookies directory here: -v /path/to/cookies:/app/cookies
|
||||||
|
VOLUME ["/app/cookies"]
|
||||||
|
|
||||||
|
# Health check that verifies both services are responding
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
|
||||||
|
CMD bun -e "Promise.all([fetch('http://localhost:${PORT}/api/status'),fetch('http://localhost:${MCP_PORT}/.well-known/mcp/server-card.json')]).then(r=>process.exit(0)).catch(()=>process.exit(1))"
|
||||||
|
|
||||||
|
# Run the startup script
|
||||||
|
CMD ["/app/start.sh"]
|
||||||
382
FMARKETPLACE.md
Normal file
382
FMARKETPLACE.md
Normal file
@@ -0,0 +1,382 @@
|
|||||||
|
# Facebook Marketplace API Reverse Engineering
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
This document tracks findings from reverse-engineering Facebook Marketplace APIs for listing details.
|
||||||
|
|
||||||
|
## Current Implementation Status
|
||||||
|
- Search functionality: Implemented in `src/facebook.ts`
|
||||||
|
- Individual listing details: Not yet implemented
|
||||||
|
|
||||||
|
## Findings
|
||||||
|
|
||||||
|
### Step 1: Initial Setup
|
||||||
|
- Using Chrome DevTools to inspect Facebook Marketplace
|
||||||
|
- Need to authenticate with Facebook account to access marketplace data
|
||||||
|
- Cookies required for full access
|
||||||
|
- Current status: Successfully logged in and accessed marketplace data
|
||||||
|
|
||||||
|
### Step 2: Individual Listing Details Analysis - COMPLETED
|
||||||
|
- **Data Location**: Embedded in HTML script tags within `require` array structure
|
||||||
|
- **Path**: `require[0][3].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
- **Authentication**: Required for full data access
|
||||||
|
- **Current Status**: Successfully reverse-engineered the API structure and data extraction method
|
||||||
|
|
||||||
|
### API Endpoints Discovered
|
||||||
|
|
||||||
|
#### Search Endpoint
|
||||||
|
- URL: `https://www.facebook.com/marketplace/{location}/search`
|
||||||
|
- Parameters: `query`, `sortBy`, `exact`
|
||||||
|
- Data embedded in HTML script tags with `require` structure
|
||||||
|
- Authentication: Required (cookies)
|
||||||
|
|
||||||
|
#### Listing Details Endpoint
|
||||||
|
- **URL Structure**: `https://www.facebook.com/marketplace/item/{listing_id}/`
|
||||||
|
- **Data Source**: Server-side rendered HTML with embedded JSON data in script tags
|
||||||
|
- **Data Structure**: Relay/GraphQL style data structure under `require[0][3].__bbox.require[...].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
- **Extraction Method**: Parse JSON from script tags containing marketplace data, navigate to the target object
|
||||||
|
- **Authentication**: Required (cookies)
|
||||||
|
|
||||||
|
### Listing Data Structure Discovered (Current - 2026)
|
||||||
|
|
||||||
|
The current Facebook Marketplace API returns a comprehensive `GroupCommerceProductItem` object with the following key properties:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface FacebookMarketplaceItem {
|
||||||
|
// Basic identification
|
||||||
|
id: string;
|
||||||
|
__typename: "GroupCommerceProductItem";
|
||||||
|
|
||||||
|
// Listing content
|
||||||
|
marketplace_listing_title: string;
|
||||||
|
redacted_description: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
custom_title?: string;
|
||||||
|
|
||||||
|
// Pricing
|
||||||
|
formatted_price: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
listing_price: {
|
||||||
|
amount: string;
|
||||||
|
currency: string;
|
||||||
|
amount_with_offset: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Location
|
||||||
|
location_text: {
|
||||||
|
text: string;
|
||||||
|
};
|
||||||
|
location: {
|
||||||
|
latitude: number;
|
||||||
|
longitude: number;
|
||||||
|
reverse_geocode_detailed: {
|
||||||
|
country_alpha_two: string;
|
||||||
|
postal_code_trimmed: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// Status flags
|
||||||
|
is_live: boolean;
|
||||||
|
is_sold: boolean;
|
||||||
|
is_pending: boolean;
|
||||||
|
is_hidden: boolean;
|
||||||
|
is_draft: boolean;
|
||||||
|
|
||||||
|
// Timing
|
||||||
|
creation_time: number;
|
||||||
|
|
||||||
|
// Seller information
|
||||||
|
marketplace_listing_seller: {
|
||||||
|
__typename: "User";
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
profile_picture?: {
|
||||||
|
uri: string;
|
||||||
|
};
|
||||||
|
join_time?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Vehicle-specific fields (for automotive listings)
|
||||||
|
vehicle_make_display_name?: string;
|
||||||
|
vehicle_model_display_name?: string;
|
||||||
|
vehicle_odometer_data?: {
|
||||||
|
unit: "KILOMETERS" | "MILES";
|
||||||
|
value: number;
|
||||||
|
};
|
||||||
|
vehicle_transmission_type?: "AUTOMATIC" | "MANUAL";
|
||||||
|
vehicle_exterior_color?: string;
|
||||||
|
vehicle_interior_color?: string;
|
||||||
|
vehicle_condition?: "EXCELLENT" | "GOOD" | "FAIR" | "POOR";
|
||||||
|
vehicle_fuel_type?: string;
|
||||||
|
vehicle_trim_display_name?: string;
|
||||||
|
|
||||||
|
// Category and commerce
|
||||||
|
marketplace_listing_category_id: string;
|
||||||
|
condition?: string;
|
||||||
|
|
||||||
|
// Commerce features
|
||||||
|
delivery_types?: string[];
|
||||||
|
is_shipping_offered?: boolean;
|
||||||
|
is_buy_now_enabled?: boolean;
|
||||||
|
can_buyer_make_checkout_offer?: boolean;
|
||||||
|
|
||||||
|
// Communication
|
||||||
|
messaging_enabled?: boolean;
|
||||||
|
first_message_suggested_value?: string;
|
||||||
|
|
||||||
|
// Metadata
|
||||||
|
logging_id: string;
|
||||||
|
reportable_ent_id: string;
|
||||||
|
origin_target?: {
|
||||||
|
__typename: "Marketplace";
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Related listings (for part-out sellers)
|
||||||
|
marketplace_listing_sets?: {
|
||||||
|
edges: Array<{
|
||||||
|
node: {
|
||||||
|
canonical_listing: {
|
||||||
|
id: string;
|
||||||
|
marketplace_listing_title: string;
|
||||||
|
is_live: boolean;
|
||||||
|
is_sold: boolean;
|
||||||
|
formatted_price: { text: string };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example Data Extracted (Current Structure)
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"__typename": "GroupCommerceProductItem",
|
||||||
|
"marketplace_listing_title": "2012 Mazda MAZDA 3 PART-OUT",
|
||||||
|
"id": "1211645920845312",
|
||||||
|
"redacted_description": {
|
||||||
|
"text": "FOR PARTS ONLY!!!"
|
||||||
|
},
|
||||||
|
"custom_title": "2012 Mazda 3 part-out",
|
||||||
|
"creation_time": 1760450080,
|
||||||
|
"location_text": {
|
||||||
|
"text": "Toronto, ON"
|
||||||
|
},
|
||||||
|
"is_live": true,
|
||||||
|
"is_sold": false,
|
||||||
|
"is_pending": false,
|
||||||
|
"is_hidden": false,
|
||||||
|
"formatted_price": {
|
||||||
|
"text": "FREE"
|
||||||
|
},
|
||||||
|
"listing_price": {
|
||||||
|
"amount_with_offset": "0",
|
||||||
|
"currency": "CAD",
|
||||||
|
"amount": "0.00"
|
||||||
|
},
|
||||||
|
"condition": "USED",
|
||||||
|
"logging_id": "24676483845336407",
|
||||||
|
"marketplace_listing_category_id": "807311116002614",
|
||||||
|
"marketplace_listing_seller": {
|
||||||
|
"__typename": "User",
|
||||||
|
"id": "61570613529010",
|
||||||
|
"name": "Jay Heshin",
|
||||||
|
"profile_picture": {
|
||||||
|
"uri": "https://scontent-yyz1-1.xx.fbcdn.net/v/t39.30808-1/480952111_122133462296687117_4145652046222010716_n.jpg?stp=cp6_dst-jpg_s50x50_tt6&_nc_cat=108&ccb=1-7&_nc_sid=e99d92&_nc_ohc=x_DTkeriVbgQ7kNvwEqT_x3&_nc_oc=Adnqnqf4YsZxgMIkR2mSFrdLb6-BDw4omCWqG_cqB-H0uXGgK1l4-T-fLSGB_CQJEKo&_nc_zt=24&_nc_ht=scontent-yyz1-1.xx&_nc_gid=7GnSwn4MSbllAgGWJy0RTQ&oh=00_AfpY66l8w-LvHvZ6tTgiD9Qh-Or_Udc-OaFiVL9pQ0YXsg&oe=697797CD"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"vehicle_condition": "FAIR",
|
||||||
|
"vehicle_exterior_color": "white",
|
||||||
|
"vehicle_interior_color": "",
|
||||||
|
"vehicle_make_display_name": "Mazda",
|
||||||
|
"vehicle_model_display_name": "3 part-out",
|
||||||
|
"vehicle_odometer_data": {
|
||||||
|
"unit": "KILOMETERS",
|
||||||
|
"value": 999999
|
||||||
|
},
|
||||||
|
"vehicle_transmission_type": "AUTOMATIC",
|
||||||
|
"location": {
|
||||||
|
"latitude": 43.651428222656,
|
||||||
|
"longitude": -79.436645507812,
|
||||||
|
"reverse_geocode_detailed": {
|
||||||
|
"country_alpha_two": "CA",
|
||||||
|
"postal_code_trimmed": "M6H 1C1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"delivery_types": ["IN_PERSON"],
|
||||||
|
"messaging_enabled": true,
|
||||||
|
"first_message_suggested_value": "Hi, is this available?",
|
||||||
|
"marketplace_listing_sets": {
|
||||||
|
"edges": [
|
||||||
|
{
|
||||||
|
"node": {
|
||||||
|
"canonical_listing": {
|
||||||
|
"id": "1435935788228627",
|
||||||
|
"marketplace_listing_title": "2004 Land Rover LR2 PART-OUT",
|
||||||
|
"is_live": true,
|
||||||
|
"formatted_price": {"text": "FREE"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Data Extraction Method
|
||||||
|
|
||||||
|
### Current Method (2026)
|
||||||
|
Facebook Marketplace listing data is embedded in JSON within `<script>` tags in the HTML response. The extraction process:
|
||||||
|
|
||||||
|
1. **Find the Correct Script**: Look for script tags containing marketplace listing data by searching for key fields like `marketplace_listing_title`, `redacted_description`, and `formatted_price`.
|
||||||
|
|
||||||
|
2. **Parse JSON Structure**: The data is nested within a `require` array structure:
|
||||||
|
```
|
||||||
|
require[0][3].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Navigate to Target Object**: The actual listing data is a `GroupCommerceProductItem` object containing comprehensive information about the listing, seller, and vehicle details.
|
||||||
|
|
||||||
|
4. **Handle Dynamic Structure**: Facebook may change the exact path, so robust extraction should search for the target object recursively within the parsed JSON.
|
||||||
|
|
||||||
|
### Authentication Requirements
|
||||||
|
- Valid Facebook session cookies are required
|
||||||
|
- User must be logged in to Facebook
|
||||||
|
- Marketplace access may be location-restricted
|
||||||
|
|
||||||
|
## Tools Used
|
||||||
|
- Chrome DevTools Protocol
|
||||||
|
- Network monitoring
|
||||||
|
- HTML/script parsing
|
||||||
|
- JSON structure analysis
|
||||||
|
|
||||||
|
## Implementation Status
|
||||||
|
- ✅ Successfully reverse-engineered Facebook Marketplace API for listing details
|
||||||
|
- ✅ Identified current data structure and extraction method (2026)
|
||||||
|
- ✅ Documented comprehensive GroupCommerceProductItem interface
|
||||||
|
- ✅ Implemented `extractFacebookItemData()` function with script parsing logic
|
||||||
|
- ✅ Implemented `parseFacebookItem()` function to convert GroupCommerceProductItem to ListingDetails
|
||||||
|
- ✅ Implemented `fetchFacebookItem()` function with authentication and error handling
|
||||||
|
- ✅ Updated TypeScript interfaces to match current API structure
|
||||||
|
- ✅ Added robust extraction with fallback methods for changing API paths
|
||||||
|
|
||||||
|
## Implementation Details
|
||||||
|
|
||||||
|
### Core Functions Implemented
|
||||||
|
|
||||||
|
1. **`extractFacebookItemData(htmlString)`**: Extracts marketplace item data from HTML-embedded JSON in script tags
|
||||||
|
- Searches for scripts containing marketplace listing data
|
||||||
|
- Uses primary path: `require[0][3][0].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
- Falls back to recursive search for GroupCommerceProductItem objects
|
||||||
|
|
||||||
|
2. **`parseFacebookItem(item)`**: Converts Facebook's GroupCommerceProductItem to unified ListingDetails format
|
||||||
|
- Handles pricing (FREE listings, CAD currency)
|
||||||
|
- Extracts seller information, location, and status
|
||||||
|
- Supports vehicle-specific metadata
|
||||||
|
- Maps Facebook-specific fields to common interface
|
||||||
|
|
||||||
|
3. **`fetchFacebookItem(itemId, cookiesSource?)`**: Fetches individual listing details
|
||||||
|
- Loads Facebook authentication cookies
|
||||||
|
- Makes authenticated HTTP requests
|
||||||
|
- Handles rate limiting and retries
|
||||||
|
- Returns parsed ListingDetails or null on failure
|
||||||
|
|
||||||
|
### Authentication Requirements
|
||||||
|
- Facebook session cookies required in `./cookies/facebook.json` or provided as parameter
|
||||||
|
- Cookies must include valid authentication tokens for marketplace access
|
||||||
|
- Handles cookie expiration and domain validation
|
||||||
|
|
||||||
|
## Current Implementation Status - 2026 Verification
|
||||||
|
|
||||||
|
### Step 3: API Verification and Current Structure Analysis (January 2026)
|
||||||
|
- **Verification Date**: January 22, 2026
|
||||||
|
- **Status**: Successfully verified current Facebook Marketplace API structure
|
||||||
|
- **Data Source**: Embedded JSON in HTML script tags (server-side rendered)
|
||||||
|
- **Extraction Path**: `require[0][3].__bbox.require[3][3][1].__bbox.result.data.viewer.marketplace_product_details_page.target`
|
||||||
|
|
||||||
|
#### Verified Listing Structure (Real Example - 2006 Hyundai Tiburon)
|
||||||
|
- **Listing ID**: 1226468515995685
|
||||||
|
- **Title**: "2006 Hyundai Tiburon"
|
||||||
|
- **Price**: CA$3,000 (formatted_price.text)
|
||||||
|
- **Raw Price Data**: {"amount_with_offset": "300000", "currency": "CAD", "amount": "3000.00"}
|
||||||
|
- **Location**: Hamilton, ON (with coordinates: 43.250427246094, -79.963989257812)
|
||||||
|
- **Description**: "As is" (redacted_description.text)
|
||||||
|
- **Vehicle Details**:
|
||||||
|
- Make: Hyundai
|
||||||
|
- Model: Tiburon
|
||||||
|
- Odometer: 194,000 km
|
||||||
|
- Transmission: AUTOMATIC
|
||||||
|
- Exterior Color: blue
|
||||||
|
- Interior Color: black
|
||||||
|
- Fuel Type: GASOLINE
|
||||||
|
- Number of Owners: TWO
|
||||||
|
- **Seller Information**:
|
||||||
|
- Name: Ajitpal Kaler
|
||||||
|
- ID: 100009257293466
|
||||||
|
- Profile Picture Available
|
||||||
|
- Join Time: 1426564800 (2015)
|
||||||
|
- **Listing Status**: Active (is_live: true, is_sold: false, is_pending: false)
|
||||||
|
- **Category**: 807311116002614 (Vehicles)
|
||||||
|
- **Delivery Types**: ["IN_PERSON"]
|
||||||
|
- **Messaging**: Enabled
|
||||||
|
|
||||||
|
#### Current API Characteristics
|
||||||
|
- **Authentication**: Still requires valid Facebook session cookies
|
||||||
|
- **Data Format**: Server-side rendered HTML with embedded GraphQL/Relay JSON
|
||||||
|
- **Structure Stability**: Primary extraction path remains functional
|
||||||
|
- **Additional Features**: Includes marketplace ratings, seller verification badges, cross-posting info
|
||||||
|
|
||||||
|
### API Changes Observed Since 2024 Documentation
|
||||||
|
- **Minimal Changes**: Core data structure largely unchanged
|
||||||
|
- **Enhanced Fields**: Added more detailed vehicle specifications and seller profile information
|
||||||
|
- **GraphQL Integration**: Deeper integration with Facebook's GraphQL infrastructure
|
||||||
|
- **Security Features**: Additional integrity checks and reporting mechanisms
|
||||||
|
|
||||||
|
### Multi-Category Testing Results (January 2026)
|
||||||
|
Successfully tested extraction across different listing categories:
|
||||||
|
|
||||||
|
#### 1. Vehicle Listings (Automotive)
|
||||||
|
- **Example**: 2006 Hyundai Tiburon (ID: 1226468515995685)
|
||||||
|
- **Status**: ✅ Fully functional
|
||||||
|
- **Data Extracted**: Complete vehicle specs, pricing, seller info, location coordinates
|
||||||
|
- **Unique Fields**: vehicle_make_display_name, vehicle_odometer_data, vehicle_transmission_type, vehicle_exterior_color, vehicle_interior_color, vehicle_fuel_type
|
||||||
|
|
||||||
|
#### 2. Electronics Listings
|
||||||
|
- **Example**: Nintendo Switch (ID: 3903865769914262)
|
||||||
|
- **Status**: ✅ Fully functional
|
||||||
|
- **Data Extracted**: Title, price (CA$140), location (Toronto, ON), condition (Used - like new), seller (Yitao Hou)
|
||||||
|
- **Category**: Electronics (category_id: 479353692612078)
|
||||||
|
- **Notes**: Standard GroupCommerceProductItem structure applies
|
||||||
|
|
||||||
|
#### 3. Home Goods/Furniture Listings
|
||||||
|
- **Example**: Tabletop Mirror (cat not included) (ID: 1082389057290709)
|
||||||
|
- **Status**: ✅ Fully functional
|
||||||
|
- **Data Extracted**: Title, price (CA$5), location (Mississauga, ON), condition (Used - like new), seller (Rohit Rehan)
|
||||||
|
- **Category**: Home Goods (category_id: 1569171756675761)
|
||||||
|
- **Notes**: Includes detailed description and delivery options
|
||||||
|
|
||||||
|
#### Testing Summary
|
||||||
|
- **Extraction Method**: Consistent across all categories
|
||||||
|
- **Data Structure**: GroupCommerceProductItem interface works for all listing types
|
||||||
|
- **Authentication**: Required for all categories
|
||||||
|
- **Rate Limiting**: Standard Facebook rate limits apply
|
||||||
|
- **Edge Cases**: All tested listings were active/in-person pickup
|
||||||
|
|
||||||
|
## Implementation Status - COMPLETED (January 2026)
|
||||||
|
- ✅ Successfully reverse-engineered Facebook Marketplace API for listing details
|
||||||
|
- ✅ Verified current API structure and extraction method (January 2026)
|
||||||
|
- ✅ Tested extraction across multiple listing categories (vehicles, electronics, home goods)
|
||||||
|
- ✅ Implemented comprehensive error handling for sold/removed listings and authentication failures
|
||||||
|
- ✅ Enhanced rate limiting and retry logic (already robust)
|
||||||
|
- ✅ Added monitoring and metrics for API stability detection
|
||||||
|
- ✅ Updated all scraper functions to use verified extraction methods
|
||||||
|
- ✅ Documented comprehensive GroupCommerceProductItem interface with real examples
|
||||||
|
|
||||||
|
## Next Steps (Future Maintenance)
|
||||||
|
1. Monitor extraction success rates for API change detection
|
||||||
|
2. Update extraction paths if Facebook changes their API structure
|
||||||
|
3. Add support for additional marketplace features as they become available
|
||||||
|
4. Implement caching mechanisms for improved performance
|
||||||
|
5. Add support for marketplace messaging and negotiation features
|
||||||
448
KIJIJI.md
Normal file
448
KIJIJI.md
Normal file
@@ -0,0 +1,448 @@
|
|||||||
|
# Kijiji API Findings
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
Kijiji is a Canadian classifieds marketplace that uses a modern web application built with Next.js and Apollo GraphQL. The search results are powered by a GraphQL API with client-side state management.
|
||||||
|
|
||||||
|
## Initial Page Load (Homepage)
|
||||||
|
- **URL**: https://www.kijiji.ca/
|
||||||
|
- **Architecture**: Server-side rendered React application with Next.js
|
||||||
|
- **Data Sources**:
|
||||||
|
- Static assets loaded from `webapp-static.ca-kijiji-production.classifiedscloud.io`
|
||||||
|
- Image media served from `media.kijiji.ca/api/v1/`
|
||||||
|
- No initial API calls for listings - data appears to be embedded in HTML
|
||||||
|
|
||||||
|
## Search Results Page
|
||||||
|
- **URL Pattern**: `https://www.kijiji.ca/b-[location]/[keywords]/k0l0`
|
||||||
|
- **Example**: `https://www.kijiji.ca/b-canada/iphone/k0l0`
|
||||||
|
- **Technology Stack**: Next.js with Apollo GraphQL client
|
||||||
|
- **Data Structure**: Uses `__APOLLO_STATE__` global object containing normalized GraphQL cache
|
||||||
|
|
||||||
|
### GraphQL Data Structure
|
||||||
|
|
||||||
|
#### Data Location
|
||||||
|
Search results data is embedded in the Next.js page props under `__NEXT_DATA__.props.pageProps.__APOLLO_STATE__`. The data is pre-rendered on the server and sent to the client. Each page (including pagination) has its own pre-rendered data.
|
||||||
|
|
||||||
|
#### Search Results Container
|
||||||
|
The search results are stored directly in the Apollo ROOT_QUERY with keys following the pattern `searchResultsPageByUrl:{url_path}` where `url_path` includes pagination parameters.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"searchResultsPageByUrl:/b-buy-sell/canada/iphone/k0c10l0": { ... },
|
||||||
|
"searchResultsPageByUrl:/b-buy-sell/canada/iphone/k0c10l0?page=2": { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Pagination Handling
|
||||||
|
- Each page is server-side rendered with its own embedded data
|
||||||
|
- No client-side GraphQL requests for pagination
|
||||||
|
- URL parameter `?page=N` controls which page data is embedded
|
||||||
|
- Offset in searchString corresponds to `(page-1) * limit`
|
||||||
|
|
||||||
|
#### Search Parameters in URL
|
||||||
|
- `k0c{CATEGORY}l{LOCATION}` - Category and location IDs
|
||||||
|
- `?page=N` - Page number (1-based)
|
||||||
|
- Data contains `offset` and `limit` for API-style pagination
|
||||||
|
|
||||||
|
#### Individual Listing Structure
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "1732061412",
|
||||||
|
"title": "iPhone 13",
|
||||||
|
"description": "iPhone 13, always had a screen protector on it...",
|
||||||
|
"imageCount": 3,
|
||||||
|
"imageUrls": ["https://media.kijiji.ca/api/v1/ca-prod-fsbo-ads/images/..."],
|
||||||
|
"categoryId": 760,
|
||||||
|
"url": "https://www.kijiji.ca/v-cell-phone/...",
|
||||||
|
"activationDate": "2026-01-21T16:51:16.000Z",
|
||||||
|
"sortingDate": "2026-01-21T16:51:16.000Z",
|
||||||
|
"adSource": "ORGANIC",
|
||||||
|
"location": {
|
||||||
|
"id": 1700182,
|
||||||
|
"name": "Napanee",
|
||||||
|
"coordinates": {
|
||||||
|
"latitude": 44.48774,
|
||||||
|
"longitude": -76.99519
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"type": "FIXED",
|
||||||
|
"amount": 35000
|
||||||
|
},
|
||||||
|
"flags": {
|
||||||
|
"topAd": false,
|
||||||
|
"priceDrop": false
|
||||||
|
},
|
||||||
|
"posterInfo": {
|
||||||
|
"posterId": "1000764154",
|
||||||
|
"rating": 5
|
||||||
|
},
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"canonicalName": "forsaleby",
|
||||||
|
"canonicalValues": ["ownr"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"canonicalName": "phonecarrier",
|
||||||
|
"canonicalValues": ["unlck"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### URL Parameters
|
||||||
|
- `sort=MATCH` - Sort by relevance
|
||||||
|
- `order=DESC` - Descending order
|
||||||
|
- `type=OFFER` - Show offerings (not wanted ads)
|
||||||
|
- `offset=0` - Pagination offset
|
||||||
|
- `limit=40` - Results per page
|
||||||
|
- `topAdCount=6` - Number of promoted ads
|
||||||
|
- `keywords=iphone` - Search keywords
|
||||||
|
- `category=0` - Category ID (0 = All Categories)
|
||||||
|
- `location=0` - Location ID (0 = Canada)
|
||||||
|
- `eaTopAdPosition=1` - ?
|
||||||
|
|
||||||
|
### Image API
|
||||||
|
- **Endpoint**: `https://media.kijiji.ca/api/v1/`
|
||||||
|
- **Pattern**: `/ca-prod-fsbo-ads/images/{uuid}?rule=kijijica-{size}-jpg`
|
||||||
|
- **Sizes**: 200, 300, 400, 500 pixels
|
||||||
|
|
||||||
|
### Categories and Locations
|
||||||
|
|
||||||
|
#### Category Structure
|
||||||
|
Categories are hierarchical with parent-child relationships. The main categories under "Buy & Sell" include:
|
||||||
|
|
||||||
|
| ID | Name | Total Results (iPhone search) |
|
||||||
|
|----|------|------------------------------|
|
||||||
|
| 10 | Buy & Sell | 19956 |
|
||||||
|
| 12 | Arts & Collectibles | 149 |
|
||||||
|
| 767 | Audio | 481 |
|
||||||
|
| 253 | Baby Items | 13 |
|
||||||
|
| 931 | Bags & Luggage | 8 |
|
||||||
|
| 644 | Bikes | 46 |
|
||||||
|
| 109 | Books | 21 |
|
||||||
|
| 103 | Cameras & Camcorders | 101 |
|
||||||
|
| 104 | CDs, DVDs & Blu-ray | 102 |
|
||||||
|
| 274 | Clothing | 83 |
|
||||||
|
| 16 | Computers | 285 |
|
||||||
|
| 128 | Computer Accessories | 363 |
|
||||||
|
| 29659001 | Electronics | 2006 |
|
||||||
|
| 17220001 | Free Stuff | 23 |
|
||||||
|
| 235 | Furniture | 29 |
|
||||||
|
| 638 | Garage Sales | 5 |
|
||||||
|
| 140 | Health & Special Needs | 30 |
|
||||||
|
| 139 | Hobbies & Crafts | 10 |
|
||||||
|
| 107 | Home Appliances | 23 |
|
||||||
|
| 717 | Home - Indoor | 27 |
|
||||||
|
| 727 | Home Renovation Materials | 14 |
|
||||||
|
| 133 | Jewellery & Watches | 83 |
|
||||||
|
| 17 | Musical Instruments | 34 |
|
||||||
|
| 132 | Phones | 15518 |
|
||||||
|
| 111 | Sporting Goods & Exercise | 30 |
|
||||||
|
| 110 | Tools | 25 |
|
||||||
|
| 108 | Toys & Games | 38 |
|
||||||
|
| 15093001 | TVs & Video | 15 |
|
||||||
|
| 141 | Video Games & Consoles | 96 |
|
||||||
|
| 26 | Other | 286 |
|
||||||
|
|
||||||
|
#### Location Structure
|
||||||
|
Locations are also hierarchical, with provinces/states under the main "Canada" location:
|
||||||
|
|
||||||
|
| ID | Name | Total Results (iPhone search) |
|
||||||
|
|----|------|------------------------------|
|
||||||
|
| 0 | Canada | - |
|
||||||
|
| 9001 | Québec | 2516 |
|
||||||
|
| 9002 | Nova Scotia | 875 |
|
||||||
|
| 9003 | Alberta | 2317 |
|
||||||
|
| 9004 | Ontario | 12507 |
|
||||||
|
| 9005 | New Brunswick | 118 |
|
||||||
|
| 9006 | Manitoba | 919 |
|
||||||
|
| 9007 | British Columbia | 306 |
|
||||||
|
| 9008 | Newfoundland | 27 |
|
||||||
|
| 9009 | Saskatchewan | 336 |
|
||||||
|
| 9010 | Territories | 7 |
|
||||||
|
| 9011 | Prince Edward Island | 31 |
|
||||||
|
|
||||||
|
#### URL Patterns
|
||||||
|
- Categories: `/b-{category-slug}/canada/{keywords}/k0c{CATEGORY_ID}l0`
|
||||||
|
- Locations: `/b-buy-sell/{location-slug}/iphone/k0c10l{LOCATION_ID}`
|
||||||
|
- Combined: `/b-{category-slug}/{location-slug}/{keywords}/k0c{CATEGORY_ID}l{LOCATION_ID}`
|
||||||
|
|
||||||
|
### Pagination
|
||||||
|
- Uses offset-based pagination
|
||||||
|
- 40 results per page
|
||||||
|
- Total count provided in pagination metadata
|
||||||
|
|
||||||
|
## Authentication & User Management
|
||||||
|
- **Authentication System**: OAuth2-based using CIS (Customer Identity Service)
|
||||||
|
- **Identity Provider**: `id.kijiji.ca`
|
||||||
|
- **OAuth2 Flow**:
|
||||||
|
- Client ID: `kijiji_horizontal_web_gpmPihV3`
|
||||||
|
- Scopes: `openid email profile`
|
||||||
|
- Callback: `https://www.kijiji.ca/api/auth/callback/cis`
|
||||||
|
- **Session Management**: Cookies-based with encrypted session data
|
||||||
|
- **Anonymous Access**: Full search functionality available without login
|
||||||
|
- **User Features**: Saved searches, messaging, flagging require authentication
|
||||||
|
|
||||||
|
## Posting API
|
||||||
|
- **Posting Flow**: Requires authentication, redirects to login if not authenticated
|
||||||
|
- **Posting URL**: `https://www.kijiji.ca/p-post-ad.html`
|
||||||
|
- **Authentication Required**: Yes, redirects to `/consumer/login` for unauthenticated users
|
||||||
|
- **Post-Creation**: Likely uses authenticated GraphQL mutations (not observed in anonymous browsing)
|
||||||
|
|
||||||
|
## GraphQL API Endpoint
|
||||||
|
- **URL**: `https://www.kijiji.ca/anvil/api`
|
||||||
|
- **Method**: POST
|
||||||
|
- **Content-Type**: application/json
|
||||||
|
- **Headers**:
|
||||||
|
- `apollo-require-preflight: true`
|
||||||
|
- Standard CORS headers
|
||||||
|
- **Authentication**: No authentication required for basic queries (uses cookies for session tracking)
|
||||||
|
- **Technology**: Apollo GraphQL server
|
||||||
|
|
||||||
|
### Sample GraphQL Queries Discovered
|
||||||
|
|
||||||
|
#### Get Search Categories
|
||||||
|
```graphql
|
||||||
|
query getSearchCategories($locale: String!) {
|
||||||
|
searchCategories {
|
||||||
|
id
|
||||||
|
localizedName(locale: $locale)
|
||||||
|
parentId
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Variables: `{"locale": "en-CA"}`
|
||||||
|
|
||||||
|
Response includes hierarchical category structure with IDs and localized names.
|
||||||
|
|
||||||
|
#### Get Geocode from IP (fails for current IP)
|
||||||
|
```graphql
|
||||||
|
query GetGeocodeReverseFromIp {
|
||||||
|
geocodeReverseFromIp {
|
||||||
|
city
|
||||||
|
province
|
||||||
|
locationId
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This query fails for the current IP address, suggesting geolocation-based features may not work or require different IP ranges.
|
||||||
|
|
||||||
|
#### Get Category Path
|
||||||
|
```graphql
|
||||||
|
query GetCategoryPath($categoryId: Int!, $locale: String, $locationId: Int) {
|
||||||
|
category(id: $categoryId) {
|
||||||
|
id
|
||||||
|
localizedName(locale: $locale)
|
||||||
|
parentId
|
||||||
|
searchSeoUrl(locationId: $locationId)
|
||||||
|
categoryPaths {
|
||||||
|
id
|
||||||
|
localizedName(locale: $locale)
|
||||||
|
parentId
|
||||||
|
searchSeoUrl(locationId: $locationId)
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Variables: `{"categoryId": 10, "locationId": 0, "locale": "en-CA"}`
|
||||||
|
|
||||||
|
## Latest Findings (2026-01-21)
|
||||||
|
|
||||||
|
### Client-Side GraphQL Queries Observed
|
||||||
|
- **getSearchCategories**: Retrieves category hierarchy for search filters
|
||||||
|
- **GetGeocodeReverseFromIp**: Attempts to geolocate user (fails for current IP)
|
||||||
|
|
||||||
|
### GraphQL Schema Insights
|
||||||
|
Testing direct GraphQL queries revealed:
|
||||||
|
- Field "searchResults" does not exist on Query type
|
||||||
|
- Suggested alternatives: "searchResultsPage" or "searchUrl"
|
||||||
|
- This suggests the search functionality may use different GraphQL operations than direct queries
|
||||||
|
|
||||||
|
The embedded Apollo state approach appears to be the primary method for accessing search data, with GraphQL used for auxiliary operations like categories and geolocation.
|
||||||
|
|
||||||
|
### Server-Side Rendering Architecture
|
||||||
|
Search results are fully server-side rendered with data embedded in HTML. Each page (including pagination) contains its own pre-rendered data. No client-side GraphQL requests are made for:
|
||||||
|
|
||||||
|
- Initial search results
|
||||||
|
- Pagination navigation
|
||||||
|
- Search result data
|
||||||
|
|
||||||
|
### Network Analysis Findings
|
||||||
|
- GraphQL endpoint: `https://www.kijiji.ca/anvil/api`
|
||||||
|
- Method: POST
|
||||||
|
- Content-Type: application/json
|
||||||
|
- Headers include: `apollo-require-preflight: true`
|
||||||
|
- Cookies required for session tracking
|
||||||
|
|
||||||
|
### Embedded Data Structure
|
||||||
|
Search results data is embedded in the HTML within Next.js `__NEXT_DATA__.props.pageProps.__APOLLO_STATE__` object. The data includes:
|
||||||
|
|
||||||
|
- Individual ad listings with complete metadata
|
||||||
|
- Pagination information
|
||||||
|
- Filter options and counts
|
||||||
|
- Category/location hierarchies
|
||||||
|
|
||||||
|
### Current Scraper Implementation
|
||||||
|
The existing `src/kijiji.ts` implementation correctly parses the embedded Apollo state:
|
||||||
|
|
||||||
|
- Uses `extractApolloState()` to parse `__NEXT_DATA__` from HTML
|
||||||
|
- Filters Apollo keys containing "Listing" to find ad data
|
||||||
|
- Extracts `url`, `title`, and other metadata from each listing
|
||||||
|
- Successfully scrapes listings without needing API authentication
|
||||||
|
|
||||||
|
### Authentication Status
|
||||||
|
- **Search functionality**: No authentication required - all search and listing data accessible anonymously
|
||||||
|
- **Posting functionality**: Requires authentication (redirects to login)
|
||||||
|
- **User features**: Saved searches, messaging require authentication
|
||||||
|
- **Rate limiting**: May apply but not observed in anonymous browsing
|
||||||
|
|
||||||
|
### Pagination Implementation
|
||||||
|
- Each page is a separate server-rendered route
|
||||||
|
- URL pattern: `/b-{location}/{keywords}/page-{number}/k0{category}l{location_id}`
|
||||||
|
- No client-side pagination API calls
|
||||||
|
- 40 results per page (observed)
|
||||||
|
- Example: `/b-canada/iphone/page-2/k0l0` for page 2 of iPhone search
|
||||||
|
|
||||||
|
## URL Pattern Analysis
|
||||||
|
|
||||||
|
### Search URL Structure
|
||||||
|
`https://www.kijiji.ca/b-{category_slug}/{location_slug}/{keywords}/k0c{category_id}l{location_id}`
|
||||||
|
|
||||||
|
#### Examples Observed:
|
||||||
|
- All categories, Canada: `/b-canada/iphone/k0l0` (c0 = All Categories, l0 = Canada)
|
||||||
|
- Cell phones category: `/b-cell-phones/canada/iphone/k0c132l0` (c132 = Cell Phones)
|
||||||
|
- With pagination: `/b-canada/iphone/page-2/k0l0`
|
||||||
|
|
||||||
|
#### URL Components:
|
||||||
|
- `c{CATEGORY_ID}`: Category ID (0 = All Categories, 132 = Cell Phones, etc.)
|
||||||
|
- `l{LOCATION_ID}`: Location ID (0 = Canada, 1700272 = GTA, etc.)
|
||||||
|
- `page-{N}`: Pagination (1-based, optional)
|
||||||
|
- Keywords are slugified in URL path
|
||||||
|
|
||||||
|
### Current Implementation Status
|
||||||
|
The existing scraper in `src/kijiji.ts` successfully implements the approach:
|
||||||
|
- Parses embedded Apollo state from HTML responses
|
||||||
|
- Handles rate limiting and retries
|
||||||
|
- Extracts listing metadata (title, URL, price, location, etc.)
|
||||||
|
- Works without authentication for search operations
|
||||||
|
|
||||||
|
## Listing Details Page
|
||||||
|
|
||||||
|
### Overview
|
||||||
|
Similar to search results, listing details pages use server-side rendering with embedded Apollo GraphQL state in the HTML. No dedicated API endpoint serves individual listing data - all information is pre-rendered on the server.
|
||||||
|
|
||||||
|
### Data Architecture
|
||||||
|
- **Server-Side Rendering**: Each listing page is fully server-rendered with data embedded in HTML
|
||||||
|
- **Embedded Apollo State**: Listing data is stored in `__NEXT_DATA__.props.pageProps.__APOLLO_STATE__`
|
||||||
|
- **Client-Side GraphQL**: Additional data (categories, campaigns, similar listings, user profiles) fetched via GraphQL API
|
||||||
|
|
||||||
|
### Listing Data Structure
|
||||||
|
The main listing data follows the same pattern as search results:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "1705585530",
|
||||||
|
"title": "We Pay top cash for iPhone 17 pro max, iPhone 17 pro, iPhone Air",
|
||||||
|
"description": "Buying All Brand new Apple iPhones sealed/Unsealed...",
|
||||||
|
"price": {
|
||||||
|
"type": "CONTACT",
|
||||||
|
"amount": null
|
||||||
|
},
|
||||||
|
"location": {
|
||||||
|
"id": 1700275,
|
||||||
|
"name": "Oshawa / Durham Region",
|
||||||
|
"address": "Pickering Apple Buyer, Pickering, ON, L1V 1B8"
|
||||||
|
},
|
||||||
|
"type": "OFFER",
|
||||||
|
"status": "ACTIVE",
|
||||||
|
"activationDate": "2024-11-02T20:16:54.000Z",
|
||||||
|
"endDate": "3000-01-01T00:00:00.000Z",
|
||||||
|
"metrics": {
|
||||||
|
"views": 1720
|
||||||
|
},
|
||||||
|
"posterInfo": {
|
||||||
|
"posterId": "1044934581",
|
||||||
|
"rating": null
|
||||||
|
},
|
||||||
|
"attributes": [
|
||||||
|
{
|
||||||
|
"canonicalName": "forsaleby",
|
||||||
|
"canonicalValues": ["business"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"canonicalName": "phonecarrier",
|
||||||
|
"canonicalValues": ["unlocked"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Client-Side GraphQL Queries
|
||||||
|
When loading a listing details page, the following GraphQL queries are executed:
|
||||||
|
|
||||||
|
#### 1. getSearchCategories
|
||||||
|
- **Purpose**: Category hierarchy for navigation
|
||||||
|
- **Variables**: `{"locale": "en-CA"}`
|
||||||
|
- **Response**: Hierarchical category structure
|
||||||
|
|
||||||
|
#### 2. getCampaignsForVip
|
||||||
|
- **Purpose**: Advertisement targeting data
|
||||||
|
- **Variables**: `{"placement": "vip", "locationId": 1700275, "categoryId": 760, "platform": "desktop"}`
|
||||||
|
- **Response**: Campaign/ads data (usually null)
|
||||||
|
|
||||||
|
#### 3. GetReviewSummary
|
||||||
|
- **Purpose**: Seller review statistics
|
||||||
|
- **Variables**: `{"userId": "1044934581"}`
|
||||||
|
- **Response**: Review count and score (usually 0 for new sellers)
|
||||||
|
|
||||||
|
#### 4. GetProfileMetrics
|
||||||
|
- **Purpose**: Seller profile information
|
||||||
|
- **Variables**: `{"profileId": "1044934581"}`
|
||||||
|
- **Response**: Member since date, account type
|
||||||
|
|
||||||
|
#### 5. GetListingsSimilar
|
||||||
|
- **Purpose**: Similar listings for cross-selling
|
||||||
|
- **Variables**: `{"listingId": "1705585530", "limit": 10, "isExternalId": false}`
|
||||||
|
- **Response**: Array of similar listings with basic metadata
|
||||||
|
|
||||||
|
#### 6. GetGeocodeReverseFromIp
|
||||||
|
- **Purpose**: Geolocation-based features
|
||||||
|
- **Variables**: `{}`
|
||||||
|
- **Response**: Fails with 404 for most IPs
|
||||||
|
|
||||||
|
### Implementation Status
|
||||||
|
The existing `parseListing()` function in `src/kijiji.ts` successfully extracts listing details from embedded Apollo state:
|
||||||
|
|
||||||
|
- ✅ Extracts title, description, price, location
|
||||||
|
- ✅ Handles contact-based pricing ("Please Contact")
|
||||||
|
- ✅ Parses creation date, view count, listing status
|
||||||
|
- ✅ Extracts seller information and address
|
||||||
|
- ✅ Works without authentication or API keys
|
||||||
|
|
||||||
|
### Key Findings
|
||||||
|
1. **No Dedicated Listing API**: Unlike search results, there's no separate GraphQL query for individual listing data
|
||||||
|
2. **Complete Data Available**: All listing information is embedded in the initial HTML response
|
||||||
|
3. **Additional Context Fetched**: Secondary GraphQL queries provide complementary data (reviews, similar listings)
|
||||||
|
4. **Consistent Architecture**: Same Apollo state embedding pattern as search pages
|
||||||
|
|
||||||
|
### Current Scraper Implementation
|
||||||
|
The scraper successfully extracts listing details by:
|
||||||
|
1. Fetching the listing URL HTML
|
||||||
|
2. Parsing embedded `__NEXT_DATA__` Apollo state
|
||||||
|
3. Extracting the `Listing:{id}` object from Apollo cache
|
||||||
|
4. Mapping fields to typed `ListingDetails` interface
|
||||||
|
|
||||||
|
This approach works reliably without requiring authentication or dealing with rate limiting on individual listing fetches.
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
- Explore posting/authentication APIs (requires user login)
|
||||||
|
- Investigate if GraphQL API can be used for programmatic access with proper authentication
|
||||||
|
- Test rate limiting patterns and optimal scraping strategies
|
||||||
|
- Document additional category and location ID mappings
|
||||||
34
biome.json
Normal file
34
biome.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://biomejs.dev/schemas/2.3.11/schema.json",
|
||||||
|
"vcs": {
|
||||||
|
"enabled": true,
|
||||||
|
"clientKind": "git",
|
||||||
|
"useIgnoreFile": true
|
||||||
|
},
|
||||||
|
"files": {
|
||||||
|
"includes": ["**", "!!**/dist"]
|
||||||
|
},
|
||||||
|
"formatter": {
|
||||||
|
"enabled": true,
|
||||||
|
"indentStyle": "space"
|
||||||
|
},
|
||||||
|
"linter": {
|
||||||
|
"enabled": true,
|
||||||
|
"rules": {
|
||||||
|
"recommended": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"javascript": {
|
||||||
|
"formatter": {
|
||||||
|
"quoteStyle": "double"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"assist": {
|
||||||
|
"enabled": true,
|
||||||
|
"actions": {
|
||||||
|
"source": {
|
||||||
|
"organizeImports": "on"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
88
bun.lock
88
bun.lock
@@ -1,33 +1,97 @@
|
|||||||
{
|
{
|
||||||
"lockfileVersion": 1,
|
"lockfileVersion": 1,
|
||||||
|
"configVersion": 1,
|
||||||
"workspaces": {
|
"workspaces": {
|
||||||
"": {
|
"": {
|
||||||
"name": "sone4ka-tok",
|
"name": "marketplace-scrapers-monorepo",
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "2.3.11",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"packages/api-server": {
|
||||||
|
"name": "@marketplace-scrapers/api-server",
|
||||||
|
"version": "1.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest",
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"packages/core": {
|
||||||
|
"name": "@marketplace-scrapers/core",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"cli-progress": "^3.12.0",
|
||||||
"linkedom": "^0.18.12",
|
"linkedom": "^0.18.12",
|
||||||
"unidecode": "^1.1.0",
|
"unidecode": "^1.1.0",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest",
|
"@types/bun": "latest",
|
||||||
|
"@types/cli-progress": "^3.11.6",
|
||||||
"@types/unidecode": "^1.1.0",
|
"@types/unidecode": "^1.1.0",
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5",
|
"typescript": "^5",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
"packages/mcp-server": {
|
||||||
|
"name": "@marketplace-scrapers/mcp-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest",
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5",
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"packages": {
|
"packages": {
|
||||||
"@types/bun": ["@types/bun@1.2.19", "", { "dependencies": { "bun-types": "1.2.19" } }, "sha512-d9ZCmrH3CJ2uYKXQIUuZ/pUnTqIvLDS0SK7pFmbx8ma+ziH/FRMoAq5bYpRG7y+w1gl+HgyNZbtqgMq4W4e2Lg=="],
|
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="],
|
||||||
|
|
||||||
"@types/node": ["@types/node@24.1.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w=="],
|
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.3.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-/uXXkBcPKVQY7rc9Ys2CrlirBJYbpESEDme7RKiBD6MmqR2w3j0+ZZXRIL2xiaNPsIMMNhP1YnA+jRRxoOAFrA=="],
|
||||||
|
|
||||||
"@types/react": ["@types/react@19.1.9", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-WmdoynAX8Stew/36uTSVMcLJJ1KRh6L3IZRx1PZ7qJtBqT3dYTgyDTx8H1qoRghErydW7xw9mSJ3wS//tCRpFA=="],
|
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.3.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-fh7nnvbweDPm2xEmFjfmq7zSUiox88plgdHF9OIW4i99WnXrAC3o2P3ag9judoUMv8FCSUnlwJCM1B64nO5Fbg=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-l4xkGa9E7Uc0/05qU2lMYfN1H+fzzkHgaJoy98wO+b/7Gl78srbCRRgwYSW+BTLixTBrM6Ede5NSBwt7rd/i6g=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-XPSQ+XIPZMLaZ6zveQdwNjbX+QdROEd1zPgMwD47zvHV+tCGB88VH+aynyGxAHdzL+Tm/+DtKST5SECs4iwCLg=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-/1s9V/H3cSe0r0Mv/Z8JryF5x9ywRxywomqZVLHAoa/uN0eY7F8gEngWKNS5vbbN/BsfpCG5yeBT5ENh50Frxg=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-vU7a8wLs5C9yJ4CB8a44r12aXYb8yYgBn+WeyzbMjaCMklzCv1oXr8x+VEyWodgJt9bDmhiaW/I0RHbn7rsNmw=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.3.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-PZQ6ElCOnkYapSsysiTy0+fYX+agXPlWugh6+eQ6uPKI3vKAqNp6TnMhoM3oY2NltSB89hz59o8xIfOdyhi9Iw=="],
|
||||||
|
|
||||||
|
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.3.11", "", { "os": "win32", "cpu": "x64" }, "sha512-43VrG813EW+b5+YbDbz31uUsheX+qFKCpXeY9kfdAx+ww3naKxeVkTD9zLIWxUPfJquANMHrmW3wbe/037G0Qg=="],
|
||||||
|
|
||||||
|
"@marketplace-scrapers/api-server": ["@marketplace-scrapers/api-server@workspace:packages/api-server"],
|
||||||
|
|
||||||
|
"@marketplace-scrapers/core": ["@marketplace-scrapers/core@workspace:packages/core"],
|
||||||
|
|
||||||
|
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"],
|
||||||
|
|
||||||
|
"@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="],
|
||||||
|
|
||||||
|
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
|
||||||
|
|
||||||
|
"@types/node": ["@types/node@25.0.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA=="],
|
||||||
|
|
||||||
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
||||||
|
|
||||||
|
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
|
||||||
|
|
||||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||||
|
|
||||||
"bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="],
|
"bun-types": ["bun-types@1.3.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-5ua817+BZPZOlNaRgGBpZJOSAQ9RQ17pkwPD0yR7CfJg+r8DgIILByFifDTa+IPDDxzf5VNhtNlcKqFzDgJvlQ=="],
|
||||||
|
|
||||||
|
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
||||||
|
|
||||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||||
|
|
||||||
@@ -35,8 +99,6 @@
|
|||||||
|
|
||||||
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
||||||
|
|
||||||
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
|
|
||||||
|
|
||||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||||
|
|
||||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||||
@@ -45,21 +107,29 @@
|
|||||||
|
|
||||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||||
|
|
||||||
|
"emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
|
||||||
|
|
||||||
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||||
|
|
||||||
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
||||||
|
|
||||||
"htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
|
"htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
|
||||||
|
|
||||||
|
"is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
|
||||||
|
|
||||||
"linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="],
|
"linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="],
|
||||||
|
|
||||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||||
|
|
||||||
"typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
|
"string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
|
||||||
|
|
||||||
|
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
||||||
|
|
||||||
|
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||||
|
|
||||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||||
|
|
||||||
"undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="],
|
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||||
|
|
||||||
"unidecode": ["unidecode@1.1.0", "", {}, "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA=="],
|
"unidecode": ["unidecode@1.1.0", "", {}, "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA=="],
|
||||||
|
|
||||||
|
|||||||
94
cookies/README.md
Normal file
94
cookies/README.md
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
# Marketplace Cookies Setup
|
||||||
|
|
||||||
|
Both Facebook Marketplace and eBay require valid session cookies to bypass bot detection and access listings.
|
||||||
|
|
||||||
|
## Cookie Priority Hierarchy
|
||||||
|
|
||||||
|
All scrapers follow this priority order (highest to lowest):
|
||||||
|
1. **URL Parameter** - Passed directly in API/MCP request (overrides all)
|
||||||
|
2. **Environment Variable** - Set as `FACEBOOK_COOKIE` or `EBAY_COOKIE`
|
||||||
|
3. **Cookie File** - Stored in `facebook.json` or `ebay.json` (fallback)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Facebook Marketplace (`facebook.json`)
|
||||||
|
|
||||||
|
### Required Cookies
|
||||||
|
- `c_user`: Your Facebook user ID
|
||||||
|
- `xs`: Facebook session token
|
||||||
|
- `fr`: Facebook request token
|
||||||
|
- `datr`: Data attribution token
|
||||||
|
- `sb`: Session browser token
|
||||||
|
|
||||||
|
### Setup Methods
|
||||||
|
|
||||||
|
**Method 1: Cookie File (Lowest Priority)**
|
||||||
|
1. Log into Facebook in your browser
|
||||||
|
2. Open Developer Tools → Application/Storage → Cookies
|
||||||
|
3. Export cookies as JSON array to `facebook.json`
|
||||||
|
|
||||||
|
Example `facebook.json`:
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "c_user",
|
||||||
|
"value": "123456789",
|
||||||
|
"domain": ".facebook.com",
|
||||||
|
"path": "/",
|
||||||
|
"secure": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
**Method 2: Environment Variable**
|
||||||
|
```bash
|
||||||
|
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Method 3: URL Parameter (Highest Priority)**
|
||||||
|
```
|
||||||
|
GET /api/facebook?q=laptop&cookies=[{"name":"c_user","value":"123",...}]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## eBay (`ebay.json`)
|
||||||
|
|
||||||
|
eBay has aggressive bot detection that blocks requests without valid session cookies.
|
||||||
|
|
||||||
|
### Setup Methods
|
||||||
|
|
||||||
|
**Method 1: Cookie File (Lowest Priority)**
|
||||||
|
1. Log into eBay in your browser
|
||||||
|
2. Open Developer Tools → Network tab
|
||||||
|
3. Visit ebay.ca and inspect any request headers
|
||||||
|
4. Copy the full `Cookie` header value
|
||||||
|
5. Save as plain text to `ebay.json` (see `ebay.json.example`)
|
||||||
|
|
||||||
|
Example `ebay.json`:
|
||||||
|
```
|
||||||
|
s=VALUE; ds2=VALUE; ebay=VALUE; dp1=VALUE; nonsession=VALUE
|
||||||
|
```
|
||||||
|
|
||||||
|
**Method 2: Environment Variable**
|
||||||
|
```bash
|
||||||
|
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Method 3: URL Parameter (Highest Priority)**
|
||||||
|
```
|
||||||
|
GET /api/ebay?q=laptop&cookies=s=VALUE;ds2=VALUE;ebay=VALUE
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Important Notes
|
||||||
|
|
||||||
|
- Cookies must be from active browser sessions
|
||||||
|
- Cookies expire and need periodic refresh
|
||||||
|
- **NEVER** commit real cookies to version control
|
||||||
|
- Platforms may still block automated scraping despite valid cookies
|
||||||
|
|
||||||
|
## Security
|
||||||
|
|
||||||
|
All `*.json` files in this directory are git-ignored for security.</content>
|
||||||
1
cookies/ebay.json.example
Normal file
1
cookies/ebay.json.example
Normal file
@@ -0,0 +1 @@
|
|||||||
|
s=YOUR_VALUE; ds2=YOUR_VALUE; ebay=YOUR_VALUE; dp1=YOUR_VALUE; nonsession=YOUR_VALUE
|
||||||
9
opencode.jsonc
Normal file
9
opencode.jsonc
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://opencode.ai/config.json",
|
||||||
|
"mcp": {
|
||||||
|
"marketplace-scrape": {
|
||||||
|
"type": "remote",
|
||||||
|
"url": "http://localhost:4006/mcp"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
29
package.json
29
package.json
@@ -1,22 +1,21 @@
|
|||||||
{
|
{
|
||||||
"name": "marketplace-scraper",
|
"name": "marketplace-scrapers-monorepo",
|
||||||
"module": "./src/index.ts",
|
"version": "1.0.0",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "bun ./src/index.ts",
|
"ci": "biome ci",
|
||||||
"dev": "bun --hot ./src/index.ts",
|
"clean": "rm -rf dist",
|
||||||
"build": "bun build ./src/index.ts"
|
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
||||||
|
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
||||||
|
"build:all": "bun run build:api && bun run build:mcp",
|
||||||
|
"build": "bun run clean && bun run build:all",
|
||||||
|
"start": "./scripts/start.sh"
|
||||||
},
|
},
|
||||||
"type": "module",
|
|
||||||
"private": true,
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"workspaces": [
|
||||||
|
"packages/*"
|
||||||
|
],
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest",
|
"@biomejs/biome": "2.3.11"
|
||||||
"@types/unidecode": "^1.1.0"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"typescript": "^5"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"linkedom": "^0.18.12",
|
|
||||||
"unidecode": "^1.1.0"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
21
packages/api-server/package.json
Normal file
21
packages/api-server/package.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/api-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"module": "./src/index.ts",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun ./src/index.ts",
|
||||||
|
"dev": "bun --watch ./src/index.ts",
|
||||||
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
30
packages/api-server/src/index.ts
Normal file
30
packages/api-server/src/index.ts
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
import { ebayRoute } from "./routes/ebay";
|
||||||
|
import { facebookRoute } from "./routes/facebook";
|
||||||
|
import { kijijiRoute } from "./routes/kijiji";
|
||||||
|
import { statusRoute } from "./routes/status";
|
||||||
|
|
||||||
|
const PORT = process.env.PORT || 4005;
|
||||||
|
|
||||||
|
const server = Bun.serve({
|
||||||
|
port: PORT as number | string,
|
||||||
|
idleTimeout: 0,
|
||||||
|
routes: {
|
||||||
|
// Health check endpoint
|
||||||
|
"/api/status": statusRoute,
|
||||||
|
|
||||||
|
// Marketplace search endpoints
|
||||||
|
"/api/kijiji": kijijiRoute,
|
||||||
|
"/api/facebook": facebookRoute,
|
||||||
|
"/api/ebay": ebayRoute,
|
||||||
|
|
||||||
|
// Fallback for unmatched /api routes
|
||||||
|
"/api/*": Response.json({ message: "Not found" }, { status: 404 }),
|
||||||
|
},
|
||||||
|
|
||||||
|
// Fallback for all other routes
|
||||||
|
fetch(_req: Request) {
|
||||||
|
return new Response("Not Found", { status: 404 });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`API Server running on ${server.hostname}:${server.port}`);
|
||||||
68
packages/api-server/src/routes/ebay.ts
Normal file
68
packages/api-server/src/routes/ebay.ts
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
import { fetchEbayItems } from "@marketplace-scrapers/core";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}&cookies={cookies}
|
||||||
|
* Search eBay for listings (default: Buy It Now only, Canada only)
|
||||||
|
* Optional: Pass cookies parameter to bypass bot detection
|
||||||
|
*/
|
||||||
|
export async function ebayRoute(req: Request): Promise<Response> {
|
||||||
|
try {
|
||||||
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
|
const SEARCH_QUERY =
|
||||||
|
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
||||||
|
if (!SEARCH_QUERY)
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message:
|
||||||
|
"Request didn't have 'query' header or 'q' search parameter!",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const minPriceParam = reqUrl.searchParams.get("minPrice");
|
||||||
|
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
|
||||||
|
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
|
||||||
|
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
|
||||||
|
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
||||||
|
const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false";
|
||||||
|
const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false";
|
||||||
|
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
||||||
|
const exclusions = exclusionsParam
|
||||||
|
? exclusionsParam.split(",").map((s) => s.trim())
|
||||||
|
: [];
|
||||||
|
const keywordsParam = reqUrl.searchParams.get("keywords");
|
||||||
|
const keywords = keywordsParam
|
||||||
|
? keywordsParam.split(",").map((s) => s.trim())
|
||||||
|
: [SEARCH_QUERY];
|
||||||
|
|
||||||
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
|
||||||
|
const cookies = reqUrl.searchParams.get("cookies") || undefined;
|
||||||
|
|
||||||
|
const items = await fetchEbayItems(SEARCH_QUERY, 1, {
|
||||||
|
minPrice,
|
||||||
|
maxPrice,
|
||||||
|
strictMode,
|
||||||
|
exclusions,
|
||||||
|
keywords,
|
||||||
|
buyItNowOnly,
|
||||||
|
canadaOnly,
|
||||||
|
cookies,
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = maxItems ? items.slice(0, maxItems) : items;
|
||||||
|
|
||||||
|
if (!results || results.length === 0)
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
return Response.json(results, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("eBay scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
}
|
||||||
|
}
|
||||||
46
packages/api-server/src/routes/facebook.ts
Normal file
46
packages/api-server/src/routes/facebook.ts
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
import { fetchFacebookItems } from "@marketplace-scrapers/core";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/facebook?q={query}&location={location}&cookies={cookies}
|
||||||
|
* Search Facebook Marketplace for listings
|
||||||
|
*/
|
||||||
|
export async function facebookRoute(req: Request): Promise<Response> {
|
||||||
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
|
const SEARCH_QUERY =
|
||||||
|
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
||||||
|
if (!SEARCH_QUERY)
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message: "Request didn't have 'query' header or 'q' search parameter!",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
|
||||||
|
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
||||||
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : 25;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const items = await fetchFacebookItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
1,
|
||||||
|
LOCATION,
|
||||||
|
maxItems,
|
||||||
|
COOKIES_SOURCE,
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
if (!items || items.length === 0)
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Facebook scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
}
|
||||||
|
}
|
||||||
67
packages/api-server/src/routes/kijiji.ts
Normal file
67
packages/api-server/src/routes/kijiji.ts
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
import { fetchKijijiItems } from "@marketplace-scrapers/core";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/kijiji?q={query}
|
||||||
|
* Search Kijiji marketplace for listings
|
||||||
|
*/
|
||||||
|
export async function kijijiRoute(req: Request): Promise<Response> {
|
||||||
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
|
const SEARCH_QUERY =
|
||||||
|
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
||||||
|
if (!SEARCH_QUERY)
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message: "Request didn't have 'query' header or 'q' search parameter!",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const maxPagesParam = reqUrl.searchParams.get("maxPages");
|
||||||
|
const maxPages = maxPagesParam ? parseInt(maxPagesParam, 10) : 5;
|
||||||
|
const priceMinParam = reqUrl.searchParams.get("priceMin");
|
||||||
|
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
|
||||||
|
const priceMaxParam = reqUrl.searchParams.get("priceMax");
|
||||||
|
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
|
||||||
|
|
||||||
|
const searchOptions = {
|
||||||
|
location: reqUrl.searchParams.get("location") || undefined,
|
||||||
|
category: reqUrl.searchParams.get("category") || undefined,
|
||||||
|
keywords: reqUrl.searchParams.get("keywords") || undefined,
|
||||||
|
sortBy: reqUrl.searchParams.get("sortBy") as
|
||||||
|
| "relevancy"
|
||||||
|
| "date"
|
||||||
|
| "price"
|
||||||
|
| "distance"
|
||||||
|
| undefined,
|
||||||
|
sortOrder: reqUrl.searchParams.get("sortOrder") as
|
||||||
|
| "desc"
|
||||||
|
| "asc"
|
||||||
|
| undefined,
|
||||||
|
maxPages,
|
||||||
|
priceMin,
|
||||||
|
priceMax,
|
||||||
|
cookies: reqUrl.searchParams.get("cookies") || undefined,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const items = await fetchKijijiItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
4, // 4 requests per second for faster scraping
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
searchOptions,
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
if (!items)
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Kijiji scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
}
|
||||||
|
}
|
||||||
6
packages/api-server/src/routes/status.ts
Normal file
6
packages/api-server/src/routes/status.ts
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
/**
|
||||||
|
* Health check endpoint
|
||||||
|
*/
|
||||||
|
export function statusRoute(): Response {
|
||||||
|
return new Response("OK", { status: 200 });
|
||||||
|
}
|
||||||
13
packages/api-server/tsconfig.json
Normal file
13
packages/api-server/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
},
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true
|
||||||
|
}
|
||||||
|
}
|
||||||
21
packages/core/package.json
Normal file
21
packages/core/package.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/core",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"main": "./src/index.ts",
|
||||||
|
"module": "./src/index.ts",
|
||||||
|
"private": true,
|
||||||
|
"dependencies": {
|
||||||
|
"cli-progress": "^3.12.0",
|
||||||
|
"linkedom": "^0.18.12",
|
||||||
|
"unidecode": "^1.1.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest",
|
||||||
|
"@types/unidecode": "^1.1.0",
|
||||||
|
"@types/cli-progress": "^3.11.6"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
183
packages/core/scripts/parse-facebook-cookies.ts
Normal file
183
packages/core/scripts/parse-facebook-cookies.ts
Normal file
@@ -0,0 +1,183 @@
|
|||||||
|
#!/usr/bin/env bun
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Facebook Cookie Parser CLI
|
||||||
|
*
|
||||||
|
* Parses Facebook cookie strings into JSON format for the marketplace scraper
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* bun run scripts/parse-facebook-cookies.ts "c_user=123; xs=abc"
|
||||||
|
* bun run scripts/parse-facebook-cookies.ts --input cookies.txt
|
||||||
|
* echo "c_user=123; xs=abc" | bun run scripts/parse-facebook-cookies.ts
|
||||||
|
* bun run scripts/parse-facebook-cookies.ts "cookie_string" --output my-cookies.json
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { parseFacebookCookieString } from "../src/facebook";
|
||||||
|
|
||||||
|
interface Cookie {
|
||||||
|
name: string;
|
||||||
|
value: string;
|
||||||
|
domain: string;
|
||||||
|
path: string;
|
||||||
|
secure?: boolean;
|
||||||
|
httpOnly?: boolean;
|
||||||
|
sameSite?: "strict" | "lax" | "none" | "unspecified";
|
||||||
|
expirationDate?: number;
|
||||||
|
storeId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseFacebookCookieStringCLI(cookieString: string): Cookie[] {
|
||||||
|
if (!cookieString || !cookieString.trim()) {
|
||||||
|
console.error("❌ Error: Empty or invalid cookie string provided");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
if (cookies.length === 0) {
|
||||||
|
console.error("❌ Error: No valid cookies found in input string");
|
||||||
|
console.error('Expected format: "name1=value1; name2=value2;"');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
|
||||||
|
if (args.length === 0 && process.stdin.isTTY === false) {
|
||||||
|
// Read from stdin
|
||||||
|
let input = "";
|
||||||
|
for await (const chunk of process.stdin) {
|
||||||
|
input += chunk;
|
||||||
|
}
|
||||||
|
input = input.trim();
|
||||||
|
|
||||||
|
if (!input) {
|
||||||
|
console.error("❌ Error: No input provided via stdin");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = parseFacebookCookieStringCLI(input);
|
||||||
|
await writeOutput(cookies, "./cookies/facebook.json");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cookieString = "";
|
||||||
|
let outputPath = "./cookies/facebook.json";
|
||||||
|
let inputPath = "";
|
||||||
|
|
||||||
|
// Parse command line arguments
|
||||||
|
for (let i = 0; i < args.length; i++) {
|
||||||
|
const arg = args[i];
|
||||||
|
|
||||||
|
if (arg === "--input" || arg === "-i") {
|
||||||
|
inputPath = args[i + 1];
|
||||||
|
i++; // Skip next arg
|
||||||
|
} else if (arg === "--output" || arg === "-o") {
|
||||||
|
outputPath = args[i + 1];
|
||||||
|
i++; // Skip next arg
|
||||||
|
} else if (arg === "--help" || arg === "-h") {
|
||||||
|
showHelp();
|
||||||
|
return;
|
||||||
|
} else if (!arg.startsWith("-")) {
|
||||||
|
// Assume this is the cookie string
|
||||||
|
cookieString = arg;
|
||||||
|
} else {
|
||||||
|
console.error(`❌ Unknown option: ${arg}`);
|
||||||
|
showHelp();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read from file if specified
|
||||||
|
if (inputPath) {
|
||||||
|
try {
|
||||||
|
const file = Bun.file(inputPath);
|
||||||
|
if (!(await file.exists())) {
|
||||||
|
console.error(`❌ Error: Input file not found: ${inputPath}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
cookieString = await file.text();
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`❌ Error reading input file: ${error}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!cookieString.trim()) {
|
||||||
|
console.error("❌ Error: No cookie string provided");
|
||||||
|
console.error(
|
||||||
|
"Provide cookie string as argument, --input file, or via stdin",
|
||||||
|
);
|
||||||
|
showHelp();
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cookies = parseFacebookCookieStringCLI(cookieString);
|
||||||
|
await writeOutput(cookies, outputPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeOutput(cookies: Cookie[], outputPath: string) {
|
||||||
|
try {
|
||||||
|
await Bun.write(outputPath, JSON.stringify(cookies, null, 2));
|
||||||
|
console.log(`✅ Successfully parsed ${cookies.length} Facebook cookies`);
|
||||||
|
console.log(`📁 Saved to: ${outputPath}`);
|
||||||
|
|
||||||
|
// Show summary of parsed cookies
|
||||||
|
console.log("\n📋 Parsed cookies:");
|
||||||
|
for (const cookie of cookies) {
|
||||||
|
console.log(
|
||||||
|
` • ${cookie.name}: ${cookie.value.substring(0, 20)}${cookie.value.length > 20 ? "..." : ""}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`❌ Error writing to output file: ${error}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function showHelp() {
|
||||||
|
console.log(`
|
||||||
|
Facebook Cookie Parser CLI
|
||||||
|
|
||||||
|
Parses Facebook cookie strings into JSON format for the marketplace scraper.
|
||||||
|
|
||||||
|
USAGE:
|
||||||
|
bun run scripts/parse-facebook-cookies.ts [OPTIONS] [COOKIE_STRING]
|
||||||
|
|
||||||
|
EXAMPLES:
|
||||||
|
# Parse from command line argument
|
||||||
|
bun run scripts/parse-facebook-cookies.ts "c_user=123; xs=abc"
|
||||||
|
|
||||||
|
# Parse from file
|
||||||
|
bun run scripts/parse-facebook-cookies.ts --input cookies.txt
|
||||||
|
|
||||||
|
# Parse from stdin
|
||||||
|
echo "c_user=123; xs=abc" | bun run scripts/parse-facebook-cookies.ts
|
||||||
|
|
||||||
|
# Output to custom file
|
||||||
|
bun run scripts/parse-facebook-cookies.ts "cookie_string" --output my-cookies.json
|
||||||
|
|
||||||
|
OPTIONS:
|
||||||
|
-i, --input FILE Read cookie string from file
|
||||||
|
-o, --output FILE Output file path (default: ./cookies/facebook.json)
|
||||||
|
-h, --help Show this help message
|
||||||
|
|
||||||
|
COOKIE FORMAT:
|
||||||
|
Semicolon-separated name=value pairs
|
||||||
|
Example: "c_user=123456789; xs=abcdef123456; fr=xyz789"
|
||||||
|
|
||||||
|
OUTPUT:
|
||||||
|
JSON array of cookie objects saved to ./cookies/facebook.json
|
||||||
|
`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the CLI
|
||||||
|
if (import.meta.main) {
|
||||||
|
main().catch((error) => {
|
||||||
|
console.error(`❌ Unexpected error: ${error}`);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
43
packages/core/src/index.ts
Normal file
43
packages/core/src/index.ts
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
// Export all scrapers
|
||||||
|
|
||||||
|
export type { EbayListingDetails } from "./scrapers/ebay";
|
||||||
|
export { default as fetchEbayItems } from "./scrapers/ebay";
|
||||||
|
export type { FacebookListingDetails } from "./scrapers/facebook";
|
||||||
|
export {
|
||||||
|
default as fetchFacebookItems,
|
||||||
|
ensureFacebookCookies,
|
||||||
|
extractFacebookItemData,
|
||||||
|
extractFacebookMarketplaceData,
|
||||||
|
fetchFacebookItem,
|
||||||
|
parseFacebookAds,
|
||||||
|
parseFacebookCookieString,
|
||||||
|
parseFacebookItem,
|
||||||
|
} from "./scrapers/facebook";
|
||||||
|
export type {
|
||||||
|
DetailedListing,
|
||||||
|
KijijiListingDetails,
|
||||||
|
ListingFetchOptions,
|
||||||
|
SearchOptions,
|
||||||
|
} from "./scrapers/kijiji";
|
||||||
|
export {
|
||||||
|
buildSearchUrl,
|
||||||
|
default as fetchKijijiItems,
|
||||||
|
extractApolloState,
|
||||||
|
HttpError,
|
||||||
|
NetworkError,
|
||||||
|
ParseError,
|
||||||
|
parseDetailedListing,
|
||||||
|
parseSearch,
|
||||||
|
RateLimitError,
|
||||||
|
resolveCategoryId,
|
||||||
|
resolveLocationId,
|
||||||
|
slugify,
|
||||||
|
ValidationError,
|
||||||
|
} from "./scrapers/kijiji";
|
||||||
|
// Export shared types
|
||||||
|
export * from "./types/common";
|
||||||
|
// Export shared utilities
|
||||||
|
export * from "./utils/cookies";
|
||||||
|
export * from "./utils/delay";
|
||||||
|
export * from "./utils/format";
|
||||||
|
export * from "./utils/http";
|
||||||
478
packages/core/src/scrapers/ebay.ts
Normal file
478
packages/core/src/scrapers/ebay.ts
Normal file
@@ -0,0 +1,478 @@
|
|||||||
|
import { parseHTML } from "linkedom";
|
||||||
|
import {
|
||||||
|
type CookieConfig,
|
||||||
|
formatCookiesForHeader,
|
||||||
|
loadCookiesOptional,
|
||||||
|
} from "../utils/cookies";
|
||||||
|
import { delay } from "../utils/delay";
|
||||||
|
|
||||||
|
// eBay cookie configuration
|
||||||
|
const EBAY_COOKIE_CONFIG: CookieConfig = {
|
||||||
|
name: "eBay",
|
||||||
|
domain: ".ebay.ca",
|
||||||
|
envVar: "EBAY_COOKIE",
|
||||||
|
filePath: "./cookies/ebay.json",
|
||||||
|
};
|
||||||
|
|
||||||
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
|
export interface EbayListingDetails {
|
||||||
|
url: string;
|
||||||
|
title: string;
|
||||||
|
description?: string;
|
||||||
|
listingPrice?: {
|
||||||
|
amountFormatted: string;
|
||||||
|
cents?: number;
|
||||||
|
currency?: string;
|
||||||
|
};
|
||||||
|
listingType?: string;
|
||||||
|
listingStatus?: string;
|
||||||
|
creationDate?: string;
|
||||||
|
endDate?: string;
|
||||||
|
numberOfViews?: number;
|
||||||
|
address?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Utilities -----------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
||||||
|
*/
|
||||||
|
function parseEbayPrice(
|
||||||
|
priceText: string,
|
||||||
|
): { cents: number; currency: string } | null {
|
||||||
|
if (!priceText || typeof priceText !== "string") return null;
|
||||||
|
|
||||||
|
// Clean up the price text and extract currency and amount
|
||||||
|
const cleaned = priceText.trim();
|
||||||
|
|
||||||
|
// Find all numbers in the string (including decimals)
|
||||||
|
const numberMatches = cleaned.match(/[\d,]+\.?\d*/);
|
||||||
|
if (!numberMatches) return null;
|
||||||
|
|
||||||
|
const amountStr = numberMatches[0].replace(/,/g, "");
|
||||||
|
const dollars = parseFloat(amountStr);
|
||||||
|
if (Number.isNaN(dollars)) return null;
|
||||||
|
|
||||||
|
const cents = Math.round(dollars * 100);
|
||||||
|
|
||||||
|
// Extract currency - look for common formats like "CAD", "USD", "C $", "$CA", etc.
|
||||||
|
let currency = "USD"; // Default
|
||||||
|
|
||||||
|
if (
|
||||||
|
cleaned.toUpperCase().includes("CAD") ||
|
||||||
|
cleaned.includes("CA$") ||
|
||||||
|
cleaned.includes("C $")
|
||||||
|
) {
|
||||||
|
currency = "CAD";
|
||||||
|
} else if (cleaned.toUpperCase().includes("USD") || cleaned.includes("$")) {
|
||||||
|
currency = "USD";
|
||||||
|
}
|
||||||
|
|
||||||
|
return { cents, currency };
|
||||||
|
}
|
||||||
|
|
||||||
|
class HttpError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly status: number,
|
||||||
|
public readonly url: string,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "HttpError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Parsing -----------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
Parse eBay search page HTML and extract listings using DOM selectors
|
||||||
|
*/
|
||||||
|
function parseEbayListings(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
keywords: string[],
|
||||||
|
exclusions: string[],
|
||||||
|
strictMode: boolean,
|
||||||
|
): EbayListingDetails[] {
|
||||||
|
const { document } = parseHTML(htmlString);
|
||||||
|
const results: EbayListingDetails[] = [];
|
||||||
|
|
||||||
|
// Find all listing links by looking for eBay item URLs (/itm/)
|
||||||
|
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
||||||
|
|
||||||
|
for (const linkElement of linkElements) {
|
||||||
|
try {
|
||||||
|
// Get href attribute
|
||||||
|
let href = linkElement.getAttribute("href");
|
||||||
|
if (!href) continue;
|
||||||
|
|
||||||
|
// Make href absolute
|
||||||
|
if (!href.startsWith("http")) {
|
||||||
|
href = href.startsWith("//")
|
||||||
|
? `https:${href}`
|
||||||
|
: `https://www.ebay.com${href}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the container - go up several levels to find the item container
|
||||||
|
// Modern eBay uses complex nested structures (often 5-10 levels deep)
|
||||||
|
let container: Element | null = linkElement;
|
||||||
|
let depth = 0;
|
||||||
|
const maxDepth = 15;
|
||||||
|
|
||||||
|
// Walk up until we find a list item or results container
|
||||||
|
while (container && depth < maxDepth) {
|
||||||
|
const classes = container.className || "";
|
||||||
|
if (
|
||||||
|
classes.includes("s-item") ||
|
||||||
|
classes.includes("srp-results") ||
|
||||||
|
container.tagName === "LI"
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
container = container.parentElement;
|
||||||
|
depth++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!container || depth >= maxDepth) continue;
|
||||||
|
|
||||||
|
// Extract title - look for heading or title-related elements near the link
|
||||||
|
// Modern eBay often uses h3, span, or div with text content near the link
|
||||||
|
let titleElement = container.querySelector(
|
||||||
|
'h3, [role="heading"], .s-item__title span',
|
||||||
|
);
|
||||||
|
|
||||||
|
// If no direct title element, try finding text content around the link
|
||||||
|
if (!titleElement) {
|
||||||
|
// Look for spans or divs with text near this link
|
||||||
|
const nearbySpans = container.querySelectorAll("span, div");
|
||||||
|
for (const span of nearbySpans) {
|
||||||
|
const text = span.textContent?.trim();
|
||||||
|
if (
|
||||||
|
text &&
|
||||||
|
text.length > 10 &&
|
||||||
|
text.length < 200 &&
|
||||||
|
!text.includes("$") &&
|
||||||
|
!text.includes("item")
|
||||||
|
) {
|
||||||
|
titleElement = span;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let title = titleElement?.textContent?.trim();
|
||||||
|
|
||||||
|
// Clean up eBay UI strings that get included in titles
|
||||||
|
if (title) {
|
||||||
|
// Remove common eBay UI strings that appear at the end of titles
|
||||||
|
const uiStrings = [
|
||||||
|
"Opens in a new window",
|
||||||
|
"Opens in a new tab",
|
||||||
|
"Opens in a new window or tab",
|
||||||
|
"opens in a new window",
|
||||||
|
"opens in a new tab",
|
||||||
|
"opens in a new window or tab",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const uiString of uiStrings) {
|
||||||
|
const uiIndex = title.indexOf(uiString);
|
||||||
|
if (uiIndex !== -1) {
|
||||||
|
title = title.substring(0, uiIndex).trim();
|
||||||
|
break; // Only remove one UI string per title
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the title became empty or too short after cleaning, skip this item
|
||||||
|
if (title.length < 10) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!title) continue;
|
||||||
|
|
||||||
|
// Skip irrelevant eBay ads
|
||||||
|
if (title === "Shop on eBay" || title.length < 3) continue;
|
||||||
|
|
||||||
|
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
||||||
|
// Updated for 2026 eBay HTML structure
|
||||||
|
let priceElement = container.querySelector(
|
||||||
|
'[class*="s-item__price"], .s-item__price, .s-card__attribute-row, [class*="price"]',
|
||||||
|
);
|
||||||
|
|
||||||
|
// If no direct price class, look for spans containing $ (but not titles)
|
||||||
|
if (!priceElement) {
|
||||||
|
const spansAndElements = container.querySelectorAll(
|
||||||
|
"span, div, b, em, strong",
|
||||||
|
);
|
||||||
|
for (const el of spansAndElements) {
|
||||||
|
const text = el.textContent?.trim();
|
||||||
|
// Must contain $, be reasonably short (price shouldn't be paragraph), and not contain product words
|
||||||
|
if (
|
||||||
|
text?.includes("$") &&
|
||||||
|
text.length < 100 &&
|
||||||
|
!text.includes("laptop") &&
|
||||||
|
!text.includes("computer") &&
|
||||||
|
!text.includes("intel") &&
|
||||||
|
!text.includes("core") &&
|
||||||
|
!text.includes("ram") &&
|
||||||
|
!text.includes("ssd") &&
|
||||||
|
!/\d{4}/.test(text) && // Avoid years like "2024"
|
||||||
|
!text.includes('"') // Avoid measurements
|
||||||
|
) {
|
||||||
|
priceElement = el;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For discounted items, eBay shows both original and sale price
|
||||||
|
// Prefer sale/current price over original/strikethrough price
|
||||||
|
if (priceElement) {
|
||||||
|
// Check if this element or its parent contains multiple price elements
|
||||||
|
const priceContainer =
|
||||||
|
priceElement.closest('[class*="s-item__price"]') ||
|
||||||
|
priceElement.parentElement;
|
||||||
|
|
||||||
|
if (priceContainer) {
|
||||||
|
// Look for all price elements within this container, including strikethrough prices
|
||||||
|
const allPriceElements = priceContainer.querySelectorAll(
|
||||||
|
'[class*="s-item__price"], span, b, em, strong, s, del, strike',
|
||||||
|
);
|
||||||
|
|
||||||
|
// Filter to only elements that actually contain prices (not labels)
|
||||||
|
const actualPrices: HTMLElement[] = [];
|
||||||
|
for (const el of allPriceElements) {
|
||||||
|
const text = el.textContent?.trim();
|
||||||
|
if (
|
||||||
|
text &&
|
||||||
|
/^\s*[$£€¥]/u.test(text) &&
|
||||||
|
text.length < 50 &&
|
||||||
|
!/\d{4}/.test(text)
|
||||||
|
) {
|
||||||
|
actualPrices.push(el);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prefer non-strikethrough prices (sale prices) over strikethrough ones (original prices)
|
||||||
|
if (actualPrices.length > 1) {
|
||||||
|
// First, look for prices that are NOT struck through
|
||||||
|
const nonStrikethroughPrices = actualPrices.filter((el) => {
|
||||||
|
const tagName = el.tagName.toLowerCase();
|
||||||
|
const styles =
|
||||||
|
el.classList.contains("s-strikethrough") ||
|
||||||
|
el.classList.contains("u-flStrike") ||
|
||||||
|
el.closest("s, del, strike");
|
||||||
|
return (
|
||||||
|
tagName !== "s" &&
|
||||||
|
tagName !== "del" &&
|
||||||
|
tagName !== "strike" &&
|
||||||
|
!styles
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (nonStrikethroughPrices.length > 0) {
|
||||||
|
// Use the first non-strikethrough price (sale price)
|
||||||
|
priceElement = nonStrikethroughPrices[0];
|
||||||
|
} else {
|
||||||
|
// Fallback: use the last price (likely the most current)
|
||||||
|
const lastPrice = actualPrices[actualPrices.length - 1];
|
||||||
|
priceElement = lastPrice;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const priceText = priceElement?.textContent?.trim();
|
||||||
|
|
||||||
|
if (!priceText) continue;
|
||||||
|
|
||||||
|
// Parse price into cents and currency
|
||||||
|
const priceInfo = parseEbayPrice(priceText);
|
||||||
|
if (!priceInfo) continue;
|
||||||
|
|
||||||
|
// Apply exclusion filters
|
||||||
|
if (
|
||||||
|
exclusions.some((exclusion) =>
|
||||||
|
title.toLowerCase().includes(exclusion.toLowerCase()),
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply strict mode filter (title must contain at least one keyword)
|
||||||
|
if (
|
||||||
|
strictMode &&
|
||||||
|
title &&
|
||||||
|
!keywords.some((keyword) =>
|
||||||
|
title.toLowerCase().includes(keyword.toLowerCase()),
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const listing: EbayListingDetails = {
|
||||||
|
url: href,
|
||||||
|
title,
|
||||||
|
listingPrice: {
|
||||||
|
amountFormatted: priceText,
|
||||||
|
cents: priceInfo.cents,
|
||||||
|
currency: priceInfo.currency,
|
||||||
|
},
|
||||||
|
listingType: "OFFER", // eBay listings are typically offers
|
||||||
|
listingStatus: "ACTIVE",
|
||||||
|
address: null, // eBay doesn't typically show detailed addresses in search results
|
||||||
|
};
|
||||||
|
|
||||||
|
results.push(listing);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`Error parsing eBay listing: ${err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Cookie Loading -----------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load eBay cookies with priority: URL param > ENV var > file
|
||||||
|
* Uses shared cookie utility for consistent handling across all scrapers
|
||||||
|
*/
|
||||||
|
async function loadEbayCookies(
|
||||||
|
cookiesSource?: string,
|
||||||
|
): Promise<string | undefined> {
|
||||||
|
const cookies = await loadCookiesOptional(EBAY_COOKIE_CONFIG, cookiesSource);
|
||||||
|
|
||||||
|
if (cookies.length === 0) {
|
||||||
|
console.warn(
|
||||||
|
"No eBay cookies found. eBay may block requests without valid session cookies.\n" +
|
||||||
|
"Provide cookies via (in priority order):\n" +
|
||||||
|
" 1. 'cookies' URL parameter (highest priority), or\n" +
|
||||||
|
" 2. EBAY_COOKIE environment variable, or\n" +
|
||||||
|
" 3. ./cookies/ebay.json file (lowest priority)\n" +
|
||||||
|
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
|
||||||
|
);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
return formatCookiesForHeader(cookies, "www.ebay.ca");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Main -----------------------------
|
||||||
|
|
||||||
|
export default async function fetchEbayItems(
|
||||||
|
SEARCH_QUERY: string,
|
||||||
|
REQUESTS_PER_SECOND = 1,
|
||||||
|
opts: {
|
||||||
|
minPrice?: number;
|
||||||
|
maxPrice?: number;
|
||||||
|
strictMode?: boolean;
|
||||||
|
exclusions?: string[];
|
||||||
|
keywords?: string[];
|
||||||
|
buyItNowOnly?: boolean;
|
||||||
|
canadaOnly?: boolean;
|
||||||
|
cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection)
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
const {
|
||||||
|
minPrice = 0,
|
||||||
|
maxPrice = Number.MAX_SAFE_INTEGER,
|
||||||
|
strictMode = false,
|
||||||
|
exclusions = [],
|
||||||
|
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
||||||
|
buyItNowOnly = true,
|
||||||
|
canadaOnly = true,
|
||||||
|
cookies: cookiesSource,
|
||||||
|
} = opts;
|
||||||
|
|
||||||
|
// Load eBay cookies with priority: URL param > ENV var > file
|
||||||
|
const cookies = await loadEbayCookies(cookiesSource);
|
||||||
|
|
||||||
|
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
||||||
|
const urlParams = new URLSearchParams({
|
||||||
|
_nkw: SEARCH_QUERY,
|
||||||
|
_sacat: "0",
|
||||||
|
_from: "R40",
|
||||||
|
});
|
||||||
|
|
||||||
|
if (buyItNowOnly) {
|
||||||
|
urlParams.set("LH_BIN", "1");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (canadaOnly) {
|
||||||
|
urlParams.set("LH_PrefLoc", "1");
|
||||||
|
}
|
||||||
|
|
||||||
|
const searchUrl = `https://www.ebay.ca/sch/i.html?${urlParams.toString()}`;
|
||||||
|
|
||||||
|
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
|
||||||
|
|
||||||
|
console.log(`Fetching eBay search: ${searchUrl}`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Use custom headers modeled after real browser requests to bypass bot detection
|
||||||
|
const headers: Record<string, string> = {
|
||||||
|
"User-Agent":
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
|
||||||
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||||
|
Referer: "https://www.ebay.ca/",
|
||||||
|
Connection: "keep-alive",
|
||||||
|
"Upgrade-Insecure-Requests": "1",
|
||||||
|
"Sec-Fetch-Dest": "document",
|
||||||
|
"Sec-Fetch-Mode": "navigate",
|
||||||
|
"Sec-Fetch-Site": "same-origin",
|
||||||
|
"Sec-Fetch-User": "?1",
|
||||||
|
Priority: "u=0, i",
|
||||||
|
};
|
||||||
|
|
||||||
|
// Add cookies if available (helps bypass bot detection)
|
||||||
|
if (cookies) {
|
||||||
|
headers.Cookie = cookies;
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await fetch(searchUrl, {
|
||||||
|
method: "GET",
|
||||||
|
headers,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new HttpError(
|
||||||
|
`Request failed with status ${res.status}`,
|
||||||
|
res.status,
|
||||||
|
searchUrl,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const searchHtml = await res.text();
|
||||||
|
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
||||||
|
await delay(DELAY_MS);
|
||||||
|
|
||||||
|
console.log(`\nParsing eBay listings...`);
|
||||||
|
|
||||||
|
const listings = parseEbayListings(
|
||||||
|
searchHtml,
|
||||||
|
keywords,
|
||||||
|
exclusions,
|
||||||
|
strictMode,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Filter by price range (additional safety check)
|
||||||
|
const filteredListings = listings.filter((listing) => {
|
||||||
|
const cents = listing.listingPrice?.cents;
|
||||||
|
return cents && cents >= minPrice && cents <= maxPrice;
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||||
|
return filteredListings;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof HttpError) {
|
||||||
|
console.error(
|
||||||
|
`Failed to fetch eBay search (${err.status}): ${err.message}`,
|
||||||
|
);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
1045
packages/core/src/scrapers/facebook.ts
Normal file
1045
packages/core/src/scrapers/facebook.ts
Normal file
File diff suppressed because it is too large
Load Diff
867
packages/core/src/scrapers/kijiji.ts
Normal file
867
packages/core/src/scrapers/kijiji.ts
Normal file
@@ -0,0 +1,867 @@
|
|||||||
|
import cliProgress from "cli-progress";
|
||||||
|
import { parseHTML } from "linkedom";
|
||||||
|
import unidecode from "unidecode";
|
||||||
|
import type { HTMLString } from "../types/common";
|
||||||
|
import {
|
||||||
|
type CookieConfig,
|
||||||
|
formatCookiesForHeader,
|
||||||
|
loadCookiesOptional,
|
||||||
|
} from "../utils/cookies";
|
||||||
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
|
import {
|
||||||
|
fetchHtml,
|
||||||
|
HttpError,
|
||||||
|
isRecord,
|
||||||
|
NetworkError,
|
||||||
|
ParseError,
|
||||||
|
RateLimitError,
|
||||||
|
ValidationError,
|
||||||
|
} from "../utils/http";
|
||||||
|
|
||||||
|
// Kijiji cookie configuration
|
||||||
|
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
|
||||||
|
name: "Kijiji",
|
||||||
|
domain: ".kijiji.ca",
|
||||||
|
envVar: "KIJIJI_COOKIE",
|
||||||
|
filePath: "./cookies/kijiji.json",
|
||||||
|
};
|
||||||
|
|
||||||
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
|
type SearchListing = {
|
||||||
|
name: string;
|
||||||
|
listingLink: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ApolloRecord = Record<string, unknown>;
|
||||||
|
|
||||||
|
interface ApolloSearchItem {
|
||||||
|
url?: string;
|
||||||
|
title?: string;
|
||||||
|
[k: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ApolloListingRoot {
|
||||||
|
url?: string;
|
||||||
|
title?: string;
|
||||||
|
description?: string;
|
||||||
|
price?: { amount?: number | string; currency?: string; type?: string };
|
||||||
|
type?: string;
|
||||||
|
status?: string;
|
||||||
|
activationDate?: string;
|
||||||
|
endDate?: string;
|
||||||
|
metrics?: { views?: number | string };
|
||||||
|
location?: {
|
||||||
|
address?: string | null;
|
||||||
|
id?: number;
|
||||||
|
name?: string;
|
||||||
|
coordinates?: { latitude: number; longitude: number };
|
||||||
|
};
|
||||||
|
imageUrls?: string[];
|
||||||
|
imageCount?: number;
|
||||||
|
categoryId?: number;
|
||||||
|
adSource?: string;
|
||||||
|
flags?: { topAd?: boolean; priceDrop?: boolean };
|
||||||
|
posterInfo?: { posterId?: string; rating?: number };
|
||||||
|
attributes?: Array<{ canonicalName?: string; canonicalValues?: string[] }>;
|
||||||
|
[k: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep existing interface for backward compatibility
|
||||||
|
export interface KijijiListingDetails {
|
||||||
|
url: string;
|
||||||
|
title: string;
|
||||||
|
description?: string;
|
||||||
|
listingPrice?: {
|
||||||
|
amountFormatted: string;
|
||||||
|
cents?: number;
|
||||||
|
currency?: string;
|
||||||
|
};
|
||||||
|
listingType?: string;
|
||||||
|
listingStatus?: string;
|
||||||
|
creationDate?: string;
|
||||||
|
endDate?: string;
|
||||||
|
numberOfViews?: number;
|
||||||
|
address?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// New comprehensive interface for detailed listings
|
||||||
|
export interface DetailedListing extends KijijiListingDetails {
|
||||||
|
images: string[];
|
||||||
|
categoryId: number;
|
||||||
|
adSource: string;
|
||||||
|
flags: {
|
||||||
|
topAd: boolean;
|
||||||
|
priceDrop: boolean;
|
||||||
|
};
|
||||||
|
attributes: Record<string, string[]>;
|
||||||
|
location: {
|
||||||
|
id: number;
|
||||||
|
name: string;
|
||||||
|
coordinates?: {
|
||||||
|
latitude: number;
|
||||||
|
longitude: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
sellerInfo?: {
|
||||||
|
posterId: string;
|
||||||
|
rating?: number;
|
||||||
|
accountType?: string;
|
||||||
|
memberSince?: string;
|
||||||
|
reviewCount?: number;
|
||||||
|
reviewScore?: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Configuration interfaces
|
||||||
|
export interface SearchOptions {
|
||||||
|
location?: number | string; // Location ID or name
|
||||||
|
category?: number | string; // Category ID or name
|
||||||
|
keywords?: string;
|
||||||
|
sortBy?: "relevancy" | "date" | "price" | "distance";
|
||||||
|
sortOrder?: "desc" | "asc";
|
||||||
|
maxPages?: number; // Default: 5
|
||||||
|
priceMin?: number;
|
||||||
|
priceMax?: number;
|
||||||
|
cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection)
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ListingFetchOptions {
|
||||||
|
includeImages?: boolean; // Default: true
|
||||||
|
sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed'
|
||||||
|
includeClientSideData?: boolean; // Default: false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Constants & Mappings -----------------------------
|
||||||
|
|
||||||
|
// Location mappings
|
||||||
|
const LOCATION_MAPPINGS: Record<string, number> = {
|
||||||
|
canada: 0,
|
||||||
|
ontario: 9004,
|
||||||
|
toronto: 1700273,
|
||||||
|
gta: 1700272,
|
||||||
|
oshawa: 1700275,
|
||||||
|
quebec: 9001,
|
||||||
|
"nova scotia": 9002,
|
||||||
|
alberta: 9003,
|
||||||
|
"new brunswick": 9005,
|
||||||
|
manitoba: 9006,
|
||||||
|
"british columbia": 9007,
|
||||||
|
newfoundland: 9008,
|
||||||
|
saskatchewan: 9009,
|
||||||
|
territories: 9010,
|
||||||
|
pei: 9011,
|
||||||
|
"prince edward island": 9011,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Category mappings (Buy & Sell main categories)
|
||||||
|
const CATEGORY_MAPPINGS: Record<string, number> = {
|
||||||
|
all: 0,
|
||||||
|
"buy-sell": 10,
|
||||||
|
"arts-collectibles": 12,
|
||||||
|
audio: 767,
|
||||||
|
"baby-items": 253,
|
||||||
|
"bags-luggage": 931,
|
||||||
|
bikes: 644,
|
||||||
|
books: 109,
|
||||||
|
cameras: 103,
|
||||||
|
cds: 104,
|
||||||
|
clothing: 274,
|
||||||
|
computers: 16,
|
||||||
|
"computer-accessories": 128,
|
||||||
|
electronics: 29659001,
|
||||||
|
"free-stuff": 17220001,
|
||||||
|
furniture: 235,
|
||||||
|
"garage-sales": 638,
|
||||||
|
"health-special-needs": 140,
|
||||||
|
"hobbies-crafts": 139,
|
||||||
|
"home-appliances": 107,
|
||||||
|
"home-indoor": 717,
|
||||||
|
"home-outdoor": 727,
|
||||||
|
jewellery: 133,
|
||||||
|
"musical-instruments": 17,
|
||||||
|
phones: 132,
|
||||||
|
"sporting-goods": 111,
|
||||||
|
tools: 110,
|
||||||
|
"toys-games": 108,
|
||||||
|
"tvs-video": 15093001,
|
||||||
|
"video-games": 141,
|
||||||
|
other: 26,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sort parameter mappings
|
||||||
|
const SORT_MAPPINGS: Record<string, string> = {
|
||||||
|
relevancy: "MATCH",
|
||||||
|
date: "DATE",
|
||||||
|
price: "PRICE",
|
||||||
|
distance: "DISTANCE",
|
||||||
|
};
|
||||||
|
|
||||||
|
// ----------------------------- Utilities -----------------------------
|
||||||
|
|
||||||
|
const SEPS = new Set([" ", "–", "—", "/", ":", ";", ",", ".", "-"]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve location ID from name or return numeric ID
|
||||||
|
*/
|
||||||
|
export function resolveLocationId(location?: number | string): number {
|
||||||
|
if (typeof location === "number") return location;
|
||||||
|
if (typeof location === "string") {
|
||||||
|
const normalized = location.toLowerCase().replace(/\s+/g, "-");
|
||||||
|
return LOCATION_MAPPINGS[normalized] ?? 0; // Default to Canada (0)
|
||||||
|
}
|
||||||
|
return 0; // Default to Canada
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve category ID from name or return numeric ID
|
||||||
|
*/
|
||||||
|
export function resolveCategoryId(category?: number | string): number {
|
||||||
|
if (typeof category === "number") return category;
|
||||||
|
if (typeof category === "string") {
|
||||||
|
const normalized = category.toLowerCase().replace(/\s+/g, "-");
|
||||||
|
return CATEGORY_MAPPINGS[normalized] ?? 0; // Default to all categories
|
||||||
|
}
|
||||||
|
return 0; // Default to all categories
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build search URL with enhanced parameters
|
||||||
|
*/
|
||||||
|
export function buildSearchUrl(
|
||||||
|
keywords: string,
|
||||||
|
options: SearchOptions & { page?: number },
|
||||||
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
|
): string {
|
||||||
|
const locationId = resolveLocationId(options.location);
|
||||||
|
const categoryId = resolveCategoryId(options.category);
|
||||||
|
|
||||||
|
const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell";
|
||||||
|
const locationSlug = locationId === 0 ? "canada" : "canada";
|
||||||
|
|
||||||
|
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
|
||||||
|
|
||||||
|
const sortParam = options.sortBy
|
||||||
|
? `&sort=${SORT_MAPPINGS[options.sortBy]}`
|
||||||
|
: "";
|
||||||
|
const sortOrder = options.sortOrder === "asc" ? "ASC" : "DESC";
|
||||||
|
const pageParam =
|
||||||
|
options.page && options.page > 1 ? `&page=${options.page}` : "";
|
||||||
|
|
||||||
|
url += `?sort=relevancyDesc&view=list${sortParam}&order=${sortOrder}${pageParam}`;
|
||||||
|
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Slugifies a string for Kijiji search URLs
|
||||||
|
*/
|
||||||
|
export function slugify(input: string): string {
|
||||||
|
const s = unidecode(input).toLowerCase();
|
||||||
|
const out: string[] = [];
|
||||||
|
let lastHyphen = false;
|
||||||
|
|
||||||
|
for (let i = 0; i < s.length; i++) {
|
||||||
|
const ch = s[i];
|
||||||
|
if (!ch) continue;
|
||||||
|
const code = ch.charCodeAt(0);
|
||||||
|
|
||||||
|
// a-z or 0-9
|
||||||
|
if ((code >= 97 && code <= 122) || (code >= 48 && code <= 57)) {
|
||||||
|
out.push(ch);
|
||||||
|
lastHyphen = false;
|
||||||
|
} else if (SEPS.has(ch)) {
|
||||||
|
if (!lastHyphen) {
|
||||||
|
out.push("-");
|
||||||
|
lastHyphen = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// else drop character
|
||||||
|
}
|
||||||
|
return out.join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- GraphQL Client -----------------------------
|
||||||
|
|
||||||
|
// GraphQL response interfaces
|
||||||
|
interface GraphQLReviewResponse {
|
||||||
|
user?: {
|
||||||
|
reviewSummary?: {
|
||||||
|
count?: number;
|
||||||
|
score?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GraphQLProfileResponse {
|
||||||
|
user?: {
|
||||||
|
memberSince?: string;
|
||||||
|
accountType?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraphQL queries
|
||||||
|
const GRAPHQL_QUERIES = {
|
||||||
|
getReviewSummary: `
|
||||||
|
query GetReviewSummary($userId: String!) {
|
||||||
|
user(id: $userId) {
|
||||||
|
reviewSummary {
|
||||||
|
count
|
||||||
|
score
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
getProfileMetrics: `
|
||||||
|
query GetProfileMetrics($profileId: String!) {
|
||||||
|
user(id: $profileId) {
|
||||||
|
memberSince
|
||||||
|
accountType
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch additional data via GraphQL API
|
||||||
|
*/
|
||||||
|
async function fetchGraphQLData(
|
||||||
|
query: string,
|
||||||
|
variables: Record<string, unknown>,
|
||||||
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
|
): Promise<unknown> {
|
||||||
|
const endpoint = `${BASE_URL}/anvil/api`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(endpoint, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"apollo-require-preflight": "true",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
query,
|
||||||
|
variables,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new HttpError(
|
||||||
|
`GraphQL request failed with status ${response.status}`,
|
||||||
|
response.status,
|
||||||
|
endpoint,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (result.errors) {
|
||||||
|
throw new ParseError(
|
||||||
|
`GraphQL errors: ${JSON.stringify(result.errors)}`,
|
||||||
|
result.errors,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.data;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof HttpError || err instanceof ParseError) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
throw new NetworkError(
|
||||||
|
`Failed to fetch GraphQL data: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
endpoint,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch additional seller data via GraphQL
|
||||||
|
*/
|
||||||
|
async function fetchSellerDetails(
|
||||||
|
posterId: string,
|
||||||
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
|
): Promise<{
|
||||||
|
reviewCount?: number;
|
||||||
|
reviewScore?: number;
|
||||||
|
memberSince?: string;
|
||||||
|
accountType?: string;
|
||||||
|
}> {
|
||||||
|
try {
|
||||||
|
const [reviewData, profileData] = await Promise.all([
|
||||||
|
fetchGraphQLData(
|
||||||
|
GRAPHQL_QUERIES.getReviewSummary,
|
||||||
|
{ userId: posterId },
|
||||||
|
BASE_URL,
|
||||||
|
),
|
||||||
|
fetchGraphQLData(
|
||||||
|
GRAPHQL_QUERIES.getProfileMetrics,
|
||||||
|
{ profileId: posterId },
|
||||||
|
BASE_URL,
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const reviewResponse = reviewData as GraphQLReviewResponse;
|
||||||
|
const profileResponse = profileData as GraphQLProfileResponse;
|
||||||
|
|
||||||
|
return {
|
||||||
|
reviewCount: reviewResponse?.user?.reviewSummary?.count,
|
||||||
|
reviewScore: reviewResponse?.user?.reviewSummary?.score,
|
||||||
|
memberSince: profileResponse?.user?.memberSince,
|
||||||
|
accountType: profileResponse?.user?.accountType,
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
// Silently fail for GraphQL errors - not critical for basic functionality
|
||||||
|
console.warn(
|
||||||
|
`Failed to fetch seller details for ${posterId}:`,
|
||||||
|
err instanceof Error ? err.message : String(err),
|
||||||
|
);
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Parsing -----------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML.
|
||||||
|
*/
|
||||||
|
export function extractApolloState(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
): ApolloRecord | null {
|
||||||
|
const { document } = parseHTML(htmlString);
|
||||||
|
const nextData = document.getElementById("__NEXT_DATA__");
|
||||||
|
if (!nextData || !nextData.textContent) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const jsonData = JSON.parse(nextData.textContent);
|
||||||
|
const apollo = jsonData?.props?.pageProps?.__APOLLO_STATE__;
|
||||||
|
return isRecord(apollo) ? apollo : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
Parse search page apollo state into SearchListing[].
|
||||||
|
Filters keys likely to be listing entities and ensures url/title exist.
|
||||||
|
*/
|
||||||
|
export function parseSearch(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
BASE_URL: string,
|
||||||
|
): SearchListing[] {
|
||||||
|
const apolloState = extractApolloState(htmlString);
|
||||||
|
if (!apolloState) return [];
|
||||||
|
|
||||||
|
const results: SearchListing[] = [];
|
||||||
|
for (const [key, value] of Object.entries(apolloState)) {
|
||||||
|
// Heuristic: Kijiji listing keys usually contain "Listing"
|
||||||
|
if (!key.includes("Listing")) continue;
|
||||||
|
if (!isRecord(value)) continue;
|
||||||
|
|
||||||
|
const item = value as ApolloSearchItem;
|
||||||
|
if (typeof item.url === "string" && typeof item.title === "string") {
|
||||||
|
results.push({
|
||||||
|
listingLink: item.url.startsWith("http")
|
||||||
|
? item.url
|
||||||
|
: `${BASE_URL}${item.url}`,
|
||||||
|
name: item.title,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
Parse a listing page into a typed object (backward compatible).
|
||||||
|
*/
|
||||||
|
function _parseListing(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
BASE_URL: string,
|
||||||
|
): KijijiListingDetails | null {
|
||||||
|
const apolloState = extractApolloState(htmlString);
|
||||||
|
if (!apolloState) return null;
|
||||||
|
|
||||||
|
// Find the listing root key
|
||||||
|
const listingKey = Object.keys(apolloState).find((k) =>
|
||||||
|
k.includes("Listing"),
|
||||||
|
);
|
||||||
|
if (!listingKey) return null;
|
||||||
|
|
||||||
|
const root = apolloState[listingKey];
|
||||||
|
if (!isRecord(root)) return null;
|
||||||
|
|
||||||
|
const {
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
price,
|
||||||
|
type,
|
||||||
|
status,
|
||||||
|
activationDate,
|
||||||
|
endDate,
|
||||||
|
metrics,
|
||||||
|
location,
|
||||||
|
} = root as ApolloListingRoot;
|
||||||
|
|
||||||
|
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
||||||
|
const amountFormatted =
|
||||||
|
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
|
||||||
|
|
||||||
|
const numberOfViews =
|
||||||
|
metrics?.views != null ? Number(metrics.views) : undefined;
|
||||||
|
|
||||||
|
const listingUrl =
|
||||||
|
typeof url === "string"
|
||||||
|
? url.startsWith("http")
|
||||||
|
? url
|
||||||
|
: `${BASE_URL}${url}`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
if (!listingUrl || !title) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: listingUrl,
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
listingPrice: amountFormatted
|
||||||
|
? {
|
||||||
|
amountFormatted,
|
||||||
|
cents:
|
||||||
|
cents !== undefined && Number.isFinite(cents) ? cents : undefined,
|
||||||
|
currency: price?.currency,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
listingType: type,
|
||||||
|
listingStatus: status,
|
||||||
|
creationDate: activationDate,
|
||||||
|
endDate,
|
||||||
|
numberOfViews:
|
||||||
|
numberOfViews !== undefined && Number.isFinite(numberOfViews)
|
||||||
|
? numberOfViews
|
||||||
|
: undefined,
|
||||||
|
address: location?.address ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse a listing page into a detailed object with all available fields
|
||||||
|
*/
|
||||||
|
export async function parseDetailedListing(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
BASE_URL: string,
|
||||||
|
options: ListingFetchOptions = {},
|
||||||
|
): Promise<DetailedListing | null> {
|
||||||
|
const apolloState = extractApolloState(htmlString);
|
||||||
|
if (!apolloState) return null;
|
||||||
|
|
||||||
|
// Find the listing root key
|
||||||
|
const listingKey = Object.keys(apolloState).find((k) =>
|
||||||
|
k.includes("Listing"),
|
||||||
|
);
|
||||||
|
if (!listingKey) return null;
|
||||||
|
|
||||||
|
const root = apolloState[listingKey];
|
||||||
|
if (!isRecord(root)) return null;
|
||||||
|
|
||||||
|
const {
|
||||||
|
url,
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
price,
|
||||||
|
type,
|
||||||
|
status,
|
||||||
|
activationDate,
|
||||||
|
endDate,
|
||||||
|
metrics,
|
||||||
|
location,
|
||||||
|
imageUrls,
|
||||||
|
categoryId,
|
||||||
|
adSource,
|
||||||
|
flags,
|
||||||
|
posterInfo,
|
||||||
|
attributes,
|
||||||
|
} = root as ApolloListingRoot;
|
||||||
|
|
||||||
|
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
||||||
|
const amountFormatted =
|
||||||
|
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
|
||||||
|
|
||||||
|
const numberOfViews =
|
||||||
|
metrics?.views != null ? Number(metrics.views) : undefined;
|
||||||
|
|
||||||
|
const listingUrl =
|
||||||
|
typeof url === "string"
|
||||||
|
? url.startsWith("http")
|
||||||
|
? url
|
||||||
|
: `${BASE_URL}${url}`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
if (!listingUrl || !title) return null;
|
||||||
|
|
||||||
|
// Only include fixed-price listings
|
||||||
|
if (!amountFormatted || cents === undefined) return null;
|
||||||
|
|
||||||
|
// Extract images if requested
|
||||||
|
const images =
|
||||||
|
options.includeImages !== false && Array.isArray(imageUrls)
|
||||||
|
? imageUrls.filter((url): url is string => typeof url === "string")
|
||||||
|
: [];
|
||||||
|
|
||||||
|
// Extract attributes as key-value pairs
|
||||||
|
const attributeMap: Record<string, string[]> = {};
|
||||||
|
if (Array.isArray(attributes)) {
|
||||||
|
for (const attr of attributes) {
|
||||||
|
if (attr?.canonicalName && Array.isArray(attr.canonicalValues)) {
|
||||||
|
attributeMap[attr.canonicalName] = attr.canonicalValues;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract seller info based on depth setting
|
||||||
|
let sellerInfo: DetailedListing["sellerInfo"];
|
||||||
|
const depth = options.sellerDataDepth ?? "detailed";
|
||||||
|
|
||||||
|
if (posterInfo?.posterId) {
|
||||||
|
sellerInfo = {
|
||||||
|
posterId: posterInfo.posterId,
|
||||||
|
rating:
|
||||||
|
typeof posterInfo.rating === "number" ? posterInfo.rating : undefined,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Add more detailed info if requested and client-side data is enabled
|
||||||
|
if (
|
||||||
|
(depth === "detailed" || depth === "full") &&
|
||||||
|
options.includeClientSideData
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const additionalData = await fetchSellerDetails(
|
||||||
|
posterInfo.posterId,
|
||||||
|
BASE_URL,
|
||||||
|
);
|
||||||
|
sellerInfo = {
|
||||||
|
...sellerInfo,
|
||||||
|
...additionalData,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
// Silently fail - GraphQL data is optional
|
||||||
|
console.warn(
|
||||||
|
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: listingUrl,
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
listingPrice: {
|
||||||
|
amountFormatted,
|
||||||
|
cents,
|
||||||
|
currency: price?.currency,
|
||||||
|
},
|
||||||
|
listingType: type,
|
||||||
|
listingStatus: status,
|
||||||
|
creationDate: activationDate,
|
||||||
|
endDate,
|
||||||
|
numberOfViews:
|
||||||
|
numberOfViews !== undefined && Number.isFinite(numberOfViews)
|
||||||
|
? numberOfViews
|
||||||
|
: undefined,
|
||||||
|
address: location?.address ?? null,
|
||||||
|
images,
|
||||||
|
categoryId: typeof categoryId === "number" ? categoryId : 0,
|
||||||
|
adSource: typeof adSource === "string" ? adSource : "UNKNOWN",
|
||||||
|
flags: {
|
||||||
|
topAd: flags?.topAd === true,
|
||||||
|
priceDrop: flags?.priceDrop === true,
|
||||||
|
},
|
||||||
|
attributes: attributeMap,
|
||||||
|
location: {
|
||||||
|
id: typeof location?.id === "number" ? location.id : 0,
|
||||||
|
name: typeof location?.name === "string" ? location.name : "Unknown",
|
||||||
|
coordinates: location?.coordinates
|
||||||
|
? {
|
||||||
|
latitude: location.coordinates.latitude,
|
||||||
|
longitude: location.coordinates.longitude,
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
|
},
|
||||||
|
sellerInfo,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Main -----------------------------
|
||||||
|
|
||||||
|
export default async function fetchKijijiItems(
|
||||||
|
SEARCH_QUERY: string,
|
||||||
|
REQUESTS_PER_SECOND = 1,
|
||||||
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
|
searchOptions: SearchOptions = {},
|
||||||
|
listingOptions: ListingFetchOptions = {},
|
||||||
|
) {
|
||||||
|
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
|
||||||
|
|
||||||
|
// Load Kijiji cookies (optional - helps bypass bot detection)
|
||||||
|
const cookies = await loadCookiesOptional(
|
||||||
|
KIJIJI_COOKIE_CONFIG,
|
||||||
|
searchOptions.cookies,
|
||||||
|
);
|
||||||
|
const cookieHeader =
|
||||||
|
cookies.length > 0
|
||||||
|
? formatCookiesForHeader(cookies, "www.kijiji.ca")
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Set defaults for configuration
|
||||||
|
const finalSearchOptions: Required<SearchOptions> = {
|
||||||
|
location: searchOptions.location ?? 1700272, // Default to GTA
|
||||||
|
category: searchOptions.category ?? 0, // Default to all categories
|
||||||
|
keywords: searchOptions.keywords ?? SEARCH_QUERY,
|
||||||
|
sortBy: searchOptions.sortBy ?? "relevancy",
|
||||||
|
sortOrder: searchOptions.sortOrder ?? "desc",
|
||||||
|
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
|
||||||
|
priceMin: searchOptions.priceMin as number,
|
||||||
|
priceMax: searchOptions.priceMax as number,
|
||||||
|
cookies: searchOptions.cookies ?? "",
|
||||||
|
};
|
||||||
|
|
||||||
|
const finalListingOptions: Required<ListingFetchOptions> = {
|
||||||
|
includeImages: listingOptions.includeImages ?? true,
|
||||||
|
sellerDataDepth: listingOptions.sellerDataDepth ?? "detailed",
|
||||||
|
includeClientSideData: listingOptions.includeClientSideData ?? false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const allListings: DetailedListing[] = [];
|
||||||
|
const seenUrls = new Set<string>();
|
||||||
|
|
||||||
|
// Fetch multiple pages
|
||||||
|
for (let page = 1; page <= finalSearchOptions.maxPages; page++) {
|
||||||
|
const searchUrl = buildSearchUrl(
|
||||||
|
finalSearchOptions.keywords,
|
||||||
|
{
|
||||||
|
...finalSearchOptions,
|
||||||
|
// Add page parameter for pagination
|
||||||
|
...(page > 1 && { page }),
|
||||||
|
},
|
||||||
|
BASE_URL,
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log(`Fetching search page ${page}: ${searchUrl}`);
|
||||||
|
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
||||||
|
onRateInfo: (remaining, reset) => {
|
||||||
|
if (remaining && reset) {
|
||||||
|
console.log(
|
||||||
|
`\nSearch - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
const searchResults = parseSearch(searchHtml, BASE_URL);
|
||||||
|
if (searchResults.length === 0) {
|
||||||
|
console.log(
|
||||||
|
`No more results found on page ${page}. Stopping pagination.`,
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deduplicate links across pages
|
||||||
|
const newListingLinks = searchResults
|
||||||
|
.map((r) => r.listingLink)
|
||||||
|
.filter((link) => !seenUrls.has(link));
|
||||||
|
|
||||||
|
for (const link of newListingLinks) {
|
||||||
|
seenUrls.add(link);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Fetch details for this page's listings with controlled concurrency
|
||||||
|
const isTTY = process.stdout?.isTTY ?? false;
|
||||||
|
const progressBar = isTTY
|
||||||
|
? new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
|
||||||
|
: null;
|
||||||
|
const totalProgress = newListingLinks.length;
|
||||||
|
let currentProgress = 0;
|
||||||
|
progressBar?.start(totalProgress, currentProgress);
|
||||||
|
|
||||||
|
// Process in batches for controlled concurrency
|
||||||
|
const CONCURRENT_REQUESTS = REQUESTS_PER_SECOND * 2; // 2x rate for faster processing
|
||||||
|
const results: (DetailedListing | null)[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < newListingLinks.length; i += CONCURRENT_REQUESTS) {
|
||||||
|
const batch = newListingLinks.slice(i, i + CONCURRENT_REQUESTS);
|
||||||
|
const batchPromises = batch.map(async (link) => {
|
||||||
|
try {
|
||||||
|
const html = await fetchHtml(link, 0, {
|
||||||
|
// No per-request delay, batch handles rate limit
|
||||||
|
onRateInfo: (remaining, reset) => {
|
||||||
|
if (remaining && reset) {
|
||||||
|
console.log(
|
||||||
|
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
|
||||||
|
});
|
||||||
|
const parsed = await parseDetailedListing(
|
||||||
|
html,
|
||||||
|
BASE_URL,
|
||||||
|
finalListingOptions,
|
||||||
|
);
|
||||||
|
return parsed;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof HttpError) {
|
||||||
|
console.error(
|
||||||
|
`\nFailed to fetch ${link}\n - ${err.statusCode} ${err.message}`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
console.error(
|
||||||
|
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
currentProgress++;
|
||||||
|
progressBar?.update(currentProgress);
|
||||||
|
if (!progressBar) {
|
||||||
|
console.log(`Progress: ${currentProgress}/${totalProgress}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const batchResults = await Promise.all(batchPromises);
|
||||||
|
results.push(...batchResults);
|
||||||
|
|
||||||
|
// Wait between batches to respect rate limit
|
||||||
|
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, DELAY_MS * batch.length),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
allListings.push(
|
||||||
|
...results.filter((r): r is DetailedListing => r !== null),
|
||||||
|
);
|
||||||
|
|
||||||
|
progressBar?.stop();
|
||||||
|
|
||||||
|
// If we got fewer results than expected (40 per page), we've reached the end
|
||||||
|
if (searchResults.length < 40) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\nParsed ${allListings.length} detailed listings.`);
|
||||||
|
return allListings;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-export error classes for convenience
|
||||||
|
export { HttpError, NetworkError, ParseError, RateLimitError, ValidationError };
|
||||||
20
packages/core/src/types/common.ts
Normal file
20
packages/core/src/types/common.ts
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
/** HTML string alias for better type clarity */
|
||||||
|
export type HTMLString = string;
|
||||||
|
|
||||||
|
/** Currency price object with formatting options */
|
||||||
|
export interface Price {
|
||||||
|
amountFormatted: string;
|
||||||
|
cents: number;
|
||||||
|
currency: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Base listing details common across all marketplaces */
|
||||||
|
export interface ListingDetails {
|
||||||
|
url: string;
|
||||||
|
title: string;
|
||||||
|
listingPrice: Price;
|
||||||
|
listingType: string;
|
||||||
|
listingStatus: string;
|
||||||
|
address?: string | null;
|
||||||
|
creationDate?: string;
|
||||||
|
}
|
||||||
227
packages/core/src/utils/cookies.ts
Normal file
227
packages/core/src/utils/cookies.ts
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
/**
|
||||||
|
* Shared cookie handling utilities for marketplace scrapers
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface Cookie {
|
||||||
|
name: string;
|
||||||
|
value: string;
|
||||||
|
domain: string;
|
||||||
|
path: string;
|
||||||
|
secure?: boolean;
|
||||||
|
httpOnly?: boolean;
|
||||||
|
sameSite?: "strict" | "lax" | "none" | "unspecified";
|
||||||
|
session?: boolean;
|
||||||
|
expirationDate?: number;
|
||||||
|
partitionKey?: Record<string, unknown>;
|
||||||
|
storeId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CookieConfig {
|
||||||
|
/** Name used in log messages (e.g., "Facebook", "Kijiji") */
|
||||||
|
name: string;
|
||||||
|
/** Domain for cookies (e.g., ".facebook.com", ".kijiji.ca") */
|
||||||
|
domain: string;
|
||||||
|
/** Environment variable name (e.g., "FACEBOOK_COOKIE") */
|
||||||
|
envVar: string;
|
||||||
|
/** Path to cookie file (e.g., "./cookies/facebook.json") */
|
||||||
|
filePath: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse cookie string format into Cookie array
|
||||||
|
* Supports format: "name1=value1; name2=value2"
|
||||||
|
*/
|
||||||
|
export function parseCookieString(
|
||||||
|
cookieString: string,
|
||||||
|
domain: string,
|
||||||
|
): Cookie[] {
|
||||||
|
if (!cookieString?.trim()) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return cookieString
|
||||||
|
.split(";")
|
||||||
|
.map((pair) => pair.trim())
|
||||||
|
.filter((pair) => pair.includes("="))
|
||||||
|
.map((pair) => {
|
||||||
|
const [name, ...valueParts] = pair.split("=");
|
||||||
|
const trimmedName = name.trim();
|
||||||
|
const trimmedValue = valueParts.join("=").trim();
|
||||||
|
|
||||||
|
if (!trimmedName || !trimmedValue) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: trimmedName,
|
||||||
|
value: decodeURIComponent(trimmedValue),
|
||||||
|
domain,
|
||||||
|
path: "/",
|
||||||
|
secure: true,
|
||||||
|
httpOnly: false,
|
||||||
|
sameSite: "lax" as const,
|
||||||
|
expirationDate: undefined,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((cookie): cookie is Cookie => cookie !== null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse JSON array format into Cookie array
|
||||||
|
* Supports format: [{"name": "foo", "value": "bar", ...}]
|
||||||
|
*/
|
||||||
|
export function parseJsonCookies(jsonString: string): Cookie[] {
|
||||||
|
const parsed = JSON.parse(jsonString);
|
||||||
|
if (!Array.isArray(parsed)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed.filter(
|
||||||
|
(cookie): cookie is Cookie =>
|
||||||
|
cookie &&
|
||||||
|
typeof cookie.name === "string" &&
|
||||||
|
typeof cookie.value === "string",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Try to parse cookies from a string (tries JSON first, then cookie string format)
|
||||||
|
*/
|
||||||
|
export function parseCookiesAuto(
|
||||||
|
input: string,
|
||||||
|
defaultDomain: string,
|
||||||
|
): Cookie[] {
|
||||||
|
// Try JSON array format first
|
||||||
|
try {
|
||||||
|
const cookies = parseJsonCookies(input);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// JSON parse failed, try cookie string format
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try cookie string format
|
||||||
|
return parseCookieString(input, defaultDomain);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load cookies from file (supports both JSON array and cookie string formats)
|
||||||
|
*/
|
||||||
|
export async function loadCookiesFromFile(
|
||||||
|
filePath: string,
|
||||||
|
defaultDomain: string,
|
||||||
|
): Promise<Cookie[]> {
|
||||||
|
const file = Bun.file(filePath);
|
||||||
|
if (!(await file.exists())) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const content = await file.text();
|
||||||
|
return parseCookiesAuto(content.trim(), defaultDomain);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format cookies array into Cookie header string for HTTP requests
|
||||||
|
*/
|
||||||
|
export function formatCookiesForHeader(
|
||||||
|
cookies: Cookie[],
|
||||||
|
targetDomain: string,
|
||||||
|
): string {
|
||||||
|
const validCookies = cookies
|
||||||
|
.filter((cookie) => {
|
||||||
|
// Check if cookie applies to this domain
|
||||||
|
if (cookie.domain.startsWith(".")) {
|
||||||
|
// Domain cookie (applies to subdomains)
|
||||||
|
return (
|
||||||
|
targetDomain.endsWith(cookie.domain.slice(1)) ||
|
||||||
|
targetDomain === cookie.domain.slice(1)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Host-only cookie
|
||||||
|
return cookie.domain === targetDomain;
|
||||||
|
})
|
||||||
|
.filter((cookie) => {
|
||||||
|
// Check expiration
|
||||||
|
if (cookie.expirationDate && cookie.expirationDate < Date.now() / 1000) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
|
return validCookies
|
||||||
|
.map((cookie) => `${cookie.name}=${cookie.value}`)
|
||||||
|
.join("; ");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load cookies with priority: URL param > ENV var > file
|
||||||
|
* Supports both JSON array and cookie string formats for all sources
|
||||||
|
*/
|
||||||
|
export async function ensureCookies(
|
||||||
|
config: CookieConfig,
|
||||||
|
cookiesSource?: string,
|
||||||
|
): Promise<Cookie[]> {
|
||||||
|
// Priority 1: URL/API parameter (if provided)
|
||||||
|
if (cookiesSource) {
|
||||||
|
const cookies = parseCookiesAuto(cookiesSource, config.domain);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
console.log(
|
||||||
|
`Loaded ${cookies.length} ${config.name} cookies from parameter`,
|
||||||
|
);
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
console.warn(
|
||||||
|
`${config.name} cookies parameter provided but no valid cookies extracted`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Environment variable
|
||||||
|
const envValue = process.env[config.envVar];
|
||||||
|
if (envValue?.trim()) {
|
||||||
|
const cookies = parseCookiesAuto(envValue, config.domain);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
console.log(
|
||||||
|
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
|
||||||
|
);
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
console.warn(`${config.envVar} env var contains no valid cookies`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: Cookie file (fallback)
|
||||||
|
try {
|
||||||
|
const cookies = await loadCookiesFromFile(config.filePath, config.domain);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
console.log(
|
||||||
|
`Loaded ${cookies.length} ${config.name} cookies from ${config.filePath}`,
|
||||||
|
);
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(`Could not load cookies from ${config.filePath}: ${e}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// No cookies found from any source
|
||||||
|
throw new Error(
|
||||||
|
`No valid ${config.name} cookies found. Provide cookies via (in priority order):\n` +
|
||||||
|
` 1. 'cookies' parameter (highest priority), or\n` +
|
||||||
|
` 2. ${config.envVar} environment variable, or\n` +
|
||||||
|
` 3. ${config.filePath} file (lowest priority)\n` +
|
||||||
|
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Try to load cookies, return empty array if none found (non-throwing version)
|
||||||
|
*/
|
||||||
|
export async function loadCookiesOptional(
|
||||||
|
config: CookieConfig,
|
||||||
|
cookiesSource?: string,
|
||||||
|
): Promise<Cookie[]> {
|
||||||
|
try {
|
||||||
|
return await ensureCookies(config, cookiesSource);
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
8
packages/core/src/utils/delay.ts
Normal file
8
packages/core/src/utils/delay.ts
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
/**
|
||||||
|
* Delay execution for a specified number of milliseconds
|
||||||
|
* @param ms - Milliseconds to delay
|
||||||
|
* @returns A promise that resolves after the specified delay
|
||||||
|
*/
|
||||||
|
export function delay(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
24
packages/core/src/utils/format.ts
Normal file
24
packages/core/src/utils/format.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
/**
|
||||||
|
* Format cents to a human-readable currency string
|
||||||
|
* @param cents - Amount in cents (integer)
|
||||||
|
* @param locale - Locale string for formatting (e.g., 'en-CA', 'en-US')
|
||||||
|
* @returns Formatted currency string
|
||||||
|
*/
|
||||||
|
export function formatCentsToCurrency(
|
||||||
|
cents: number,
|
||||||
|
locale: string = "en-CA",
|
||||||
|
): string {
|
||||||
|
try {
|
||||||
|
const formatter = new Intl.NumberFormat(locale, {
|
||||||
|
style: "currency",
|
||||||
|
currency: "CAD",
|
||||||
|
minimumFractionDigits: 2,
|
||||||
|
maximumFractionDigits: 2,
|
||||||
|
});
|
||||||
|
return formatter.format(cents / 100);
|
||||||
|
} catch {
|
||||||
|
// Fallback if locale is not supported
|
||||||
|
const dollars = (cents / 100).toFixed(2);
|
||||||
|
return `$${dollars}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
200
packages/core/src/utils/http.ts
Normal file
200
packages/core/src/utils/http.ts
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
/** Custom error class for HTTP-related failures */
|
||||||
|
export class HttpError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly statusCode: number,
|
||||||
|
public readonly url?: string,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "HttpError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for network failures (timeouts, connection issues) */
|
||||||
|
export class NetworkError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly url: string,
|
||||||
|
public readonly cause?: Error,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "NetworkError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for parsing failures */
|
||||||
|
export class ParseError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly data?: unknown,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ParseError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for rate limiting */
|
||||||
|
export class RateLimitError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly url: string,
|
||||||
|
public readonly resetTime?: number,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "RateLimitError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for validation failures */
|
||||||
|
export class ValidationError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ValidationError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Type guard to check if a value is a record (object) */
|
||||||
|
export function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate exponential backoff delay with jitter
|
||||||
|
*/
|
||||||
|
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
||||||
|
const exponentialDelay = baseMs * 2 ** attempt;
|
||||||
|
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
||||||
|
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Options for fetchHtml */
|
||||||
|
export interface FetchHtmlOptions {
|
||||||
|
maxRetries?: number;
|
||||||
|
retryBaseMs?: number;
|
||||||
|
timeoutMs?: number;
|
||||||
|
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch HTML content from a URL with automatic retries, timeout, and exponential backoff
|
||||||
|
* @param url - The URL to fetch
|
||||||
|
* @param delayMs - Delay in milliseconds between requests (rate limiting)
|
||||||
|
* @param opts - Optional fetch options
|
||||||
|
* @returns The HTML content as a string
|
||||||
|
* @throws HttpError, NetworkError, or RateLimitError on failure
|
||||||
|
*/
|
||||||
|
export async function fetchHtml(
|
||||||
|
url: string,
|
||||||
|
delayMs: number,
|
||||||
|
opts?: FetchHtmlOptions,
|
||||||
|
): Promise<string> {
|
||||||
|
const maxRetries = opts?.maxRetries ?? 3;
|
||||||
|
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
||||||
|
const timeoutMs = opts?.timeoutMs ?? 30000;
|
||||||
|
|
||||||
|
const defaultHeaders: Record<string, string> = {
|
||||||
|
accept:
|
||||||
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||||
|
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
"upgrade-insecure-requests": "1",
|
||||||
|
"user-agent":
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
||||||
|
};
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
|
try {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method: "GET",
|
||||||
|
headers: { ...defaultHeaders, ...opts?.headers },
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
|
||||||
|
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
||||||
|
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
||||||
|
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
// Handle rate limiting
|
||||||
|
if (res.status === 429) {
|
||||||
|
const resetSeconds = rateLimitReset
|
||||||
|
? Number(rateLimitReset)
|
||||||
|
: Number.NaN;
|
||||||
|
const waitMs = Number.isFinite(resetSeconds)
|
||||||
|
? Math.max(0, resetSeconds * 1000)
|
||||||
|
: calculateBackoffDelay(attempt, retryBaseMs);
|
||||||
|
|
||||||
|
if (attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new RateLimitError(
|
||||||
|
`Rate limit exceeded for ${url}`,
|
||||||
|
url,
|
||||||
|
resetSeconds,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retry on server errors
|
||||||
|
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new HttpError(
|
||||||
|
`Request failed with status ${res.status}`,
|
||||||
|
res.status,
|
||||||
|
url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = await res.text();
|
||||||
|
|
||||||
|
// Respect per-request delay to maintain rate limiting
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
|
return html;
|
||||||
|
} catch (err) {
|
||||||
|
// Re-throw known errors
|
||||||
|
if (
|
||||||
|
err instanceof RateLimitError ||
|
||||||
|
err instanceof HttpError ||
|
||||||
|
err instanceof NetworkError
|
||||||
|
) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (err instanceof Error && err.name === "AbortError") {
|
||||||
|
if (attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new NetworkError(`Request timeout for ${url}`, url, err);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Network or other errors
|
||||||
|
if (attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new NetworkError(
|
||||||
|
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
url,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
|
||||||
|
}
|
||||||
833
packages/core/test/facebook-core.test.ts
Normal file
833
packages/core/test/facebook-core.test.ts
Normal file
@@ -0,0 +1,833 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import {
|
||||||
|
extractFacebookItemData,
|
||||||
|
extractFacebookMarketplaceData,
|
||||||
|
fetchFacebookItem,
|
||||||
|
formatCentsToCurrency,
|
||||||
|
formatCookiesForHeader,
|
||||||
|
parseFacebookAds,
|
||||||
|
parseFacebookCookieString,
|
||||||
|
parseFacebookItem,
|
||||||
|
} from "../src/scrapers/facebook";
|
||||||
|
|
||||||
|
// Mock fetch globally
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Cookie Parsing", () => {
|
||||||
|
describe("parseFacebookCookieString", () => {
|
||||||
|
test("should parse valid cookie string", () => {
|
||||||
|
const cookieString = "c_user=123456789; xs=abcdef123456; fr=xyz789";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result).toHaveLength(3);
|
||||||
|
expect(result[0]).toEqual({
|
||||||
|
name: "c_user",
|
||||||
|
value: "123456789",
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
secure: true,
|
||||||
|
httpOnly: false,
|
||||||
|
sameSite: "lax",
|
||||||
|
expirationDate: undefined,
|
||||||
|
});
|
||||||
|
expect(result[1]).toEqual({
|
||||||
|
name: "xs",
|
||||||
|
value: "abcdef123456",
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
secure: true,
|
||||||
|
httpOnly: false,
|
||||||
|
sameSite: "lax",
|
||||||
|
expirationDate: undefined,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle URL-encoded values", () => {
|
||||||
|
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result[0].value).toBe("123+456");
|
||||||
|
expect(result[1].value).toBe("abc=def");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out malformed cookies", () => {
|
||||||
|
const cookieString = "c_user=123; invalid; xs=abc; =empty";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result.map((c) => c.name)).toEqual(["c_user", "xs"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty input", () => {
|
||||||
|
expect(parseFacebookCookieString("")).toEqual([]);
|
||||||
|
expect(parseFacebookCookieString(" ")).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle extra whitespace", () => {
|
||||||
|
const cookieString = " c_user = 123 ; xs=abc ";
|
||||||
|
const result = parseFacebookCookieString(cookieString);
|
||||||
|
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result[0].name).toBe("c_user");
|
||||||
|
expect(result[0].value).toBe("123");
|
||||||
|
expect(result[1].name).toBe("xs");
|
||||||
|
expect(result[1].value).toBe("abc");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Facebook Item Fetching", () => {
|
||||||
|
describe("fetchFacebookItem", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should handle authentication errors", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 401,
|
||||||
|
text: () => Promise.resolve("Authentication required"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("123", mockCookies);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle item not found", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
text: () => Promise.resolve("Not found"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("nonexistent", mockCookies);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle rate limiting", async () => {
|
||||||
|
let attempts = 0;
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
attempts++;
|
||||||
|
if (attempts === 1) {
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 429,
|
||||||
|
headers: {
|
||||||
|
get: (header: string) => {
|
||||||
|
if (header === "X-RateLimit-Reset") return "1";
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
text: () => Promise.resolve("Rate limited"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: {
|
||||||
|
id: "123",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Test Item",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const _result = await fetchFacebookItem("123", mockCookies);
|
||||||
|
expect(attempts).toBe(2);
|
||||||
|
// Should eventually succeed after retry
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle sold items", async () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: {
|
||||||
|
id: "456",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Sold Item",
|
||||||
|
is_sold: true,
|
||||||
|
is_live: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("456", mockCookies);
|
||||||
|
expect(result?.listingStatus).toBe("SOLD");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing authentication cookies", async () => {
|
||||||
|
// Use a test-specific cookie file that doesn't exist
|
||||||
|
const testCookiePath = "./cookies/facebook-test.json";
|
||||||
|
|
||||||
|
// Test with no cookies available (test file doesn't exist)
|
||||||
|
await expect(
|
||||||
|
fetchFacebookItem("123", undefined, testCookiePath),
|
||||||
|
).rejects.toThrow("No valid Facebook cookies found");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle successful item extraction", async () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: {
|
||||||
|
id: "789",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Working Item",
|
||||||
|
formatted_price: { text: "$299.00" },
|
||||||
|
listing_price: {
|
||||||
|
amount: "299.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
creation_time: 1640995200,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("789", mockCookies);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("Working Item");
|
||||||
|
expect(result?.listingPrice?.amountFormatted).toBe("$299.00");
|
||||||
|
expect(result?.listingStatus).toBe("ACTIVE");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle server errors", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 500,
|
||||||
|
text: () => Promise.resolve("Internal Server Error"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await fetchFacebookItem("error", mockCookies);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Data Extraction", () => {
|
||||||
|
describe("extractFacebookItemData", () => {
|
||||||
|
test("should extract item data from standard require structure", () => {
|
||||||
|
const mockItemData = {
|
||||||
|
id: "123456",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "Test Item",
|
||||||
|
formatted_price: { text: "$100.00" },
|
||||||
|
listing_price: { amount: "100.00", currency: "CAD" },
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: mockItemData,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.id).toBe("123456");
|
||||||
|
expect(result?.marketplace_listing_title).toBe("Test Item");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing item data", () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle malformed HTML", () => {
|
||||||
|
const result = extractFacebookItemData(
|
||||||
|
"<html><body>Invalid HTML</body></html>",
|
||||||
|
);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle invalid JSON in script tags", () => {
|
||||||
|
const html =
|
||||||
|
"<html><body><script>{invalid: json}</script></body></html>";
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should extract item with vehicle data", () => {
|
||||||
|
const mockVehicleItem = {
|
||||||
|
id: "789",
|
||||||
|
__typename: "GroupCommerceProductItem",
|
||||||
|
marketplace_listing_title: "2006 Honda Civic",
|
||||||
|
formatted_price: { text: "$5,000" },
|
||||||
|
listing_price: { amount: "5000.00", currency: "CAD" },
|
||||||
|
vehicle_make_display_name: "Honda",
|
||||||
|
vehicle_model_display_name: "Civic",
|
||||||
|
vehicle_odometer_data: { unit: "KILOMETERS", value: 150000 },
|
||||||
|
vehicle_transmission_type: "AUTOMATIC",
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
viewer: {
|
||||||
|
marketplace_product_details_page: {
|
||||||
|
target: mockVehicleItem,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookItemData(html);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.vehicle_make_display_name).toBe("Honda");
|
||||||
|
expect(result?.vehicle_odometer_data?.value).toBe(150000);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("extractFacebookMarketplaceData", () => {
|
||||||
|
test("should extract search results from marketplace data", () => {
|
||||||
|
const mockMarketplaceData = {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Item 1",
|
||||||
|
listing_price: { amount: "10.00", currency: "CAD" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "Item 2",
|
||||||
|
listing_price: { amount: "20.00", currency: "CAD" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: mockMarketplaceData,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookMarketplaceData(html);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
|
||||||
|
"Item 1",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty search results", () => {
|
||||||
|
const mockData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: { edges: [] },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||||
|
|
||||||
|
const result = extractFacebookMarketplaceData(html);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Data Parsing", () => {
|
||||||
|
describe("parseFacebookItem", () => {
|
||||||
|
test("should parse complete item with all fields", () => {
|
||||||
|
const item = {
|
||||||
|
id: "123456",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "iPhone 13 Pro",
|
||||||
|
redacted_description: { text: "Excellent condition" },
|
||||||
|
formatted_price: { text: "$800.00" },
|
||||||
|
listing_price: { amount: "800.00", currency: "CAD" },
|
||||||
|
location_text: { text: "Toronto, ON" },
|
||||||
|
is_live: true,
|
||||||
|
creation_time: 1640995200,
|
||||||
|
marketplace_listing_seller: {
|
||||||
|
id: "seller1",
|
||||||
|
name: "John Doe",
|
||||||
|
},
|
||||||
|
delivery_types: ["IN_PERSON"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(item);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("iPhone 13 Pro");
|
||||||
|
expect(result?.description).toBe("Excellent condition");
|
||||||
|
expect(result?.listingPrice?.amountFormatted).toBe("$800.00");
|
||||||
|
expect(result?.listingPrice?.cents).toBe(80000);
|
||||||
|
expect(result?.listingPrice?.currency).toBe("CAD");
|
||||||
|
expect(result?.address).toBe("Toronto, ON");
|
||||||
|
expect(result?.listingStatus).toBe("ACTIVE");
|
||||||
|
expect(result?.seller?.name).toBe("John Doe");
|
||||||
|
expect(result?.deliveryTypes).toEqual(["IN_PERSON"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should parse FREE items", () => {
|
||||||
|
const item = {
|
||||||
|
id: "789",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Free Sofa",
|
||||||
|
formatted_price: { text: "FREE" },
|
||||||
|
listing_price: { amount: "0.00", currency: "CAD" },
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(item);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("Free Sofa");
|
||||||
|
expect(result?.listingPrice?.amountFormatted).toBe("FREE");
|
||||||
|
expect(result?.listingPrice?.cents).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing optional fields", () => {
|
||||||
|
const item = {
|
||||||
|
id: "456",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Minimal Item",
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(item);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result?.title).toBe("Minimal Item");
|
||||||
|
expect(result?.description).toBeUndefined();
|
||||||
|
expect(result?.seller).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should identify vehicle listings", () => {
|
||||||
|
const vehicleItem = {
|
||||||
|
id: "999",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "2012 Mazda 3",
|
||||||
|
formatted_price: { text: "$8,000" },
|
||||||
|
listing_price: { amount: "8000.00", currency: "CAD" },
|
||||||
|
vehicle_make_display_name: "Mazda",
|
||||||
|
vehicle_model_display_name: "3",
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(vehicleItem);
|
||||||
|
expect(result?.listingType).toBe("vehicle");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle different listing statuses", () => {
|
||||||
|
const soldItem = {
|
||||||
|
id: "111",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Sold Item",
|
||||||
|
is_sold: true,
|
||||||
|
is_live: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const pendingItem = {
|
||||||
|
id: "222",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Pending Item",
|
||||||
|
is_pending: true,
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const hiddenItem = {
|
||||||
|
id: "333",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
marketplace_listing_title: "Hidden Item",
|
||||||
|
is_hidden: true,
|
||||||
|
is_live: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(parseFacebookItem(soldItem)?.listingStatus).toBe("SOLD");
|
||||||
|
expect(parseFacebookItem(pendingItem)?.listingStatus).toBe("PENDING");
|
||||||
|
expect(parseFacebookItem(hiddenItem)?.listingStatus).toBe("HIDDEN");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return null for items without title", () => {
|
||||||
|
const invalidItem = {
|
||||||
|
id: "invalid",
|
||||||
|
__typename: "GroupCommerceProductItem" as const,
|
||||||
|
is_live: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = parseFacebookItem(invalidItem);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseFacebookAds", () => {
|
||||||
|
test("should parse search result ads", () => {
|
||||||
|
const ads = [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Ad 1",
|
||||||
|
listing_price: {
|
||||||
|
amount: "50.00",
|
||||||
|
formatted_amount: "$50.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: { city_page: { display_name: "Toronto" } },
|
||||||
|
},
|
||||||
|
creation_time: 1640995200,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "Ad 2",
|
||||||
|
listing_price: {
|
||||||
|
amount: "75.00",
|
||||||
|
formatted_amount: "$75.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: { city_page: { display_name: "Ottawa" } },
|
||||||
|
},
|
||||||
|
creation_time: 1640995300,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const results = parseFacebookAds(ads);
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
expect(results[0].title).toBe("Ad 1");
|
||||||
|
expect(results[0].listingPrice?.cents).toBe(5000);
|
||||||
|
expect(results[0].address).toBe("Toronto");
|
||||||
|
expect(results[1].title).toBe("Ad 2");
|
||||||
|
expect(results[1].address).toBe("Ottawa");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out ads without price", () => {
|
||||||
|
const ads = [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "With Price",
|
||||||
|
listing_price: {
|
||||||
|
amount: "100.00",
|
||||||
|
formatted_amount: "$100.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "No Price",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const results = parseFacebookAds(ads);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("With Price");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle malformed ads gracefully", () => {
|
||||||
|
const ads = [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Valid Ad",
|
||||||
|
listing_price: {
|
||||||
|
amount: "50.00",
|
||||||
|
formatted_amount: "$50.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
// Missing listing
|
||||||
|
},
|
||||||
|
} as { node: { listing?: unknown } },
|
||||||
|
];
|
||||||
|
|
||||||
|
const results = parseFacebookAds(ads);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("Valid Ad");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Utility Functions", () => {
|
||||||
|
describe("formatCentsToCurrency", () => {
|
||||||
|
test("should format cents to currency string", () => {
|
||||||
|
expect(formatCentsToCurrency(100)).toBe("$1.00");
|
||||||
|
expect(formatCentsToCurrency(1000)).toBe("$10.00");
|
||||||
|
expect(formatCentsToCurrency(9999)).toBe("$99.99");
|
||||||
|
expect(formatCentsToCurrency(123456)).toBe("$1,234.56");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle string inputs", () => {
|
||||||
|
expect(formatCentsToCurrency("100")).toBe("$1.00");
|
||||||
|
expect(formatCentsToCurrency("1000")).toBe("$10.00");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle zero", () => {
|
||||||
|
expect(formatCentsToCurrency(0)).toBe("$0.00");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle null and undefined", () => {
|
||||||
|
expect(formatCentsToCurrency(null)).toBe("");
|
||||||
|
expect(formatCentsToCurrency(undefined)).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle invalid inputs", () => {
|
||||||
|
expect(formatCentsToCurrency("invalid")).toBe("");
|
||||||
|
expect(formatCentsToCurrency(Number.NaN)).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("formatCookiesForHeader", () => {
|
||||||
|
const mockCookies = [
|
||||||
|
{ name: "c_user", value: "123456", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abcdef", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "session_id", value: "xyz", domain: "other.com", path: "/" },
|
||||||
|
];
|
||||||
|
|
||||||
|
test("should format cookies for header string", () => {
|
||||||
|
const result = formatCookiesForHeader(mockCookies, "www.facebook.com");
|
||||||
|
expect(result).toBe("c_user=123456; xs=abcdef");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter expired cookies", () => {
|
||||||
|
const cookiesWithExpiration = [
|
||||||
|
...mockCookies,
|
||||||
|
{
|
||||||
|
name: "expired",
|
||||||
|
value: "old",
|
||||||
|
domain: ".facebook.com",
|
||||||
|
path: "/",
|
||||||
|
expirationDate: Date.now() / 1000 - 1000,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const result = formatCookiesForHeader(
|
||||||
|
cookiesWithExpiration,
|
||||||
|
"www.facebook.com",
|
||||||
|
);
|
||||||
|
expect(result).not.toContain("expired");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle no matching cookies", () => {
|
||||||
|
const result = formatCookiesForHeader(mockCookies, "www.google.com");
|
||||||
|
expect(result).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty cookie array", () => {
|
||||||
|
const result = formatCookiesForHeader([], "www.facebook.com");
|
||||||
|
expect(result).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
712
packages/core/test/facebook-integration.test.ts
Normal file
712
packages/core/test/facebook-integration.test.ts
Normal file
@@ -0,0 +1,712 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import { fetchFacebookItems } from "../src/scrapers/facebook";
|
||||||
|
|
||||||
|
// Mock fetch globally
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Main Search Function", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should successfully fetch search results", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "iPhone 13 Pro",
|
||||||
|
listing_price: {
|
||||||
|
amount: "800.00",
|
||||||
|
formatted_amount: "$800.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Toronto" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
creation_time: 1640995200,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "Samsung Galaxy",
|
||||||
|
listing_price: {
|
||||||
|
amount: "600.00",
|
||||||
|
formatted_amount: "$600.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Mississauga" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
creation_time: 1640995300,
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"iPhone",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
expect(results[0].title).toBe("iPhone 13 Pro");
|
||||||
|
expect(results[1].title).toBe("Samsung Galaxy");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out items without price", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "With Price",
|
||||||
|
listing_price: {
|
||||||
|
amount: "100.00",
|
||||||
|
formatted_amount: "$100.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "No Price",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("With Price");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should respect MAX_ITEMS parameter", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: Array.from({ length: 10 }, (_, i) => ({
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: String(i),
|
||||||
|
marketplace_listing_title: `Item ${i}`,
|
||||||
|
listing_price: {
|
||||||
|
amount: `${(i + 1) * 10}.00`,
|
||||||
|
formatted_amount: `$${(i + 1) * 10}.00`,
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
5,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return empty array for no results", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"nonexistent query",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle authentication errors gracefully", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 401,
|
||||||
|
text: () => Promise.resolve("Unauthorized"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle network errors", async () => {
|
||||||
|
global.fetch = mock(() => Promise.reject(new Error("Network error")));
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
fetchFacebookItems("test", 1, "toronto", 25, mockCookies),
|
||||||
|
).rejects.toThrow("Network error");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle rate limiting with retry", async () => {
|
||||||
|
let attempts = 0;
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
attempts++;
|
||||||
|
if (attempts === 1) {
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 429,
|
||||||
|
headers: {
|
||||||
|
get: (header: string) => {
|
||||||
|
if (header === "X-RateLimit-Reset") return "1";
|
||||||
|
return null;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
text: () => Promise.resolve("Rate limited"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Item 1",
|
||||||
|
listing_price: {
|
||||||
|
amount: "100.00",
|
||||||
|
formatted_amount: "$100.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
return Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(attempts).toBe(2);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Vehicle Listing Integration", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should correctly identify and parse vehicle listings", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "2006 Honda Civic",
|
||||||
|
listing_price: {
|
||||||
|
amount: "8000.00",
|
||||||
|
formatted_amount: "$8,000.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "2",
|
||||||
|
marketplace_listing_title: "iPhone 13",
|
||||||
|
listing_price: {
|
||||||
|
amount: "800.00",
|
||||||
|
formatted_amount: "$800.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"cars",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
// Both should be classified as "item" type in search results (vehicle detection is for item details)
|
||||||
|
expect(results[0].title).toBe("2006 Honda Civic");
|
||||||
|
expect(results[1].title).toBe("iPhone 13");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Different Categories", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should handle electronics listings", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Nintendo Switch",
|
||||||
|
listing_price: {
|
||||||
|
amount: "250.00",
|
||||||
|
formatted_amount: "$250.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Toronto" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
marketplace_listing_category_id:
|
||||||
|
"479353692612078",
|
||||||
|
condition: "USED",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"nintendo switch",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("Nintendo Switch");
|
||||||
|
expect(results[0].categoryId).toBe("479353692612078");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle home goods/furniture listings", async () => {
|
||||||
|
const mockSearchData = {
|
||||||
|
require: [
|
||||||
|
[
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
__bbox: {
|
||||||
|
result: {
|
||||||
|
data: {
|
||||||
|
marketplace_search: {
|
||||||
|
feed_units: {
|
||||||
|
edges: [
|
||||||
|
{
|
||||||
|
node: {
|
||||||
|
listing: {
|
||||||
|
id: "1",
|
||||||
|
marketplace_listing_title: "Dining Table",
|
||||||
|
listing_price: {
|
||||||
|
amount: "150.00",
|
||||||
|
formatted_amount: "$150.00",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Mississauga" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
marketplace_listing_category_id:
|
||||||
|
"1569171756675761",
|
||||||
|
condition: "USED",
|
||||||
|
is_live: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"table",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].title).toBe("Dining Table");
|
||||||
|
expect(results[0].categoryId).toBe("1569171756675761");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Error Scenarios", () => {
|
||||||
|
const mockCookies = JSON.stringify([
|
||||||
|
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||||
|
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
test("should handle malformed HTML responses", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: true,
|
||||||
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
"<html><body>Invalid HTML without JSON data</body></html>",
|
||||||
|
),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle 404 errors gracefully", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
text: () => Promise.resolve("Not found"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle 500 errors gracefully", async () => {
|
||||||
|
global.fetch = mock(() =>
|
||||||
|
Promise.resolve({
|
||||||
|
ok: false,
|
||||||
|
status: 500,
|
||||||
|
text: () => Promise.resolve("Internal Server Error"),
|
||||||
|
headers: {
|
||||||
|
get: () => null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await fetchFacebookItems(
|
||||||
|
"test",
|
||||||
|
1,
|
||||||
|
"toronto",
|
||||||
|
25,
|
||||||
|
mockCookies,
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
157
packages/core/test/kijiji-core.test.ts
Normal file
157
packages/core/test/kijiji-core.test.ts
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import {
|
||||||
|
buildSearchUrl,
|
||||||
|
NetworkError,
|
||||||
|
ParseError,
|
||||||
|
RateLimitError,
|
||||||
|
resolveCategoryId,
|
||||||
|
resolveLocationId,
|
||||||
|
ValidationError,
|
||||||
|
} from "../src/scrapers/kijiji";
|
||||||
|
|
||||||
|
describe("Location and Category Resolution", () => {
|
||||||
|
describe("resolveLocationId", () => {
|
||||||
|
test("should return numeric IDs as-is", () => {
|
||||||
|
expect(resolveLocationId(1700272)).toBe(1700272);
|
||||||
|
expect(resolveLocationId(0)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should resolve string location names", () => {
|
||||||
|
expect(resolveLocationId("canada")).toBe(0);
|
||||||
|
expect(resolveLocationId("ontario")).toBe(9004);
|
||||||
|
expect(resolveLocationId("toronto")).toBe(1700273);
|
||||||
|
expect(resolveLocationId("gta")).toBe(1700272);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle case insensitive matching", () => {
|
||||||
|
expect(resolveLocationId("Canada")).toBe(0);
|
||||||
|
expect(resolveLocationId("ONTARIO")).toBe(9004);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should default to Canada for unknown locations", () => {
|
||||||
|
expect(resolveLocationId("unknown")).toBe(0);
|
||||||
|
expect(resolveLocationId("")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle undefined input", () => {
|
||||||
|
expect(resolveLocationId(undefined)).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("resolveCategoryId", () => {
|
||||||
|
test("should return numeric IDs as-is", () => {
|
||||||
|
expect(resolveCategoryId(132)).toBe(132);
|
||||||
|
expect(resolveCategoryId(0)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should resolve string category names", () => {
|
||||||
|
expect(resolveCategoryId("all")).toBe(0);
|
||||||
|
expect(resolveCategoryId("phones")).toBe(132);
|
||||||
|
expect(resolveCategoryId("electronics")).toBe(29659001);
|
||||||
|
expect(resolveCategoryId("buy-sell")).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle case insensitive matching", () => {
|
||||||
|
expect(resolveCategoryId("All")).toBe(0);
|
||||||
|
expect(resolveCategoryId("PHONES")).toBe(132);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should default to all categories for unknown categories", () => {
|
||||||
|
expect(resolveCategoryId("unknown")).toBe(0);
|
||||||
|
expect(resolveCategoryId("")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle undefined input", () => {
|
||||||
|
expect(resolveCategoryId(undefined)).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("URL Construction", () => {
|
||||||
|
describe("buildSearchUrl", () => {
|
||||||
|
test("should build basic search URL", () => {
|
||||||
|
const url = buildSearchUrl("iphone", {
|
||||||
|
location: 1700272,
|
||||||
|
category: 132,
|
||||||
|
sortBy: "relevancy",
|
||||||
|
sortOrder: "desc",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(url).toContain("b-buy-sell/canada/iphone/k0c132l1700272");
|
||||||
|
expect(url).toContain("sort=relevancyDesc");
|
||||||
|
expect(url).toContain("order=DESC");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle pagination", () => {
|
||||||
|
const url = buildSearchUrl("iphone", {
|
||||||
|
location: 1700272,
|
||||||
|
category: 132,
|
||||||
|
page: 2,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(url).toContain("&page=2");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle different sort options", () => {
|
||||||
|
const dateUrl = buildSearchUrl("iphone", {
|
||||||
|
sortBy: "date",
|
||||||
|
sortOrder: "asc",
|
||||||
|
});
|
||||||
|
expect(dateUrl).toContain("sort=DATE");
|
||||||
|
expect(dateUrl).toContain("order=ASC");
|
||||||
|
|
||||||
|
const priceUrl = buildSearchUrl("iphone", {
|
||||||
|
sortBy: "price",
|
||||||
|
sortOrder: "desc",
|
||||||
|
});
|
||||||
|
expect(priceUrl).toContain("sort=PRICE");
|
||||||
|
expect(priceUrl).toContain("order=DESC");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle string location/category inputs", () => {
|
||||||
|
const url = buildSearchUrl("iphone", {
|
||||||
|
location: "toronto",
|
||||||
|
category: "phones",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(url).toContain("k0c132l1700273"); // phones + toronto
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Error Classes", () => {
|
||||||
|
test("NetworkError should store URL and cause", () => {
|
||||||
|
const cause = new Error("Connection failed");
|
||||||
|
const error = new NetworkError(
|
||||||
|
"Network error",
|
||||||
|
"https://example.com",
|
||||||
|
cause,
|
||||||
|
);
|
||||||
|
expect(error.message).toBe("Network error");
|
||||||
|
expect(error.url).toBe("https://example.com");
|
||||||
|
expect(error.cause).toBe(cause);
|
||||||
|
expect(error.name).toBe("NetworkError");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ParseError should store data", () => {
|
||||||
|
const data = { invalid: "json" };
|
||||||
|
const error = new ParseError("Invalid JSON", data);
|
||||||
|
expect(error.message).toBe("Invalid JSON");
|
||||||
|
expect(error.data).toBe(data);
|
||||||
|
expect(error.name).toBe("ParseError");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("RateLimitError should store URL and reset time", () => {
|
||||||
|
const error = new RateLimitError("Rate limited", "https://example.com", 60);
|
||||||
|
expect(error.message).toBe("Rate limited");
|
||||||
|
expect(error.url).toBe("https://example.com");
|
||||||
|
expect(error.resetTime).toBe(60);
|
||||||
|
expect(error.name).toBe("RateLimitError");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ValidationError should work without field", () => {
|
||||||
|
const error = new ValidationError("Invalid value");
|
||||||
|
expect(error.message).toBe("Invalid value");
|
||||||
|
expect(error.name).toBe("ValidationError");
|
||||||
|
});
|
||||||
|
});
|
||||||
363
packages/core/test/kijiji-integration.test.ts
Normal file
363
packages/core/test/kijiji-integration.test.ts
Normal file
@@ -0,0 +1,363 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
import {
|
||||||
|
extractApolloState,
|
||||||
|
parseDetailedListing,
|
||||||
|
parseSearch,
|
||||||
|
} from "../src/scrapers/kijiji";
|
||||||
|
|
||||||
|
// Mock fetch globally
|
||||||
|
const originalFetch = global.fetch;
|
||||||
|
|
||||||
|
describe("HTML Parsing Integration", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
// Mock fetch for all tests
|
||||||
|
global.fetch = mock(() => {
|
||||||
|
throw new Error("fetch should be mocked in individual tests");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
global.fetch = originalFetch;
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("extractApolloState", () => {
|
||||||
|
test("should extract Apollo state from valid HTML", () => {
|
||||||
|
const mockHtml =
|
||||||
|
'<html><head><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"__APOLLO_STATE__":{"ROOT_QUERY":{"test":"value"}}}}}</script></head></html>';
|
||||||
|
|
||||||
|
const result = extractApolloState(mockHtml);
|
||||||
|
expect(result).toEqual({
|
||||||
|
ROOT_QUERY: { test: "value" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return null for HTML without Apollo state", () => {
|
||||||
|
const mockHtml = "<html><body>No data here</body></html>";
|
||||||
|
const result = extractApolloState(mockHtml);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return null for malformed JSON", () => {
|
||||||
|
const mockHtml =
|
||||||
|
'<html><script id="__NEXT_DATA__" type="application/json">{"invalid": json}</script></html>';
|
||||||
|
|
||||||
|
const result = extractApolloState(mockHtml);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing __NEXT_DATA__ element", () => {
|
||||||
|
const mockHtml = "<html><body><div>Content</div></body></html>";
|
||||||
|
const result = extractApolloState(mockHtml);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseSearch", () => {
|
||||||
|
test("should parse search results from HTML", () => {
|
||||||
|
const mockHtml = `
|
||||||
|
<html>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
${JSON.stringify({
|
||||||
|
props: {
|
||||||
|
pageProps: {
|
||||||
|
__APOLLO_STATE__: {
|
||||||
|
"Listing:123": {
|
||||||
|
url: "/v-iphone/k0l0",
|
||||||
|
title: "iPhone 13 Pro",
|
||||||
|
},
|
||||||
|
"Listing:456": {
|
||||||
|
url: "/v-samsung/k0l0",
|
||||||
|
title: "Samsung Galaxy",
|
||||||
|
},
|
||||||
|
ROOT_QUERY: { test: "value" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||||
|
expect(results).toHaveLength(2);
|
||||||
|
expect(results[0]).toEqual({
|
||||||
|
name: "iPhone 13 Pro",
|
||||||
|
listingLink: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
|
});
|
||||||
|
expect(results[1]).toEqual({
|
||||||
|
name: "Samsung Galaxy",
|
||||||
|
listingLink: "https://www.kijiji.ca/v-samsung/k0l0",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle absolute URLs", () => {
|
||||||
|
const mockHtml = `
|
||||||
|
<html>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
${JSON.stringify({
|
||||||
|
props: {
|
||||||
|
pageProps: {
|
||||||
|
__APOLLO_STATE__: {
|
||||||
|
"Listing:123": {
|
||||||
|
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
|
title: "iPhone 13 Pro",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||||
|
expect(results[0].listingLink).toBe(
|
||||||
|
"https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should filter out invalid listings", () => {
|
||||||
|
const mockHtml = `
|
||||||
|
<html>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
${JSON.stringify({
|
||||||
|
props: {
|
||||||
|
pageProps: {
|
||||||
|
__APOLLO_STATE__: {
|
||||||
|
"Listing:123": {
|
||||||
|
url: "/v-iphone/k0l0",
|
||||||
|
title: "iPhone 13 Pro",
|
||||||
|
},
|
||||||
|
"Listing:456": {
|
||||||
|
url: "/v-samsung/k0l0",
|
||||||
|
// Missing title
|
||||||
|
},
|
||||||
|
"Other:789": {
|
||||||
|
url: "/v-other/k0l0",
|
||||||
|
title: "Other Item",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||||
|
expect(results).toHaveLength(1);
|
||||||
|
expect(results[0].name).toBe("iPhone 13 Pro");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return empty array for invalid HTML", () => {
|
||||||
|
const results = parseSearch(
|
||||||
|
"<html><body>Invalid</body></html>",
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
|
expect(results).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("parseDetailedListing", () => {
|
||||||
|
test("should parse detailed listing with all fields", async () => {
|
||||||
|
const mockHtml = `
|
||||||
|
<html>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
${JSON.stringify({
|
||||||
|
props: {
|
||||||
|
pageProps: {
|
||||||
|
__APOLLO_STATE__: {
|
||||||
|
"Listing:123": {
|
||||||
|
url: "/v-iphone-13-pro/k0l0",
|
||||||
|
title: "iPhone 13 Pro 256GB",
|
||||||
|
description: "Excellent condition iPhone 13 Pro",
|
||||||
|
price: {
|
||||||
|
amount: 80000,
|
||||||
|
currency: "CAD",
|
||||||
|
type: "FIXED",
|
||||||
|
},
|
||||||
|
type: "OFFER",
|
||||||
|
status: "ACTIVE",
|
||||||
|
activationDate: "2024-01-15T10:00:00.000Z",
|
||||||
|
endDate: "2025-01-15T10:00:00.000Z",
|
||||||
|
metrics: { views: 150 },
|
||||||
|
location: {
|
||||||
|
address: "Toronto, ON",
|
||||||
|
id: 1700273,
|
||||||
|
name: "Toronto",
|
||||||
|
coordinates: {
|
||||||
|
latitude: 43.6532,
|
||||||
|
longitude: -79.3832,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
imageUrls: [
|
||||||
|
"https://media.kijiji.ca/api/v1/image1.jpg",
|
||||||
|
"https://media.kijiji.ca/api/v1/image2.jpg",
|
||||||
|
],
|
||||||
|
imageCount: 2,
|
||||||
|
categoryId: 132,
|
||||||
|
adSource: "ORGANIC",
|
||||||
|
flags: {
|
||||||
|
topAd: false,
|
||||||
|
priceDrop: true,
|
||||||
|
},
|
||||||
|
posterInfo: {
|
||||||
|
posterId: "user123",
|
||||||
|
rating: 4.8,
|
||||||
|
},
|
||||||
|
attributes: [
|
||||||
|
{
|
||||||
|
canonicalName: "forsaleby",
|
||||||
|
canonicalValues: ["ownr"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
canonicalName: "phonecarrier",
|
||||||
|
canonicalValues: ["unlocked"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const result = await parseDetailedListing(
|
||||||
|
mockHtml,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
|
expect(result).toEqual({
|
||||||
|
url: "https://www.kijiji.ca/v-iphone-13-pro/k0l0",
|
||||||
|
title: "iPhone 13 Pro 256GB",
|
||||||
|
description: "Excellent condition iPhone 13 Pro",
|
||||||
|
listingPrice: {
|
||||||
|
amountFormatted: "$800.00",
|
||||||
|
cents: 80000,
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
listingType: "OFFER",
|
||||||
|
listingStatus: "ACTIVE",
|
||||||
|
creationDate: "2024-01-15T10:00:00.000Z",
|
||||||
|
endDate: "2025-01-15T10:00:00.000Z",
|
||||||
|
numberOfViews: 150,
|
||||||
|
address: "Toronto, ON",
|
||||||
|
images: [
|
||||||
|
"https://media.kijiji.ca/api/v1/image1.jpg",
|
||||||
|
"https://media.kijiji.ca/api/v1/image2.jpg",
|
||||||
|
],
|
||||||
|
categoryId: 132,
|
||||||
|
adSource: "ORGANIC",
|
||||||
|
flags: {
|
||||||
|
topAd: false,
|
||||||
|
priceDrop: true,
|
||||||
|
},
|
||||||
|
attributes: {
|
||||||
|
forsaleby: ["ownr"],
|
||||||
|
phonecarrier: ["unlocked"],
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
id: 1700273,
|
||||||
|
name: "Toronto",
|
||||||
|
coordinates: {
|
||||||
|
latitude: 43.6532,
|
||||||
|
longitude: -79.3832,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sellerInfo: {
|
||||||
|
posterId: "user123",
|
||||||
|
rating: 4.8,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should return null for contact-based pricing", async () => {
|
||||||
|
const mockHtml = `
|
||||||
|
<html>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
${JSON.stringify({
|
||||||
|
props: {
|
||||||
|
pageProps: {
|
||||||
|
__APOLLO_STATE__: {
|
||||||
|
"Listing:123": {
|
||||||
|
url: "/v-iphone/k0l0",
|
||||||
|
title: "iPhone for Sale",
|
||||||
|
price: {
|
||||||
|
type: "CONTACT",
|
||||||
|
amount: null,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const result = await parseDetailedListing(
|
||||||
|
mockHtml,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle missing optional fields", async () => {
|
||||||
|
const mockHtml = `
|
||||||
|
<html>
|
||||||
|
<script id="__NEXT_DATA__" type="application/json">
|
||||||
|
${JSON.stringify({
|
||||||
|
props: {
|
||||||
|
pageProps: {
|
||||||
|
__APOLLO_STATE__: {
|
||||||
|
"Listing:123": {
|
||||||
|
url: "/v-iphone/k0l0",
|
||||||
|
title: "iPhone 13",
|
||||||
|
price: { amount: 50000 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})}
|
||||||
|
</script>
|
||||||
|
</html>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const result = await parseDetailedListing(
|
||||||
|
mockHtml,
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
);
|
||||||
|
expect(result).toEqual({
|
||||||
|
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||||
|
title: "iPhone 13",
|
||||||
|
description: undefined,
|
||||||
|
listingPrice: {
|
||||||
|
amountFormatted: "$500.00",
|
||||||
|
cents: 50000,
|
||||||
|
currency: undefined,
|
||||||
|
},
|
||||||
|
listingType: undefined,
|
||||||
|
listingStatus: undefined,
|
||||||
|
creationDate: undefined,
|
||||||
|
endDate: undefined,
|
||||||
|
numberOfViews: undefined,
|
||||||
|
address: null,
|
||||||
|
images: [],
|
||||||
|
categoryId: 0,
|
||||||
|
adSource: "UNKNOWN",
|
||||||
|
flags: {
|
||||||
|
topAd: false,
|
||||||
|
priceDrop: false,
|
||||||
|
},
|
||||||
|
attributes: {},
|
||||||
|
location: {
|
||||||
|
id: 0,
|
||||||
|
name: "Unknown",
|
||||||
|
coordinates: undefined,
|
||||||
|
},
|
||||||
|
sellerInfo: undefined,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
54
packages/core/test/kijiji-utils.test.ts
Normal file
54
packages/core/test/kijiji-utils.test.ts
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
import { describe, expect, test } from "bun:test";
|
||||||
|
import { formatCentsToCurrency, slugify } from "../src/scrapers/kijiji";
|
||||||
|
|
||||||
|
describe("Utility Functions", () => {
|
||||||
|
describe("slugify", () => {
|
||||||
|
test("should convert basic strings to slugs", () => {
|
||||||
|
expect(slugify("Hello World")).toBe("hello-world");
|
||||||
|
expect(slugify("iPhone 13 Pro")).toBe("iphone-13-pro");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle special characters", () => {
|
||||||
|
expect(slugify("Café & Restaurant")).toBe("cafe-restaurant");
|
||||||
|
expect(slugify("100% New")).toBe("100-new");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle empty and edge cases", () => {
|
||||||
|
expect(slugify("")).toBe("");
|
||||||
|
expect(slugify(" ")).toBe("-");
|
||||||
|
expect(slugify("---")).toBe("-");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should preserve numbers and valid characters", () => {
|
||||||
|
expect(slugify("iPhone 13")).toBe("iphone-13");
|
||||||
|
expect(slugify("item123")).toBe("item123");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("formatCentsToCurrency", () => {
|
||||||
|
test("should format valid cent values", () => {
|
||||||
|
expect(formatCentsToCurrency(100)).toBe("$1.00");
|
||||||
|
expect(formatCentsToCurrency(1999)).toBe("$19.99");
|
||||||
|
expect(formatCentsToCurrency(0)).toBe("$0.00");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle string inputs", () => {
|
||||||
|
expect(formatCentsToCurrency("100")).toBe("$1.00");
|
||||||
|
expect(formatCentsToCurrency("1999")).toBe("$19.99");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle null/undefined inputs", () => {
|
||||||
|
expect(formatCentsToCurrency(null)).toBe("");
|
||||||
|
expect(formatCentsToCurrency(undefined)).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should handle invalid inputs", () => {
|
||||||
|
expect(formatCentsToCurrency("invalid")).toBe("");
|
||||||
|
expect(formatCentsToCurrency(Number.NaN)).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("should use en-US locale formatting", () => {
|
||||||
|
expect(formatCentsToCurrency(123456)).toBe("$1,234.56");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
11
packages/core/test/setup.ts
Normal file
11
packages/core/test/setup.ts
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
// Test setup for Bun test runner
|
||||||
|
// This file is loaded before any tests run due to bunfig.toml preload
|
||||||
|
|
||||||
|
// Mock fetch globally for tests
|
||||||
|
global.fetch =
|
||||||
|
global.fetch ||
|
||||||
|
(() => {
|
||||||
|
throw new Error("fetch is not available in test environment");
|
||||||
|
});
|
||||||
|
|
||||||
|
// Add any global test utilities here
|
||||||
13
packages/core/tsconfig.json
Normal file
13
packages/core/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
},
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true
|
||||||
|
}
|
||||||
|
}
|
||||||
21
packages/mcp-server/package.json
Normal file
21
packages/mcp-server/package.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/mcp-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"module": "./src/index.ts",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun ./src/index.ts",
|
||||||
|
"dev": "bun --watch ./src/index.ts",
|
||||||
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
36
packages/mcp-server/src/index.ts
Normal file
36
packages/mcp-server/src/index.ts
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
import { handleMcpRequest } from "./protocol/handler";
|
||||||
|
import { serverCard } from "./protocol/metadata";
|
||||||
|
|
||||||
|
const PORT = process.env.MCP_PORT || 4006;
|
||||||
|
|
||||||
|
const server = Bun.serve({
|
||||||
|
port: PORT as number | string,
|
||||||
|
idleTimeout: 255, // 255 seconds (max allowed)
|
||||||
|
routes: {
|
||||||
|
// MCP metadata discovery endpoint
|
||||||
|
"/.well-known/mcp/server-card.json": new Response(
|
||||||
|
JSON.stringify(serverCard),
|
||||||
|
{
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
},
|
||||||
|
),
|
||||||
|
|
||||||
|
// MCP JSON-RPC 2.0 protocol endpoint
|
||||||
|
"/mcp": async (req: Request) => {
|
||||||
|
if (req.method === "POST") {
|
||||||
|
return await handleMcpRequest(req);
|
||||||
|
}
|
||||||
|
return Response.json(
|
||||||
|
{ message: "MCP endpoint requires POST request" },
|
||||||
|
{ status: 405 },
|
||||||
|
);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
// Fallback for all other routes
|
||||||
|
fetch(_req: Request) {
|
||||||
|
return new Response("Not Found", { status: 404 });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`MCP Server running on ${server.hostname}:${server.port}`);
|
||||||
292
packages/mcp-server/src/protocol/handler.ts
Normal file
292
packages/mcp-server/src/protocol/handler.ts
Normal file
@@ -0,0 +1,292 @@
|
|||||||
|
import { tools } from "./tools";
|
||||||
|
|
||||||
|
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
|
||||||
|
const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; // 3 minutes default
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle MCP JSON-RPC 2.0 protocol requests
|
||||||
|
*/
|
||||||
|
export async function handleMcpRequest(req: Request): Promise<Response> {
|
||||||
|
try {
|
||||||
|
const body = await req.json();
|
||||||
|
|
||||||
|
// Validate JSON-RPC 2.0 format
|
||||||
|
if (!body.jsonrpc || body.jsonrpc !== "2.0" || !body.method) {
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
error: { code: -32600, message: "Invalid Request" },
|
||||||
|
id: body.id,
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { method, params, id } = body;
|
||||||
|
|
||||||
|
// Handle initialize method
|
||||||
|
if (method === "initialize") {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
result: {
|
||||||
|
protocolVersion: "2025-06-18",
|
||||||
|
capabilities: {
|
||||||
|
tools: {
|
||||||
|
listChanged: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
serverInfo: {
|
||||||
|
name: "marketplace-scrapers",
|
||||||
|
version: "1.0.0",
|
||||||
|
},
|
||||||
|
instructions:
|
||||||
|
"Use search_kijiji, search_facebook, or search_ebay tools to find listings across Canadian marketplaces",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tools/list method
|
||||||
|
if (method === "tools/list") {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
result: {
|
||||||
|
tools,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle notifications (messages without id field should not get a response)
|
||||||
|
if (!id) {
|
||||||
|
// Notifications don't require a response
|
||||||
|
if (method === "notifications/initialized") {
|
||||||
|
// Client initialized successfully, no response needed
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
}
|
||||||
|
if (method === "notifications/progress") {
|
||||||
|
// Progress notifications, no response needed
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
}
|
||||||
|
// Unknown notification - still no response for notifications
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tools/call method
|
||||||
|
if (method === "tools/call") {
|
||||||
|
const { name, arguments: args } = params || {};
|
||||||
|
|
||||||
|
if (!name || !args) {
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: {
|
||||||
|
code: -32602,
|
||||||
|
message: "Invalid params: name and arguments required",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route tool calls to appropriate handlers
|
||||||
|
try {
|
||||||
|
let result: unknown;
|
||||||
|
|
||||||
|
if (name === "search_kijiji") {
|
||||||
|
const query = args.query;
|
||||||
|
if (!query) {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const params = new URLSearchParams({ q: query });
|
||||||
|
if (args.location) params.append("location", args.location);
|
||||||
|
if (args.category) params.append("category", args.category);
|
||||||
|
if (args.keywords) params.append("keywords", args.keywords);
|
||||||
|
if (args.sortBy) params.append("sortBy", args.sortBy);
|
||||||
|
if (args.sortOrder) params.append("sortOrder", args.sortOrder);
|
||||||
|
if (args.maxPages)
|
||||||
|
params.append("maxPages", args.maxPages.toString());
|
||||||
|
if (args.priceMin)
|
||||||
|
params.append("priceMin", args.priceMin.toString());
|
||||||
|
if (args.priceMax)
|
||||||
|
params.append("priceMax", args.priceMax.toString());
|
||||||
|
if (args.cookies) params.append("cookies", args.cookies);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
|
||||||
|
);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(`${API_BASE_URL}/kijiji?${params.toString()}`),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() =>
|
||||||
|
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
console.error(
|
||||||
|
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
result = await response.json();
|
||||||
|
console.log(
|
||||||
|
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
|
);
|
||||||
|
} else if (name === "search_facebook") {
|
||||||
|
const query = args.query;
|
||||||
|
if (!query) {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const params = new URLSearchParams({ q: query });
|
||||||
|
if (args.location) params.append("location", args.location);
|
||||||
|
if (args.maxItems)
|
||||||
|
params.append("maxItems", args.maxItems.toString());
|
||||||
|
if (args.cookiesSource) params.append("cookies", args.cookiesSource);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
|
||||||
|
);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(`${API_BASE_URL}/facebook?${params.toString()}`),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() =>
|
||||||
|
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
console.error(
|
||||||
|
`[MCP] Facebook API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
result = await response.json();
|
||||||
|
console.log(
|
||||||
|
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
|
);
|
||||||
|
} else if (name === "search_ebay") {
|
||||||
|
const query = args.query;
|
||||||
|
if (!query) {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const params = new URLSearchParams({ q: query });
|
||||||
|
if (args.minPrice)
|
||||||
|
params.append("minPrice", args.minPrice.toString());
|
||||||
|
if (args.maxPrice)
|
||||||
|
params.append("maxPrice", args.maxPrice.toString());
|
||||||
|
if (args.strictMode !== undefined)
|
||||||
|
params.append("strictMode", args.strictMode.toString());
|
||||||
|
if (args.exclusions?.length)
|
||||||
|
params.append("exclusions", args.exclusions.join(","));
|
||||||
|
if (args.keywords?.length)
|
||||||
|
params.append("keywords", args.keywords.join(","));
|
||||||
|
if (args.buyItNowOnly !== undefined)
|
||||||
|
params.append("buyItNowOnly", args.buyItNowOnly.toString());
|
||||||
|
if (args.canadaOnly !== undefined)
|
||||||
|
params.append("canadaOnly", args.canadaOnly.toString());
|
||||||
|
if (args.maxItems)
|
||||||
|
params.append("maxItems", args.maxItems.toString());
|
||||||
|
if (args.cookies) params.append("cookies", args.cookies);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
|
||||||
|
);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(`${API_BASE_URL}/ebay?${params.toString()}`),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() =>
|
||||||
|
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
console.error(
|
||||||
|
`[MCP] eBay API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
result = await response.json();
|
||||||
|
console.log(
|
||||||
|
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32601, message: `Unknown tool: ${name}` },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
result: {
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: JSON.stringify(result, null, 2),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error";
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: {
|
||||||
|
code: -32603,
|
||||||
|
message: `Tool execution failed: ${errorMessage}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Method not found
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32601, message: `Method not found: ${method}` },
|
||||||
|
},
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error";
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
error: { code: -32700, message: `Parse error: ${errorMessage}` },
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
27
packages/mcp-server/src/protocol/metadata.ts
Normal file
27
packages/mcp-server/src/protocol/metadata.ts
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
/**
|
||||||
|
* MCP Server metadata for discovery
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const serverCard = {
|
||||||
|
$schema:
|
||||||
|
"https://static.modelcontextprotocol.io/schemas/mcp-server-card/v1.json",
|
||||||
|
version: "1.0",
|
||||||
|
protocolVersion: "2025-06-18",
|
||||||
|
serverInfo: {
|
||||||
|
name: "marketplace-scrapers",
|
||||||
|
title: "Marketplace Scrapers MCP Server",
|
||||||
|
version: "1.0.0",
|
||||||
|
},
|
||||||
|
transport: {
|
||||||
|
type: "streamable-http",
|
||||||
|
endpoint: "/mcp",
|
||||||
|
},
|
||||||
|
capabilities: {
|
||||||
|
tools: {
|
||||||
|
listChanged: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
description:
|
||||||
|
"Scrapes marketplace listings from Kijiji, Facebook Marketplace, and eBay",
|
||||||
|
tools: "dynamic",
|
||||||
|
};
|
||||||
150
packages/mcp-server/src/protocol/tools.ts
Normal file
150
packages/mcp-server/src/protocol/tools.ts
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
/**
|
||||||
|
* MCP tool definitions for marketplace scrapers
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const tools = [
|
||||||
|
{
|
||||||
|
name: "search_kijiji",
|
||||||
|
description: "Search Kijiji marketplace for listings matching a query",
|
||||||
|
inputSchema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
query: {
|
||||||
|
type: "string",
|
||||||
|
description: "Search query for Kijiji listings",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Location name or ID (e.g., 'toronto', 'gta', 'ontario')",
|
||||||
|
},
|
||||||
|
category: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Category name or ID (e.g., 'computers', 'furniture', 'bikes')",
|
||||||
|
},
|
||||||
|
keywords: {
|
||||||
|
type: "string",
|
||||||
|
description: "Additional keywords to filter results",
|
||||||
|
},
|
||||||
|
sortBy: {
|
||||||
|
type: "string",
|
||||||
|
description: "Sort results by field",
|
||||||
|
enum: ["relevancy", "date", "price", "distance"],
|
||||||
|
default: "relevancy",
|
||||||
|
},
|
||||||
|
sortOrder: {
|
||||||
|
type: "string",
|
||||||
|
description: "Sort order",
|
||||||
|
enum: ["asc", "desc"],
|
||||||
|
default: "desc",
|
||||||
|
},
|
||||||
|
maxPages: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum pages to fetch (~40 items per page)",
|
||||||
|
default: 5,
|
||||||
|
},
|
||||||
|
priceMin: {
|
||||||
|
type: "number",
|
||||||
|
description: "Minimum price in cents",
|
||||||
|
},
|
||||||
|
priceMax: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum price in cents",
|
||||||
|
},
|
||||||
|
cookies: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Optional: Kijiji session cookies to bypass bot detection (JSON array or 'name1=value1; name2=value2')",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "search_facebook",
|
||||||
|
description: "Search Facebook Marketplace for listings matching a query",
|
||||||
|
inputSchema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
query: {
|
||||||
|
type: "string",
|
||||||
|
description: "Search query for Facebook Marketplace listings",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
type: "string",
|
||||||
|
description: "Location for search (e.g., 'toronto')",
|
||||||
|
default: "toronto",
|
||||||
|
},
|
||||||
|
maxItems: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum number of items to return",
|
||||||
|
default: 5,
|
||||||
|
},
|
||||||
|
cookiesSource: {
|
||||||
|
type: "string",
|
||||||
|
description: "Optional Facebook session cookies source",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "search_ebay",
|
||||||
|
description:
|
||||||
|
"Search eBay for listings matching a query (default: Buy It Now only, Canada only)",
|
||||||
|
inputSchema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
query: {
|
||||||
|
type: "string",
|
||||||
|
description: "Search query for eBay listings",
|
||||||
|
},
|
||||||
|
minPrice: {
|
||||||
|
type: "number",
|
||||||
|
description: "Minimum price filter",
|
||||||
|
},
|
||||||
|
maxPrice: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum price filter",
|
||||||
|
},
|
||||||
|
strictMode: {
|
||||||
|
type: "boolean",
|
||||||
|
description: "Enable strict search mode",
|
||||||
|
default: false,
|
||||||
|
},
|
||||||
|
exclusions: {
|
||||||
|
type: "array",
|
||||||
|
items: { type: "string" },
|
||||||
|
description: "Terms to exclude from results",
|
||||||
|
},
|
||||||
|
keywords: {
|
||||||
|
type: "array",
|
||||||
|
items: { type: "string" },
|
||||||
|
description: "Keywords to include in search",
|
||||||
|
},
|
||||||
|
buyItNowOnly: {
|
||||||
|
type: "boolean",
|
||||||
|
description: "Include only Buy It Now listings (exclude auctions)",
|
||||||
|
default: true,
|
||||||
|
},
|
||||||
|
canadaOnly: {
|
||||||
|
type: "boolean",
|
||||||
|
description: "Include only Canadian sellers/listings",
|
||||||
|
default: true,
|
||||||
|
},
|
||||||
|
maxItems: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum number of items to return",
|
||||||
|
default: 5,
|
||||||
|
},
|
||||||
|
cookies: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Optional: eBay session cookies to bypass bot detection (format: 'name1=value1; name2=value2')",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
13
packages/mcp-server/tsconfig.json
Normal file
13
packages/mcp-server/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
},
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true
|
||||||
|
}
|
||||||
|
}
|
||||||
26
scripts/biome-symlink.sh
Executable file
26
scripts/biome-symlink.sh
Executable file
@@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# Get the path to the system biome executable
|
||||||
|
BIOME_PATH=$(which biome)
|
||||||
|
|
||||||
|
if [ -z "$BIOME_PATH" ]; then
|
||||||
|
echo "Error: biome executable not found in PATH"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Find all biome executables in node_modules
|
||||||
|
files=$(fd biome node_modules --type executable --no-ignore --follow)
|
||||||
|
|
||||||
|
if [ -z "$files" ]; then
|
||||||
|
echo "No biome executables found in node_modules"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Replace each with a symlink to the system biome
|
||||||
|
for file in $files; do
|
||||||
|
echo "Replacing $file with symlink to $BIOME_PATH"
|
||||||
|
rm "$file"
|
||||||
|
ln -s "$BIOME_PATH" "$file"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Done."
|
||||||
30
scripts/remove-eslint.sh
Executable file
30
scripts/remove-eslint.sh
Executable file
@@ -0,0 +1,30 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
PATTERN="eslint"
|
||||||
|
FILES="$(fd .)" # Or use 'find .' to search recursively
|
||||||
|
|
||||||
|
for file in $FILES; do
|
||||||
|
if [[ -f "$file" ]]; then
|
||||||
|
# 1. Use rg with line numbers (-n) and only the matched line (-o)
|
||||||
|
# 2. Use awk to print ONLY the line number (field 1)
|
||||||
|
# 3. Use xargs to pass multiple line numbers to a single sed command
|
||||||
|
|
||||||
|
LINE_NUMBERS=$(rg --line-number --no-filename "$PATTERN" "$file" | awk -F':' '{print $1}' | tr '\n' ',')
|
||||||
|
|
||||||
|
# Remove trailing comma if any
|
||||||
|
LINE_NUMBERS=${LINE_NUMBERS%,}
|
||||||
|
|
||||||
|
if [[ -n "$LINE_NUMBERS" ]]; then
|
||||||
|
echo "Deleting lines $LINE_NUMBERS from $file..."
|
||||||
|
|
||||||
|
# Use sed to delete the specified comma-separated line numbers in-place (-i)
|
||||||
|
# NOTE: The syntax for -i might vary slightly between GNU sed (Linux) and BSD sed (macOS).
|
||||||
|
sed -i.bak "${LINE_NUMBERS}d" "$file"
|
||||||
|
|
||||||
|
# Optional: Remove the backup file created by sed -i.bak
|
||||||
|
# rm "${file}.bak"
|
||||||
|
else
|
||||||
|
echo "$file: No lines matching pattern found."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
25
scripts/start.sh
Executable file
25
scripts/start.sh
Executable file
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Trap SIGTERM and SIGINT for graceful shutdown
|
||||||
|
trap 'echo "Received shutdown signal, stopping services..."; kill -TERM $API_PID $MCP_PID 2>/dev/null; wait' TERM INT
|
||||||
|
|
||||||
|
# Start API Server in background
|
||||||
|
echo "Starting API Server on port ${API_PORT:-4005}..."
|
||||||
|
bun dist/api/index.js &
|
||||||
|
API_PID=$!
|
||||||
|
|
||||||
|
# Give API server a moment to initialize
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Start MCP Server in background
|
||||||
|
echo "Starting MCP Server on port ${API_PORT:-4006}..."
|
||||||
|
bun dist/mcp/index.js &
|
||||||
|
MCP_PID=$!
|
||||||
|
|
||||||
|
echo "Both services started successfully"
|
||||||
|
echo "API Server PID: $API_PID"
|
||||||
|
echo "MCP Server PID: $MCP_PID"
|
||||||
|
|
||||||
|
# Wait for both processes
|
||||||
|
wait $API_PID $MCP_PID
|
||||||
54
src/index.ts
54
src/index.ts
@@ -1,54 +0,0 @@
|
|||||||
import fetchKijijiItems from "@/kijiji";
|
|
||||||
|
|
||||||
const PORT = process.env.PORT || 4005;
|
|
||||||
|
|
||||||
const server = Bun.serve({
|
|
||||||
port: PORT,
|
|
||||||
idleTimeout: 0,
|
|
||||||
routes: {
|
|
||||||
// Static routes
|
|
||||||
"/api/status": new Response("OK"),
|
|
||||||
|
|
||||||
// Dynamic routes
|
|
||||||
"/api/kijiji": async (req: Request) => {
|
|
||||||
const reqUrl = new URL(req.url);
|
|
||||||
|
|
||||||
const SEARCH_QUERY =
|
|
||||||
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
|
||||||
if (!SEARCH_QUERY)
|
|
||||||
return Response.json(
|
|
||||||
{
|
|
||||||
message:
|
|
||||||
"Request didn't have 'query' header or 'q' search parameter!",
|
|
||||||
},
|
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
|
|
||||||
const items = await fetchKijijiItems(SEARCH_QUERY, 5);
|
|
||||||
if (!items)
|
|
||||||
return Response.json(
|
|
||||||
{ message: "Search didn't return any results!" },
|
|
||||||
{ status: 404 },
|
|
||||||
);
|
|
||||||
return Response.json(items, { status: 200 });
|
|
||||||
},
|
|
||||||
|
|
||||||
// Wildcard route for all routes that start with "/api/" and aren't otherwise matched
|
|
||||||
"/api/*": Response.json({ message: "Not found" }, { status: 404 }),
|
|
||||||
|
|
||||||
// // Serve a file by buffering it in memory
|
|
||||||
// "/favicon.ico": new Response(await Bun.file("./favicon.ico").bytes(), {
|
|
||||||
// headers: {
|
|
||||||
// "Content-Type": "image/x-icon",
|
|
||||||
// },
|
|
||||||
// }),
|
|
||||||
},
|
|
||||||
|
|
||||||
// (optional) fallback for unmatched routes:
|
|
||||||
// Required if Bun's version < 1.2.3
|
|
||||||
fetch(req: Request) {
|
|
||||||
return new Response("Not Found", { status: 404 });
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Serving on ${server.hostname}:${server.port}`);
|
|
||||||
380
src/kijiji.ts
380
src/kijiji.ts
@@ -1,380 +0,0 @@
|
|||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
||||||
import { parseHTML } from "linkedom";
|
|
||||||
import unidecode from "unidecode";
|
|
||||||
|
|
||||||
// const unidecode = require("unidecode");
|
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
|
||||||
|
|
||||||
type HTMLString = string;
|
|
||||||
|
|
||||||
type SearchListing = {
|
|
||||||
name: string;
|
|
||||||
listingLink: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type ApolloRecord = Record<string, unknown>;
|
|
||||||
|
|
||||||
interface ApolloSearchItem {
|
|
||||||
url?: string;
|
|
||||||
title?: string;
|
|
||||||
[k: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ApolloListingRoot {
|
|
||||||
url?: string;
|
|
||||||
title?: string;
|
|
||||||
description?: string;
|
|
||||||
price?: { amount?: number | string; currency?: string };
|
|
||||||
type?: string;
|
|
||||||
status?: string;
|
|
||||||
activationDate?: string;
|
|
||||||
endDate?: string;
|
|
||||||
metrics?: { views?: number | string };
|
|
||||||
location?: { address?: string | null };
|
|
||||||
[k: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
type ListingDetails = {
|
|
||||||
url: string;
|
|
||||||
title: string;
|
|
||||||
description?: string;
|
|
||||||
listingPrice?: {
|
|
||||||
amountFormatted: string;
|
|
||||||
cents?: number;
|
|
||||||
currency?: string;
|
|
||||||
};
|
|
||||||
listingType?: string;
|
|
||||||
listingStatus?: string;
|
|
||||||
creationDate?: string;
|
|
||||||
endDate?: string;
|
|
||||||
numberOfViews?: number;
|
|
||||||
address?: string | null;
|
|
||||||
};
|
|
||||||
|
|
||||||
// ----------------------------- Utilities -----------------------------
|
|
||||||
|
|
||||||
const SEPS = new Set([" ", "–", "—", "/", ":", ";", ",", ".", "-"]);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Slugifies a string for search
|
|
||||||
*/
|
|
||||||
export function slugify(input: string): string {
|
|
||||||
const s = unidecode(input).toLowerCase();
|
|
||||||
const out: string[] = [];
|
|
||||||
let lastHyphen = false;
|
|
||||||
|
|
||||||
for (let i = 0; i < s.length; i++) {
|
|
||||||
const ch = s[i];
|
|
||||||
const code = ch!.charCodeAt(0);
|
|
||||||
|
|
||||||
// a-z or 0-9
|
|
||||||
if ((code >= 97 && code <= 122) || (code >= 48 && code <= 57)) {
|
|
||||||
out.push(ch!);
|
|
||||||
lastHyphen = false;
|
|
||||||
} else if (SEPS.has(ch!)) {
|
|
||||||
if (!lastHyphen) {
|
|
||||||
out.push("-");
|
|
||||||
lastHyphen = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// else drop character
|
|
||||||
}
|
|
||||||
return out.join("");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Turns cents to localized currency string.
|
|
||||||
*/
|
|
||||||
function formatCentsToCurrency(
|
|
||||||
num: number | string | undefined,
|
|
||||||
locale = "en-US",
|
|
||||||
): string {
|
|
||||||
if (num == null) return "";
|
|
||||||
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
|
|
||||||
if (Number.isNaN(cents)) return "";
|
|
||||||
const dollars = cents / 100;
|
|
||||||
const formatter = new Intl.NumberFormat(locale, {
|
|
||||||
minimumFractionDigits: 2,
|
|
||||||
maximumFractionDigits: 2,
|
|
||||||
useGrouping: true,
|
|
||||||
});
|
|
||||||
return formatter.format(dollars);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
||||||
return typeof value === "object" && value !== null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function delay(ms: number): Promise<void> {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
class HttpError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly status: number,
|
|
||||||
public readonly url: string,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "HttpError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- HTTP Client -----------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
Fetch HTML with a basic retry strategy and simple rate-limit delay between calls.
|
|
||||||
- Retries on 429 and 5xx
|
|
||||||
- Respects X-RateLimit-Reset when present (seconds)
|
|
||||||
*/
|
|
||||||
async function fetchHtml(
|
|
||||||
url: string,
|
|
||||||
DELAY_MS: number,
|
|
||||||
opts?: {
|
|
||||||
maxRetries?: number;
|
|
||||||
retryBaseMs?: number;
|
|
||||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
|
||||||
},
|
|
||||||
): Promise<HTMLString> {
|
|
||||||
const maxRetries = opts?.maxRetries ?? 3;
|
|
||||||
const retryBaseMs = opts?.retryBaseMs ?? 500;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
||||||
try {
|
|
||||||
console.log(`Fetching: `, url);
|
|
||||||
const res = await fetch(url, {
|
|
||||||
method: "GET",
|
|
||||||
headers: {
|
|
||||||
accept:
|
|
||||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
||||||
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
|
||||||
"cache-control": "no-cache",
|
|
||||||
"upgrade-insecure-requests": "1",
|
|
||||||
"user-agent":
|
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
|
||||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
|
||||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
// Respect 429 reset if provided
|
|
||||||
if (res.status === 429) {
|
|
||||||
const resetSeconds = rateLimitReset ? Number(rateLimitReset) : NaN;
|
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
|
||||||
? Math.max(0, resetSeconds * 1000)
|
|
||||||
: (attempt + 1) * retryBaseMs;
|
|
||||||
await delay(waitMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Retry on 5xx
|
|
||||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
|
||||||
await delay((attempt + 1) * retryBaseMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${res.status}`,
|
|
||||||
res.status,
|
|
||||||
url,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await res.text();
|
|
||||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
|
||||||
await delay(DELAY_MS);
|
|
||||||
return html;
|
|
||||||
} catch (err) {
|
|
||||||
if (attempt >= maxRetries) throw err;
|
|
||||||
await delay((attempt + 1) * retryBaseMs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error("Exhausted retries without response");
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- Parsing -----------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML.
|
|
||||||
*/
|
|
||||||
function extractApolloState(htmlString: HTMLString): ApolloRecord | null {
|
|
||||||
const { document } = parseHTML(htmlString);
|
|
||||||
const nextData = document.getElementById("__NEXT_DATA__");
|
|
||||||
if (!nextData || !nextData.textContent) return null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const jsonData = JSON.parse(nextData.textContent);
|
|
||||||
const apollo = jsonData?.props?.pageProps?.__APOLLO_STATE__;
|
|
||||||
return isRecord(apollo) ? apollo : null;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
Parse search page apollo state into SearchListing[].
|
|
||||||
Filters keys likely to be listing entities and ensures url/title exist.
|
|
||||||
*/
|
|
||||||
function parseSearch(
|
|
||||||
htmlString: HTMLString,
|
|
||||||
BASE_URL: string,
|
|
||||||
): SearchListing[] {
|
|
||||||
const apolloState = extractApolloState(htmlString);
|
|
||||||
if (!apolloState) return [];
|
|
||||||
|
|
||||||
const results: SearchListing[] = [];
|
|
||||||
for (const [key, value] of Object.entries(apolloState)) {
|
|
||||||
// Heuristic: Kijiji listing keys usually contain "Listing"
|
|
||||||
if (!key.includes("Listing")) continue;
|
|
||||||
if (!isRecord(value)) continue;
|
|
||||||
|
|
||||||
const item = value as ApolloSearchItem;
|
|
||||||
if (typeof item.url === "string" && typeof item.title === "string") {
|
|
||||||
results.push({
|
|
||||||
listingLink: item.url.startsWith("http")
|
|
||||||
? item.url
|
|
||||||
: `${BASE_URL}${item.url}`,
|
|
||||||
name: item.title,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
Parse a listing page into a typed object.
|
|
||||||
*/
|
|
||||||
function parseListing(
|
|
||||||
htmlString: HTMLString,
|
|
||||||
BASE_URL: string,
|
|
||||||
): ListingDetails | null {
|
|
||||||
const apolloState = extractApolloState(htmlString);
|
|
||||||
if (!apolloState) return null;
|
|
||||||
|
|
||||||
// Find the listing root key
|
|
||||||
const listingKey = Object.keys(apolloState).find((k) =>
|
|
||||||
k.includes("Listing"),
|
|
||||||
);
|
|
||||||
if (!listingKey) return null;
|
|
||||||
|
|
||||||
const root = apolloState[listingKey];
|
|
||||||
if (!isRecord(root)) return null;
|
|
||||||
|
|
||||||
const {
|
|
||||||
url,
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
price,
|
|
||||||
type,
|
|
||||||
status,
|
|
||||||
activationDate,
|
|
||||||
endDate,
|
|
||||||
metrics,
|
|
||||||
location,
|
|
||||||
} = root as ApolloListingRoot;
|
|
||||||
|
|
||||||
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
|
||||||
const amountFormatted = formatCentsToCurrency(cents);
|
|
||||||
|
|
||||||
const numberOfViews =
|
|
||||||
metrics?.views != null ? Number(metrics.views) : undefined;
|
|
||||||
|
|
||||||
const listingUrl =
|
|
||||||
typeof url === "string"
|
|
||||||
? url.startsWith("http")
|
|
||||||
? url
|
|
||||||
: `${BASE_URL}${url}`
|
|
||||||
: "";
|
|
||||||
|
|
||||||
if (!listingUrl || !title) return null;
|
|
||||||
|
|
||||||
return {
|
|
||||||
url: listingUrl,
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
listingPrice: amountFormatted
|
|
||||||
? {
|
|
||||||
amountFormatted,
|
|
||||||
cents: Number.isFinite(cents!) ? cents : undefined,
|
|
||||||
currency: price?.currency,
|
|
||||||
}
|
|
||||||
: undefined,
|
|
||||||
listingType: type,
|
|
||||||
listingStatus: status,
|
|
||||||
creationDate: activationDate,
|
|
||||||
endDate,
|
|
||||||
numberOfViews: Number.isFinite(numberOfViews!) ? numberOfViews : undefined,
|
|
||||||
address: location?.address ?? null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- Main -----------------------------
|
|
||||||
|
|
||||||
export default async function fetchKijijiItems(
|
|
||||||
SEARCH_QUERY: string,
|
|
||||||
REQUESTS_PER_SECOND = 1,
|
|
||||||
BASE_URL = "https://www.kijiji.ca",
|
|
||||||
) {
|
|
||||||
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
|
|
||||||
|
|
||||||
const searchUrl = `${BASE_URL}/b-gta-greater-toronto-area/${slugify(SEARCH_QUERY)}/k0l1700272?sort=relevancyDesc&view=list`;
|
|
||||||
|
|
||||||
console.log(`Fetching search: ${searchUrl}`);
|
|
||||||
const searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
|
||||||
onRateInfo: (remaining, reset) => {
|
|
||||||
if (remaining && reset) {
|
|
||||||
console.log(
|
|
||||||
`Search - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const searchResults = parseSearch(searchHtml, BASE_URL);
|
|
||||||
if (searchResults.length === 0) {
|
|
||||||
console.warn("No search results parsed from page.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deduplicate links
|
|
||||||
const listingLinks = Array.from(
|
|
||||||
new Set(searchResults.map((r) => r.listingLink)),
|
|
||||||
);
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
`Found ${listingLinks.length} listing links. Fetching details...`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const items: ListingDetails[] = [];
|
|
||||||
for (const link of listingLinks) {
|
|
||||||
try {
|
|
||||||
const html = await fetchHtml(link, DELAY_MS, {
|
|
||||||
onRateInfo: (remaining, reset) => {
|
|
||||||
if (remaining && reset) {
|
|
||||||
console.log(
|
|
||||||
`Item - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const parsed = parseListing(html, BASE_URL);
|
|
||||||
if (parsed) {
|
|
||||||
if (parsed.listingPrice?.cents) items.push(parsed);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof HttpError) {
|
|
||||||
console.error(`Failed to fetch ${link} - ${err.status} ${err.message}`);
|
|
||||||
} else {
|
|
||||||
console.error(
|
|
||||||
`Failed to fetch ${link} - ${String((err as Error)?.message || err)}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Parsed ${items.length} listings.`);
|
|
||||||
return items;
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
// Environment setup & latest features
|
|
||||||
"lib": ["dom"],
|
|
||||||
"target": "ESNext",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleDetection": "force",
|
|
||||||
"jsx": "react-jsx",
|
|
||||||
"allowJs": true,
|
|
||||||
|
|
||||||
// Bundler mode
|
|
||||||
"moduleResolution": "bundler",
|
|
||||||
"allowImportingTsExtensions": true,
|
|
||||||
"verbatimModuleSyntax": true,
|
|
||||||
"noEmit": true,
|
|
||||||
|
|
||||||
// Best practices
|
|
||||||
"strict": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"noFallthroughCasesInSwitch": true,
|
|
||||||
"noUncheckedIndexedAccess": true,
|
|
||||||
"noImplicitAny": true,
|
|
||||||
|
|
||||||
// Some stricter flags (disabled by default)
|
|
||||||
"noUnusedLocals": false,
|
|
||||||
"noUnusedParameters": false,
|
|
||||||
"noPropertyAccessFromIndexSignature": false,
|
|
||||||
|
|
||||||
"paths": {
|
|
||||||
"@/*": ["./src/*"]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
|
||||||
"exclude": ["node_modules"]
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user