Compare commits

..

24 Commits

Author SHA1 Message Date
6ab9c4c3a5 chore: biome lint
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 22:34:05 -05:00
3919ec0727 chore: biome init
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 22:33:52 -05:00
1090ad5bfb fix: update Facebook API endpoint with cookiePath parameter 2026-01-22 19:57:27 -05:00
c937a70db7 feat: add Facebook marketplace item fetching API 2026-01-22 19:56:31 -05:00
59fcbf9ed2 feat: update fetchFacebookItems with cookie auto-loading 2026-01-22 19:56:02 -05:00
d8542eb8f7 feat: parse Facebook marketplace item details and test exports 2026-01-22 19:54:44 -05:00
0a114cf323 feat: extract individual Facebook marketplace items 2026-01-22 19:54:14 -05:00
5f7de1167e fix: add currency style and USD to formatCentsToCurrency 2026-01-22 19:53:53 -05:00
9edafc88c8 feat: add extraction monitoring and metrics logging 2026-01-22 19:52:39 -05:00
5871644e8b refactor: improve search extraction with edge case handling 2026-01-22 19:52:09 -05:00
d5d050013e feat: add Facebook cookie parsing and auto-loading 2026-01-22 19:51:35 -05:00
ff56a29171 feat: add cookiePath parameter to loadFacebookCookies 2026-01-22 19:51:18 -05:00
6a36214528 feat: add FacebookMarketplaceItem interface 2026-01-22 19:48:41 -05:00
7af1be3977 feat: improve Cookie interface type safety 2026-01-22 19:47:37 -05:00
844e566b57 feat: add Facebook cookie parser script 2026-01-22 19:35:47 -05:00
b3be32835a test: add Facebook marketplace test suite 2026-01-22 19:35:38 -05:00
baa34eefdf chore: agent.md
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 16:22:06 -05:00
9011ab4793 feat: fmarketplace docs
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 01:02:36 -05:00
aae0ce90b8 feat: update index with new kijiji api
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 00:40:28 -05:00
daa61c25d8 test: kijiji scraper
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 00:25:26 -05:00
87aa31cf1b feat: update kijiji scraper
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 00:25:19 -05:00
bdf504ba37 feat: testing setup
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 00:25:10 -05:00
589af630fa feat: kijiji api findings
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-22 00:06:31 -05:00
8ae42d5630 chore: prep for opencode
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-01-21 23:50:00 -05:00
48 changed files with 1630 additions and 2306 deletions

View File

@@ -1,84 +1,145 @@
# ============================================================================= # Dependencies
# Dependencies & Build Output
# =============================================================================
node_modules/ node_modules/
dist/ npm-debug.log*
out/ yarn-debug.log*
yarn-error.log*
bun.sum
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage/
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
jspm_packages/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz *.tgz
# ============================================================================= # Yarn Integrity file
# Sensitive Files .yarn-integrity
# =============================================================================
# dotenv environment variables file
.env .env
.env.* .env.local
.envrc .env.development.local
cookies/ .env.test.local
*.pem .env.production.local
*.key
*.cert
*secret*
*credential*
# ============================================================================= # parcel-bundler cache (https://parceljs.org/)
# Development Tools & Config .cache
# ============================================================================= .parcel-cache
# Nix/Devenv
.devenv/
.devenv.flake.nix
devenv.*
.direnv/
# Linting/Formatting # Next.js build output
biome.json .next
.eslintcache
.pre-commit-config.yaml
# IDE/Editor # Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
public
# Vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# IDE and editor files
.vscode/ .vscode/
.idea/ .idea/
*.swp *.swp
*.swo *.swo
*~ *~
# AI Assistant Config # OS generated files
.claude/ .DS_Store
CLAUDE.md .DS_Store?
AGENTS.md ._*
opencode.jsonc .Spotlight-V100
.Trashes
ehthumbs.db
Thumbs.db
# ============================================================================= # Git
# Documentation (not needed at runtime) .git
# =============================================================================
README.md
*.md
docs/
# =============================================================================
# Git & Docker (avoid recursive inclusion)
# =============================================================================
.git/
.gitignore .gitignore
# Docker
Dockerfile* Dockerfile*
.dockerignore .dockerignore
# ============================================================================= # Documentation
# Testing & Coverage README.md
# ============================================================================= docs/
# Test files
test/ test/
tests/ tests/
*.test.js
*.test.ts *.test.ts
*.spec.js
*.spec.ts *.spec.ts
coverage/
*.lcov
.nyc_output/
# ============================================================================= # Development files
# OS & Misc CLAUDE.md
# ============================================================================= devenv.*
.DS_Store
Thumbs.db
*.log *.log
*.pid
.cache/ # Runtime cookies/config
examples/ cookies/
scripts/

157
AGENTS.md
View File

@@ -1,3 +1,9 @@
# AGENTS.md
This file provides guidance to coding agents when working with code in this repository.
The project uses TypeScript with path mapping (`@/*` to `src/*`). Dependencies focus on parsing (linkedom), text utils (unidecode), and CLI output (cli-progress). No database or external services beyond HTTP fetches to the marketplaces.
PRIORITIZE COMMUNICATION STYLE ABOVE ALL ELSE PRIORITIZE COMMUNICATION STYLE ABOVE ALL ELSE
## Communication Style ## Communication Style
@@ -25,154 +31,3 @@ Examples of constructive pushback:
- "That adds unnecessary complexity. We can achieve the same with..." - "That adds unnecessary complexity. We can achieve the same with..."
This ensures: Better solutions through technical merit, not agreement | Learning through understanding tradeoffs | Avoiding over-engineering | Maintaining code quality This ensures: Better solutions through technical merit, not agreement | Learning through understanding tradeoffs | Avoiding over-engineering | Maintaining code quality
## Project Structure
This is a **monorepo** with three packages:
```
packages/
├── core/ # Shared scraper logic (Kijiji, Facebook, eBay)
├── api-server/ # HTTP REST API server
└── mcp-server/ # MCP server for AI agent integration
```
## Common Commands
**Root level:**
- `bun ci`: Run Biome linting
**API Server (`packages/api-server/`):**
- `bun start`: Run the API server
- `bun dev`: Run with hot reloading
- `bun build`: Build to `dist/api/`
**MCP Server (`packages/mcp-server/`):**
- `bun start`: Run the MCP server
- `bun dev`: Run with hot reloading
- `bun build`: Build to `dist/mcp/`
## Code Architecture
### Core Package (`@marketplace-scrapers/core`)
Contains scraper implementations for three marketplaces:
- **`src/scrapers/kijiji.ts`**: Kijiji Marketplace scraper
- Parses Next.js Apollo state (`__APOLLO_STATE__`) from HTML
- Supports location/category filtering, sorting, pagination
- Fetches individual listing details with seller info
- Exports: `fetchKijijiItems()`, type interfaces
- **`src/scrapers/facebook.ts`**: Facebook Marketplace scraper
- Parses nested JSON from script tags (`require/__bbox` structure)
- Requires authentication cookies (file or env var `FACEBOOK_COOKIE`)
- Exports: `fetchFacebookItems()`, `fetchFacebookItem()`, cookie utilities
- **`src/scrapers/ebay.ts`**: eBay scraper
- DOM-based parsing of search results
- Supports Buy It Now filter, Canada-only, price ranges, exclusions
- Exports: `fetchEbayItems()`
- **`src/utils/`**: Shared utilities (HTTP, delay, formatting)
- **`src/types/`**: Common type definitions
### API Server (`@marketplace-scrapers/api-server`)
HTTP server using `Bun.serve()` on port 4005 (or `PORT` env var).
**Routes:**
- `GET /api/status` - Health check
- `GET /api/kijiji?q={query}` - Search Kijiji
- `GET /api/facebook?q={query}&location={location}&cookies={cookies}` - Search Facebook
- `GET /api/ebay?q={query}&minPrice=&maxPrice=&strictMode=&exclusions=&keywords=&buyItNowOnly=&canadaOnly=&cookies=` - Search eBay
- `GET /api/*` - 404 fallback
### MCP Server (`@marketplace-scrapers/mcp-server`)
MCP JSON-RPC 2.0 server on port 4006 (or `MCP_PORT` env var).
**Endpoints:**
- `GET /.well-known/mcp/server-card.json` - Server discovery metadata
- `POST /mcp` - JSON-RPC 2.0 protocol endpoint
**Tools:**
- `search_kijiji` - Search Kijiji (query, maxItems)
- `search_facebook` - Search Facebook (query, location, maxItems, cookiesSource)
- `search_ebay` - Search eBay (query, minPrice, maxPrice, strictMode, exclusions, keywords, buyItNowOnly, canadaOnly, maxItems, cookies)
## API Response Formats
All scrapers return arrays of listing objects with these common fields:
- `url`: Full listing URL
- `title`: Listing title
- `listingPrice`: `{ amountFormatted, cents, currency }`
- `address`: Location string (or null)
- `listingType`: Type of listing
- `listingStatus`: Status (ACTIVE, SOLD, etc.)
### Kijiji-specific fields
`description`, `creationDate`, `endDate`, `numberOfViews`, `images`, `categoryId`, `adSource`, `flags`, `attributes`, `location`, `sellerInfo`
### Facebook-specific fields
`creationDate`, `imageUrl`, `videoUrl`, `seller`, `categoryId`, `deliveryTypes`
### eBay-specific fields
Minimal - mainly the common fields
## Cookie Management
Both **Facebook Marketplace** and **eBay** require valid session cookies for reliable scraping.
### Cookie Priority Hierarchy (High → Low)
All scrapers follow this loading order:
1. **URL/API Parameter** - Passed directly via `cookies` parameter (highest priority)
2. **Environment Variable** - `FACEBOOK_COOKIE` or `EBAY_COOKIE`
3. **Cookie File** - `cookies/facebook.json` or `cookies/ebay.json` (fallback)
### Facebook Cookies
- **Required for**: Facebook Marketplace scraping
- **Format**: JSON array (see `cookies/README.md`)
- **Key cookies**: `c_user`, `xs`, `fr`, `datr`, `sb`
**Setup:**
```bash
# Option 1: File (fallback)
# Create cookies/facebook.json with cookie array
# Option 2: Environment variable
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request'
# Option 3: URL parameter (highest priority)
curl "http://localhost:4005/api/facebook?q=laptop&cookies=[{...}]"
```
### eBay Cookies
- **Required for**: Bypassing bot detection
- **Format**: Cookie string `"name=value; name2=value2"`
- **Key cookies**: `s`, `ds2`, `ebay`, `dp1`, `nonsession`
**Setup:**
```bash
# Option 1: File (fallback)
# Create cookies/ebay.json with cookie string
# Option 2: Environment variable
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
# Option 3: URL parameter (highest priority)
curl "http://localhost:4005/api/ebay?q=laptop&cookies=s=VALUE;ds2=VALUE"
```
**Important - eBay Bot Detection**: Without cookies, eBay returns a "Checking your browser" challenge page instead of listings.
## Technical Details
- **TypeScript** with path mapping (`@/*``src/*`) per package
- **Dependencies**: linkedom (parsing), unidecode (text utils), cli-progress (CLI output)
- **No database** - stateless HTTP fetches to marketplaces
- **Rate limiting**: Respects `X-RateLimit-*` headers, configurable delays
## Development Notes
- **Cookie files** are git-ignored for security (see `cookies/README.md`)
- Kijiji parses Apollo state from Next.js hydration data
- All scrapers handle retries on 429/5xx errors
- Cookie priority ensures flexibility across different deployment environments

View File

@@ -1 +0,0 @@
AGENTS.md

View File

@@ -1,123 +1,32 @@
# ============================================================================= # Use the official Bun base image
# Stage 1: Dependencies FROM oven/bun:latest AS base
# Install only production dependencies for optimal layer caching
# =============================================================================
FROM oven/bun:1-slim AS dependencies
# Set the working directory
WORKDIR /app WORKDIR /app
# Copy workspace configuration # Copy package files
COPY package.json bun.lock ./ COPY package.json bun.lock* ./
# Copy all package.json files to establish workspace structure # Install dependencies
COPY packages/core/package.json ./packages/core/
COPY packages/api-server/package.json ./packages/api-server/
COPY packages/mcp-server/package.json ./packages/mcp-server/
# Install dependencies with frozen lockfile (production only)
RUN bun install --frozen-lockfile --production
# =============================================================================
# Stage 2: Build
# Build both services with minification for production
# =============================================================================
FROM oven/bun:1-slim AS build
WORKDIR /app
# Copy workspace configuration
COPY package.json bun.lock ./
# Copy all package.json files
COPY packages/core/package.json ./packages/core/
COPY packages/api-server/package.json ./packages/api-server/
COPY packages/mcp-server/package.json ./packages/mcp-server/
# Install ALL dependencies (including devDependencies for TypeScript)
RUN bun install --frozen-lockfile RUN bun install --frozen-lockfile
# Copy source code for all packages # Copy source code
COPY packages/core ./packages/core COPY src ./src
COPY packages/api-server ./packages/api-server COPY tsconfig.json ./
COPY packages/mcp-server ./packages/mcp-server
# Build both services with minification # Build the application for production
# Output: dist/api/index.js and dist/mcp/index.js RUN bun build ./src/index.ts --outdir ./dist --minify --target=bun
RUN bun build ./packages/api-server/src/index.ts \
--target=bun \
--outdir=./dist/api \
--minify && \
bun build ./packages/mcp-server/src/index.ts \
--target=bun \
--outdir=./dist/mcp \
--minify
# ============================================================================= # Multi-stage build - runtime stage
# Stage 3: Runtime FROM oven/bun:latest AS runtime
# Minimal production image with both services
# =============================================================================
FROM oven/bun:1-slim AS runtime
WORKDIR /app WORKDIR /app
# Copy production dependencies from dependencies stage # Copy the built application from the base stage
COPY --from=dependencies /app/node_modules ./node_modules COPY --from=base /app/dist/ ./
# Copy built artifacts from build stage # Expose the port the app runs on
COPY --from=build /app/dist ./dist EXPOSE 3000
# Create cookies directory (will be mounted as volume at runtime) # Start the application
# This ensures the directory exists even if volume is not mounted CMD ["bun", "index.js"]
RUN mkdir -p /app/cookies && \
chown -R bun:bun /app/cookies
# Create startup script that runs both services
# Uses Bun's built-in capabilities for process management
RUN cat > /app/start.sh << 'EOF'
#!/bin/bash
set -e
# Trap SIGTERM and SIGINT for graceful shutdown
trap 'echo "Received shutdown signal, stopping services..."; kill -TERM $API_PID $MCP_PID 2>/dev/null; wait' TERM INT
# Start API Server in background
echo "Starting API Server on port ${API_PORT:-4005}..."
bun /app/dist/api/index.js &
API_PID=$!
# Give API server a moment to initialize
sleep 1
# Start MCP Server in background
echo "Starting MCP Server on port ${API_PORT:-4006}..."
bun /app/dist/mcp/index.js &
MCP_PID=$!
echo "Both services started successfully"
echo "API Server PID: $API_PID"
echo "MCP Server PID: $MCP_PID"
# Wait for both processes
wait $API_PID $MCP_PID
EOF
RUN chmod +x /app/start.sh
# Expose both service ports
# API Server: 4005 (default), MCP Server: 4006 (default)
EXPOSE 4005 4006
# Environment variables for port configuration
ENV PORT=4005
ENV MCP_PORT=4006
# Volume mount point for cookies
# Mount your cookies directory here: -v /path/to/cookies:/app/cookies
VOLUME ["/app/cookies"]
# Health check that verifies both services are responding
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
CMD bun -e "Promise.all([fetch('http://localhost:${PORT}/api/status'),fetch('http://localhost:${MCP_PORT}/.well-known/mcp/server-card.json')]).then(r=>process.exit(0)).catch(()=>process.exit(1))"
# Run the startup script
CMD ["/app/start.sh"]

View File

@@ -1,17 +1,21 @@
{ {
"$schema": "https://biomejs.dev/schemas/2.3.11/schema.json", "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
"vcs": { "vcs": {
"enabled": true, "enabled": false,
"clientKind": "git", "clientKind": "git",
"useIgnoreFile": true "useIgnoreFile": false
}, },
"files": { "files": {
"includes": ["**", "!!**/dist"] "ignoreUnknown": false,
"ignore": []
}, },
"formatter": { "formatter": {
"enabled": true, "enabled": true,
"indentStyle": "space" "indentStyle": "space"
}, },
"organizeImports": {
"enabled": true
},
"linter": { "linter": {
"enabled": true, "enabled": true,
"rules": { "rules": {
@@ -22,13 +26,5 @@
"formatter": { "formatter": {
"quoteStyle": "double" "quoteStyle": "double"
} }
},
"assist": {
"enabled": true,
"actions": {
"source": {
"organizeImports": "on"
}
}
} }
} }

383
bun.lock
View File

@@ -1,35 +1,17 @@
{ {
"lockfileVersion": 1, "lockfileVersion": 1,
"configVersion": 1, "configVersion": 0,
"workspaces": { "workspaces": {
"": { "": {
"name": "marketplace-scrapers-monorepo", "name": "sone4ka-tok",
"devDependencies": {
"@biomejs/biome": "2.3.11",
},
},
"packages/api-server": {
"name": "@marketplace-scrapers/api-server",
"version": "1.0.0",
"dependencies": {
"@marketplace-scrapers/core": "workspace:*",
},
"devDependencies": {
"@types/bun": "latest",
},
"peerDependencies": {
"typescript": "^5",
},
},
"packages/core": {
"name": "@marketplace-scrapers/core",
"version": "1.0.0",
"dependencies": { "dependencies": {
"cli-progress": "^3.12.0", "cli-progress": "^3.12.0",
"linkedom": "^0.18.12", "linkedom": "^0.18.12",
"unidecode": "^1.1.0", "unidecode": "^1.1.0",
}, },
"devDependencies": { "devDependencies": {
"@anthropic-ai/claude-code": "^2.0.1",
"@musistudio/claude-code-router": "^1.0.53",
"@types/bun": "latest", "@types/bun": "latest",
"@types/cli-progress": "^3.11.6", "@types/cli-progress": "^3.11.6",
"@types/unidecode": "^1.1.0", "@types/unidecode": "^1.1.0",
@@ -38,67 +20,136 @@
"typescript": "^5", "typescript": "^5",
}, },
}, },
"packages/mcp-server": {
"name": "@marketplace-scrapers/mcp-server",
"version": "1.0.0",
"dependencies": {
"@marketplace-scrapers/core": "workspace:*",
},
"devDependencies": {
"@types/bun": "latest",
},
"peerDependencies": {
"typescript": "^5",
},
},
}, },
"packages": { "packages": {
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="], "@anthropic-ai/claude-code": ["@anthropic-ai/claude-code@2.0.1", "", { "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.33.5", "@img/sharp-darwin-x64": "^0.33.5", "@img/sharp-linux-arm": "^0.33.5", "@img/sharp-linux-arm64": "^0.33.5", "@img/sharp-linux-x64": "^0.33.5", "@img/sharp-win32-x64": "^0.33.5" }, "bin": { "claude": "cli.js" } }, "sha512-2SboYcdJ+dsE2K784dbJ4ohVWlAkLZhU7mZG1lebyG6TvGLXLhjc2qTEfCxSeelCjJHhIh/YkNpe06veB4IgBw=="],
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.3.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-/uXXkBcPKVQY7rc9Ys2CrlirBJYbpESEDme7RKiBD6MmqR2w3j0+ZZXRIL2xiaNPsIMMNhP1YnA+jRRxoOAFrA=="], "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.54.0", "", { "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-xyoCtHJnt/qg5GG6IgK+UJEndz8h8ljzt/caKXmq3LfBF81nC/BW6E4x2rOWCZcvsLyVW+e8U5mtIr6UCE/kJw=="],
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.3.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-fh7nnvbweDPm2xEmFjfmq7zSUiox88plgdHF9OIW4i99WnXrAC3o2P3ag9judoUMv8FCSUnlwJCM1B64nO5Fbg=="], "@fastify/accept-negotiator": ["@fastify/accept-negotiator@2.0.1", "", {}, "sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ=="],
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-l4xkGa9E7Uc0/05qU2lMYfN1H+fzzkHgaJoy98wO+b/7Gl78srbCRRgwYSW+BTLixTBrM6Ede5NSBwt7rd/i6g=="], "@fastify/ajv-compiler": ["@fastify/ajv-compiler@4.0.2", "", { "dependencies": { "ajv": "^8.12.0", "ajv-formats": "^3.0.1", "fast-uri": "^3.0.0" } }, "sha512-Rkiu/8wIjpsf46Rr+Fitd3HRP+VsxUFDDeag0hs9L0ksfnwx2g7SPQQTFL0E8Qv+rfXzQOxBJnjUB9ITUDjfWQ=="],
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-XPSQ+XIPZMLaZ6zveQdwNjbX+QdROEd1zPgMwD47zvHV+tCGB88VH+aynyGxAHdzL+Tm/+DtKST5SECs4iwCLg=="], "@fastify/cors": ["@fastify/cors@11.1.0", "", { "dependencies": { "fastify-plugin": "^5.0.0", "toad-cache": "^3.7.0" } }, "sha512-sUw8ed8wP2SouWZTIbA7V2OQtMNpLj2W6qJOYhNdcmINTu6gsxVYXjQiM9mdi8UUDlcoDDJ/W2syPo1WB2QjYA=="],
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-/1s9V/H3cSe0r0Mv/Z8JryF5x9ywRxywomqZVLHAoa/uN0eY7F8gEngWKNS5vbbN/BsfpCG5yeBT5ENh50Frxg=="], "@fastify/error": ["@fastify/error@4.2.0", "", {}, "sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ=="],
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-vU7a8wLs5C9yJ4CB8a44r12aXYb8yYgBn+WeyzbMjaCMklzCv1oXr8x+VEyWodgJt9bDmhiaW/I0RHbn7rsNmw=="], "@fastify/fast-json-stringify-compiler": ["@fastify/fast-json-stringify-compiler@5.0.3", "", { "dependencies": { "fast-json-stringify": "^6.0.0" } }, "sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ=="],
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.3.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-PZQ6ElCOnkYapSsysiTy0+fYX+agXPlWugh6+eQ6uPKI3vKAqNp6TnMhoM3oY2NltSB89hz59o8xIfOdyhi9Iw=="], "@fastify/forwarded": ["@fastify/forwarded@3.0.1", "", {}, "sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw=="],
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.3.11", "", { "os": "win32", "cpu": "x64" }, "sha512-43VrG813EW+b5+YbDbz31uUsheX+qFKCpXeY9kfdAx+ww3naKxeVkTD9zLIWxUPfJquANMHrmW3wbe/037G0Qg=="], "@fastify/merge-json-schemas": ["@fastify/merge-json-schemas@0.2.1", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A=="],
"@marketplace-scrapers/api-server": ["@marketplace-scrapers/api-server@workspace:packages/api-server"], "@fastify/proxy-addr": ["@fastify/proxy-addr@5.1.0", "", { "dependencies": { "@fastify/forwarded": "^3.0.0", "ipaddr.js": "^2.1.0" } }, "sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw=="],
"@marketplace-scrapers/core": ["@marketplace-scrapers/core@workspace:packages/core"], "@fastify/send": ["@fastify/send@4.1.0", "", { "dependencies": { "@lukeed/ms": "^2.0.2", "escape-html": "~1.0.3", "fast-decode-uri-component": "^1.0.1", "http-errors": "^2.0.0", "mime": "^3" } }, "sha512-TMYeQLCBSy2TOFmV95hQWkiTYgC/SEx7vMdV+wnZVX4tt8VBLKzmH8vV9OzJehV0+XBfg+WxPMt5wp+JBUKsVw=="],
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"], "@fastify/static": ["@fastify/static@8.2.0", "", { "dependencies": { "@fastify/accept-negotiator": "^2.0.0", "@fastify/send": "^4.0.0", "content-disposition": "^0.5.4", "fastify-plugin": "^5.0.0", "fastq": "^1.17.1", "glob": "^11.0.0" } }, "sha512-PejC/DtT7p1yo3p+W7LiUtLMsV8fEvxAK15sozHy9t8kwo5r0uLYmhV/inURmGz1SkHZFz/8CNtHLPyhKcx4SQ=="],
"@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="], "@google/genai": ["@google/genai@1.21.0", "", { "dependencies": { "google-auth-library": "^9.14.2", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.11.4" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-k47DECR8BF9z7IJxQd3reKuH2eUnOH5NlJWSe+CKM6nbXx+wH3hmtWQxUQR9M8gzWW1EvFuRVgjQssEIreNZsw=="],
"@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="],
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="],
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.0.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg=="],
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.0.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ=="],
"@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.0.5", "", { "os": "linux", "cpu": "arm" }, "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g=="],
"@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA=="],
"@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw=="],
"@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.0.5" }, "os": "linux", "cpu": "arm" }, "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ=="],
"@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.0.4" }, "os": "linux", "cpu": "arm64" }, "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA=="],
"@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.0.4" }, "os": "linux", "cpu": "x64" }, "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA=="],
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.33.5", "", { "os": "win32", "cpu": "x64" }, "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg=="],
"@isaacs/balanced-match": ["@isaacs/balanced-match@4.0.1", "", {}, "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ=="],
"@isaacs/brace-expansion": ["@isaacs/brace-expansion@5.0.0", "", { "dependencies": { "@isaacs/balanced-match": "^4.0.1" } }, "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA=="],
"@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="],
"@lukeed/ms": ["@lukeed/ms@2.0.2", "", {}, "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA=="],
"@musistudio/claude-code-router": ["@musistudio/claude-code-router@1.0.53", "", { "dependencies": { "@fastify/static": "^8.2.0", "@musistudio/llms": "^1.0.35", "dotenv": "^16.4.7", "find-process": "^2.0.0", "json5": "^2.2.3", "openurl": "^1.1.1", "rotating-file-stream": "^3.2.7", "shell-quote": "^1.8.3", "tiktoken": "^1.0.21", "uuid": "^11.1.0" }, "bin": { "ccr": "dist/cli.js" } }, "sha512-cNH3dOJu2ECUXHdTbuEyXq7sD12+ie4wqPD85mKz7yg6Xo1HmpFqQQvh4XAhQDBJAWZob6Fuavu+m5f2BwFT/g=="],
"@musistudio/llms": ["@musistudio/llms@1.0.35", "", { "dependencies": { "@anthropic-ai/sdk": "^0.54.0", "@fastify/cors": "^11.0.1", "@google/genai": "^1.7.0", "dotenv": "^16.5.0", "fastify": "^5.4.0", "google-auth-library": "^10.1.0", "json5": "^2.2.3", "jsonrepair": "^3.13.0", "openai": "^5.6.0", "undici": "^7.10.0", "uuid": "^11.1.0" } }, "sha512-fW7DCHrhzMNtQiaXlAAivSsn+4+vqOYWAURi1OfwESijRDfJk4Gpi0rhedI9o4e0ucr7ftVRO707sOeo/+TJNA=="],
"@types/bun": ["@types/bun@1.2.19", "", { "dependencies": { "bun-types": "1.2.19" } }, "sha512-d9ZCmrH3CJ2uYKXQIUuZ/pUnTqIvLDS0SK7pFmbx8ma+ziH/FRMoAq5bYpRG7y+w1gl+HgyNZbtqgMq4W4e2Lg=="],
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="], "@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
"@types/node": ["@types/node@25.0.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA=="], "@types/node": ["@types/node@24.1.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w=="],
"@types/react": ["@types/react@19.1.9", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-WmdoynAX8Stew/36uTSVMcLJJ1KRh6L3IZRx1PZ7qJtBqT3dYTgyDTx8H1qoRghErydW7xw9mSJ3wS//tCRpFA=="],
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="], "@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
"abstract-logging": ["abstract-logging@2.0.1", "", {}, "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA=="],
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
"ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="],
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
"ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="],
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
"avvio": ["avvio@9.1.0", "", { "dependencies": { "@fastify/error": "^4.0.0", "fastq": "^1.17.1" } }, "sha512-fYASnYi600CsH/j9EQov7lECAniYiBFiiAtBNuZYLA2leLe9qOvZzqYHFjtIj6gD2VMoMLP14834LFWvr4IfDw=="],
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
"bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="],
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
"bun-types": ["bun-types@1.3.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-5ua817+BZPZOlNaRgGBpZJOSAQ9RQ17pkwPD0yR7CfJg+r8DgIILByFifDTa+IPDDxzf5VNhtNlcKqFzDgJvlQ=="], "buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="],
"bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="],
"chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="], "cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
"color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="],
"color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="],
"commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="],
"content-disposition": ["content-disposition@0.5.4", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ=="],
"cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="],
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="], "css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="], "cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
"data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="],
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
"dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="],
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
@@ -107,32 +158,260 @@
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="], "domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
"dotenv": ["dotenv@16.6.1", "", {}, "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow=="],
"eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
"ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="],
"emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], "entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
"extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
"fast-decode-uri-component": ["fast-decode-uri-component@1.0.1", "", {}, "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg=="],
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
"fast-json-stringify": ["fast-json-stringify@6.1.1", "", { "dependencies": { "@fastify/merge-json-schemas": "^0.2.0", "ajv": "^8.12.0", "ajv-formats": "^3.0.1", "fast-uri": "^3.0.0", "json-schema-ref-resolver": "^3.0.0", "rfdc": "^1.2.0" } }, "sha512-DbgptncYEXZqDUOEl4krff4mUiVrTZZVI7BBrQR/T3BqMj/eM1flTC1Uk2uUoLcWCxjT95xKulV/Lc6hhOZsBQ=="],
"fast-querystring": ["fast-querystring@1.1.2", "", { "dependencies": { "fast-decode-uri-component": "^1.0.1" } }, "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg=="],
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
"fastify": ["fastify@5.6.1", "", { "dependencies": { "@fastify/ajv-compiler": "^4.0.0", "@fastify/error": "^4.0.0", "@fastify/fast-json-stringify-compiler": "^5.0.0", "@fastify/proxy-addr": "^5.0.0", "abstract-logging": "^2.0.1", "avvio": "^9.0.0", "fast-json-stringify": "^6.0.0", "find-my-way": "^9.0.0", "light-my-request": "^6.0.0", "pino": "^9.0.0", "process-warning": "^5.0.0", "rfdc": "^1.3.1", "secure-json-parse": "^4.0.0", "semver": "^7.6.0", "toad-cache": "^3.7.0" } }, "sha512-WjjlOciBF0K8pDUPZoGPhqhKrQJ02I8DKaDIfO51EL0kbSMwQFl85cRwhOvmSDWoukNOdTo27gLN549pLCcH7Q=="],
"fastify-plugin": ["fastify-plugin@5.1.0", "", {}, "sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw=="],
"fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="],
"fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
"find-my-way": ["find-my-way@9.3.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-querystring": "^1.0.0", "safe-regex2": "^5.0.0" } }, "sha512-eRoFWQw+Yv2tuYlK2pjFS2jGXSxSppAs3hSQjfxVKxM5amECzIgYYc1FEI8ZmhSh/Ig+FrKEz43NLRKJjYCZVg=="],
"find-process": ["find-process@2.0.0", "", { "dependencies": { "chalk": "~4.1.2", "commander": "^12.1.0", "loglevel": "^1.9.2" }, "bin": { "find-process": "dist/bin/find-process.js" } }, "sha512-YUBQnteWGASJoEVVsOXy6XtKAY2O1FCsWnnvQ8y0YwgY1rZiKeVptnFvMu6RSELZAJOGklqseTnUGGs5D0bKmg=="],
"foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="],
"formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="],
"gaxios": ["gaxios@7.1.2", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2" } }, "sha512-/Szrn8nr+2TsQT1Gp8iIe/BEytJmbyfrbFh419DfGQSkEgNEhbPi7JRJuughjkTzPWgU9gBQf5AVu3DbHt0OXA=="],
"gcp-metadata": ["gcp-metadata@7.0.1", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ=="],
"glob": ["glob@11.0.3", "", { "dependencies": { "foreground-child": "^3.3.1", "jackspeak": "^4.1.1", "minimatch": "^10.0.3", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA=="],
"google-auth-library": ["google-auth-library@10.4.0", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.0.0", "gcp-metadata": "^7.0.0", "google-logging-utils": "^1.0.0", "gtoken": "^8.0.0", "jws": "^4.0.0" } }, "sha512-CmIrSy1bqMQUsPmA9+hcSbAXL80cFhu40cGMUjCaLpNKVzzvi+0uAHq8GNZxkoGYIsTX4ZQ7e4aInAqWxgn4fg=="],
"google-logging-utils": ["google-logging-utils@1.1.1", "", {}, "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A=="],
"gtoken": ["gtoken@8.0.0", "", { "dependencies": { "gaxios": "^7.0.0", "jws": "^4.0.0" } }, "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw=="],
"has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="],
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="], "html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
"htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="], "htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
"http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
"ipaddr.js": ["ipaddr.js@2.2.0", "", {}, "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA=="],
"is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
"is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="],
"isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
"jackspeak": ["jackspeak@4.1.1", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" } }, "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ=="],
"json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],
"json-schema-ref-resolver": ["json-schema-ref-resolver@3.0.0", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A=="],
"json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
"json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="],
"jsonrepair": ["jsonrepair@3.13.1", "", { "bin": { "jsonrepair": "bin/cli.js" } }, "sha512-WJeiE0jGfxYmtLwBTEk8+y/mYcaleyLXWaqp5bJu0/ZTSeG0KQq/wWQ8pmnkKenEdN6pdnn6QtcoSUkbqDHWNw=="],
"jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="],
"jws": ["jws@4.0.0", "", { "dependencies": { "jwa": "^2.0.0", "safe-buffer": "^5.0.1" } }, "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg=="],
"light-my-request": ["light-my-request@6.6.0", "", { "dependencies": { "cookie": "^1.0.1", "process-warning": "^4.0.0", "set-cookie-parser": "^2.6.0" } }, "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A=="],
"linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="], "linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="],
"loglevel": ["loglevel@1.9.2", "", {}, "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg=="],
"lru-cache": ["lru-cache@11.2.2", "", {}, "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg=="],
"mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="],
"minimatch": ["minimatch@10.0.3", "", { "dependencies": { "@isaacs/brace-expansion": "^5.0.0" } }, "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw=="],
"minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="],
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
"node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="],
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
"on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="],
"openai": ["openai@5.23.2", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg=="],
"openurl": ["openurl@1.1.1", "", {}, "sha512-d/gTkTb1i1GKz5k3XE3XFV/PxQ1k45zDqGP2OA7YhgsaLoqm6qRvARAZOFer1fcXritWlGBRCu/UgeS4HAnXAA=="],
"package-json-from-dist": ["package-json-from-dist@1.0.1", "", {}, "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="],
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
"path-scurry": ["path-scurry@2.0.0", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg=="],
"pino": ["pino@9.12.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "slow-redact": "^0.3.0", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-0Gd0OezGvqtqMwgYxpL7P0pSHHzTJ0Lx992h+mNlMtRVfNnqweWmf0JmRWk5gJzHalyd2mxTzKjhiNbGS2Ztfw=="],
"pino-abstract-transport": ["pino-abstract-transport@2.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw=="],
"pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="],
"process-warning": ["process-warning@5.0.0", "", {}, "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA=="],
"quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="],
"real-require": ["real-require@0.2.0", "", {}, "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="],
"require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
"ret": ["ret@0.5.0", "", {}, "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw=="],
"reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="],
"rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="],
"rotating-file-stream": ["rotating-file-stream@3.2.7", "", {}, "sha512-SVquhBEVvRFY+nWLUc791Y0MIlyZrEClRZwZFLLRgJKldHyV1z4e2e/dp9LPqCS3AM//uq/c3PnOFgjqnm5P+A=="],
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
"safe-regex2": ["safe-regex2@5.0.0", "", { "dependencies": { "ret": "~0.5.0" } }, "sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw=="],
"safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="],
"secure-json-parse": ["secure-json-parse@4.0.0", "", {}, "sha512-dxtLJO6sc35jWidmLxo7ij+Eg48PM/kleBsxpC8QJE0qJICe+KawkDQmvCMZUr9u7WKVHgMW6vy3fQ7zMiFZMA=="],
"semver": ["semver@7.7.2", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA=="],
"set-cookie-parser": ["set-cookie-parser@2.7.1", "", {}, "sha512-IOc8uWeOZgnb3ptbCURJWNjWUPcO3ZnTTdzsurqERrP6nPyv+paC55vJM0LpOlT2ne+Ix+9+CRG1MNLlyZ4GjQ=="],
"setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
"shell-quote": ["shell-quote@1.8.3", "", {}, "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw=="],
"signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
"slow-redact": ["slow-redact@0.3.0", "", {}, "sha512-cf723wn9JeRIYP9tdtd86GuqoR5937u64Io+CYjlm2i7jvu7g0H+Cp0l0ShAf/4ZL+ISUTVT+8Qzz7RZmp9FjA=="],
"sonic-boom": ["sonic-boom@4.2.0", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww=="],
"split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
"statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="],
"string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
"string-width-cjs": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "strip-ansi-cjs": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
"supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],
"thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="],
"tiktoken": ["tiktoken@1.0.22", "", {}, "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA=="],
"toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="],
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
"typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="], "uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], "undici": ["undici@7.16.0", "", {}, "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g=="],
"undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="],
"unidecode": ["unidecode@1.1.0", "", {}, "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA=="], "unidecode": ["unidecode@1.1.0", "", {}, "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA=="],
"uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="],
"web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="],
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
"wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="],
"wrap-ansi-cjs": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="],
"ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="],
"@google/genai/google-auth-library": ["google-auth-library@9.15.1", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^6.1.1", "gcp-metadata": "^6.1.0", "gtoken": "^7.0.0", "jws": "^4.0.0" } }, "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng=="],
"@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
"@isaacs/cliui/strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="],
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
"light-my-request/process-warning": ["process-warning@4.0.1", "", {}, "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q=="],
"wrap-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="],
"wrap-ansi/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
"wrap-ansi/strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="],
"@google/genai/google-auth-library/gaxios": ["gaxios@6.7.1", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", "node-fetch": "^2.6.9", "uuid": "^9.0.1" } }, "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ=="],
"@google/genai/google-auth-library/gcp-metadata": ["gcp-metadata@6.1.1", "", { "dependencies": { "gaxios": "^6.1.1", "google-logging-utils": "^0.0.2", "json-bigint": "^1.0.0" } }, "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A=="],
"@google/genai/google-auth-library/gtoken": ["gtoken@7.1.0", "", { "dependencies": { "gaxios": "^6.0.0", "jws": "^4.0.0" } }, "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw=="],
"@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
"@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
"wrap-ansi/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
"wrap-ansi/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
"@google/genai/google-auth-library/gaxios/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
"@google/genai/google-auth-library/gaxios/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
"@google/genai/google-auth-library/gcp-metadata/google-logging-utils": ["google-logging-utils@0.0.2", "", {}, "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ=="],
} }
} }

3
bunfig.toml Normal file
View File

@@ -0,0 +1,3 @@
[test]
# Test configuration
preload = ["./test/setup.ts"]

View File

@@ -1,33 +1,24 @@
# Marketplace Cookies Setup # Facebook Marketplace Cookies Setup
Both Facebook Marketplace and eBay require valid session cookies to bypass bot detection and access listings. To use the Facebook Marketplace scraper, you need to provide valid Facebook session cookies.
## Cookie Priority Hierarchy ## Option 1: Cookies File (`facebook.json`)
All scrapers follow this priority order (highest to lowest): 1. Log into Facebook in your browser
1. **URL Parameter** - Passed directly in API/MCP request (overrides all) 2. Open Developer Tools → Network tab
2. **Environment Variable** - Set as `FACEBOOK_COOKIE` or `EBAY_COOKIE` 3. Visit facebook.com/marketplace (ensure you're logged in)
3. **Cookie File** - Stored in `facebook.json` or `ebay.json` (fallback) 4. Look for any marketplace-related requests in the Network tab
5. Export cookies from the browser's Application/Storage → Cookies section
6. Save the cookies as a JSON array to `facebook.json`
--- The `facebook.json` file should contain Facebook session cookies, particularly:
## Facebook Marketplace (`facebook.json`)
### Required Cookies
- `c_user`: Your Facebook user ID - `c_user`: Your Facebook user ID
- `xs`: Facebook session token - `xs`: Facebook session token
- `fr`: Facebook request token - `fr`: Facebook request token
- `datr`: Data attribution token - `datr`: Data attribution token
- `sb`: Session browser token - `sb`: Session browser token
### Setup Methods Example structure:
**Method 1: Cookie File (Lowest Priority)**
1. Log into Facebook in your browser
2. Open Developer Tools → Application/Storage → Cookies
3. Export cookies as JSON array to `facebook.json`
Example `facebook.json`:
```json ```json
[ [
{ {
@@ -36,59 +27,26 @@ Example `facebook.json`:
"domain": ".facebook.com", "domain": ".facebook.com",
"path": "/", "path": "/",
"secure": true "secure": true
} },
// ... other cookies
] ]
``` ```
**Method 2: Environment Variable** ## Option 2: URL Parameter
```bash
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request' You can pass cookies directly via the `cookies` URL parameter:
``` ```
GET /api/facebook?q=laptop&cookies=[{"name":"c_user","value":"123","domain":".facebook.com",...}]
**Method 3: URL Parameter (Highest Priority)**
``` ```
GET /api/facebook?q=laptop&cookies=[{"name":"c_user","value":"123",...}]
```
---
## eBay (`ebay.json`)
eBay has aggressive bot detection that blocks requests without valid session cookies.
### Setup Methods
**Method 1: Cookie File (Lowest Priority)**
1. Log into eBay in your browser
2. Open Developer Tools → Network tab
3. Visit ebay.ca and inspect any request headers
4. Copy the full `Cookie` header value
5. Save as plain text to `ebay.json` (see `ebay.json.example`)
Example `ebay.json`:
```
s=VALUE; ds2=VALUE; ebay=VALUE; dp1=VALUE; nonsession=VALUE
```
**Method 2: Environment Variable**
```bash
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
```
**Method 3: URL Parameter (Highest Priority)**
```
GET /api/ebay?q=laptop&cookies=s=VALUE;ds2=VALUE;ebay=VALUE
```
---
## Important Notes ## Important Notes
- Cookies must be from active browser sessions - Cookies must be from an active Facebook session
- Cookies expire and need periodic refresh - Cookies expire, so you may need to refresh them periodically
- **NEVER** commit real cookies to version control - Never share real cookies or commit them to version control
- Platforms may still block automated scraping despite valid cookies - Facebook may block automated scraping even with valid cookies
## Security ## Security
All `*.json` files in this directory are git-ignored for security.</content> The cookies file is intentionally left out of version control for security reasons.</content>

View File

@@ -1 +0,0 @@
s=YOUR_VALUE; ds2=YOUR_VALUE; ebay=YOUR_VALUE; dp1=YOUR_VALUE; nonsession=YOUR_VALUE

22
docker-compose.yml Normal file
View File

@@ -0,0 +1,22 @@
services:
ca-marketplace-scraper:
container_name: ca-marketplace-scraper
build: .
ports:
- "4005:4005"
environment:
- NODE_ENV=production
- PORT=4005
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:4005/api/status"]
interval: 30s
timeout: 10s
retries: 3
start_period: 5s
restart: unless-stopped
networks:
- internal
networks:
internal:
driver: bridge
name: ca-marketplace-scraper-network

View File

@@ -1,9 +1,25 @@
{ {
"$schema": "https://opencode.ai/config.json", "$schema": "https://opencode.ai/config.json",
"mcp": { "mcp": {
"marketplace-scrape": { "chrome-devtools": {
"type": "local",
"command": [
"bunx",
"--bun",
"chrome-devtools-mcp@latest",
"--log-file",
"./debug.log",
"--headless=false",
"--isolated=false",
"-e",
"/nix/store/lz8ajxhnkkw2llj752bdz41wqr645h9c-google-chrome-dev-146.0.7635.0/bin/google-chrome-unstable",
"--ignore-default-chrome-arg='--disable-extensions'"
]
},
"bun-docs": {
"type": "remote", "type": "remote",
"url": "http://localhost:4006/mcp" "url": "https://bun.com/docs/mcp",
"timeout": 3000
} }
} }
} }

View File

@@ -1,21 +1,26 @@
{ {
"name": "marketplace-scrapers-monorepo", "name": "ca-marketplace-scraper",
"version": "1.0.0", "module": "./src/index.ts",
"scripts": { "scripts": {
"ci": "biome ci", "start": "bun ./src/index.ts",
"clean": "rm -rf dist", "dev": "bun --watch ./src/index.ts",
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify", "build": "bun build ./src/index.ts"
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
"build:all": "bun run build:api && bun run build:mcp",
"build": "bun run clean && bun run build:all",
"start": "./scripts/start.sh"
}, },
"private": true,
"type": "module", "type": "module",
"workspaces": [ "private": true,
"packages/*"
],
"devDependencies": { "devDependencies": {
"@biomejs/biome": "2.3.11" "@anthropic-ai/claude-code": "^2.0.1",
"@musistudio/claude-code-router": "^1.0.53",
"@types/bun": "latest",
"@types/unidecode": "^1.1.0",
"@types/cli-progress": "^3.11.6"
},
"peerDependencies": {
"typescript": "^5"
},
"dependencies": {
"cli-progress": "^3.12.0",
"linkedom": "^0.18.12",
"unidecode": "^1.1.0"
} }
} }

View File

@@ -1,21 +0,0 @@
{
"name": "@marketplace-scrapers/api-server",
"version": "1.0.0",
"type": "module",
"module": "./src/index.ts",
"private": true,
"scripts": {
"start": "bun ./src/index.ts",
"dev": "bun --watch ./src/index.ts",
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api"
},
"dependencies": {
"@marketplace-scrapers/core": "workspace:*"
},
"devDependencies": {
"@types/bun": "latest"
},
"peerDependencies": {
"typescript": "^5"
}
}

View File

@@ -1,30 +0,0 @@
import { ebayRoute } from "./routes/ebay";
import { facebookRoute } from "./routes/facebook";
import { kijijiRoute } from "./routes/kijiji";
import { statusRoute } from "./routes/status";
const PORT = process.env.PORT || 4005;
const server = Bun.serve({
port: PORT as number | string,
idleTimeout: 0,
routes: {
// Health check endpoint
"/api/status": statusRoute,
// Marketplace search endpoints
"/api/kijiji": kijijiRoute,
"/api/facebook": facebookRoute,
"/api/ebay": ebayRoute,
// Fallback for unmatched /api routes
"/api/*": Response.json({ message: "Not found" }, { status: 404 }),
},
// Fallback for all other routes
fetch(_req: Request) {
return new Response("Not Found", { status: 404 });
},
});
console.log(`API Server running on ${server.hostname}:${server.port}`);

View File

@@ -1,68 +0,0 @@
import { fetchEbayItems } from "@marketplace-scrapers/core";
/**
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}&cookies={cookies}
* Search eBay for listings (default: Buy It Now only, Canada only)
* Optional: Pass cookies parameter to bypass bot detection
*/
export async function ebayRoute(req: Request): Promise<Response> {
try {
const reqUrl = new URL(req.url);
const SEARCH_QUERY =
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
if (!SEARCH_QUERY)
return Response.json(
{
message:
"Request didn't have 'query' header or 'q' search parameter!",
},
{ status: 400 },
);
const minPriceParam = reqUrl.searchParams.get("minPrice");
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false";
const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false";
const exclusionsParam = reqUrl.searchParams.get("exclusions");
const exclusions = exclusionsParam
? exclusionsParam.split(",").map((s) => s.trim())
: [];
const keywordsParam = reqUrl.searchParams.get("keywords");
const keywords = keywordsParam
? keywordsParam.split(",").map((s) => s.trim())
: [SEARCH_QUERY];
const maxItemsParam = reqUrl.searchParams.get("maxItems");
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
const cookies = reqUrl.searchParams.get("cookies") || undefined;
const items = await fetchEbayItems(SEARCH_QUERY, 1, {
minPrice,
maxPrice,
strictMode,
exclusions,
keywords,
buyItNowOnly,
canadaOnly,
cookies,
});
const results = maxItems ? items.slice(0, maxItems) : items;
if (!results || results.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(results, { status: 200 });
} catch (error) {
console.error("eBay scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
}

View File

@@ -1,46 +0,0 @@
import { fetchFacebookItems } from "@marketplace-scrapers/core";
/**
* GET /api/facebook?q={query}&location={location}&cookies={cookies}
* Search Facebook Marketplace for listings
*/
export async function facebookRoute(req: Request): Promise<Response> {
const reqUrl = new URL(req.url);
const SEARCH_QUERY =
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
if (!SEARCH_QUERY)
return Response.json(
{
message: "Request didn't have 'query' header or 'q' search parameter!",
},
{ status: 400 },
);
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
const maxItemsParam = reqUrl.searchParams.get("maxItems");
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : 25;
try {
const items = await fetchFacebookItems(
SEARCH_QUERY,
1,
LOCATION,
maxItems,
COOKIES_SOURCE,
undefined,
);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Facebook scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
}

View File

@@ -1,67 +0,0 @@
import { fetchKijijiItems } from "@marketplace-scrapers/core";
/**
* GET /api/kijiji?q={query}
* Search Kijiji marketplace for listings
*/
export async function kijijiRoute(req: Request): Promise<Response> {
const reqUrl = new URL(req.url);
const SEARCH_QUERY =
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
if (!SEARCH_QUERY)
return Response.json(
{
message: "Request didn't have 'query' header or 'q' search parameter!",
},
{ status: 400 },
);
const maxPagesParam = reqUrl.searchParams.get("maxPages");
const maxPages = maxPagesParam ? parseInt(maxPagesParam, 10) : 5;
const priceMinParam = reqUrl.searchParams.get("priceMin");
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
const priceMaxParam = reqUrl.searchParams.get("priceMax");
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
const searchOptions = {
location: reqUrl.searchParams.get("location") || undefined,
category: reqUrl.searchParams.get("category") || undefined,
keywords: reqUrl.searchParams.get("keywords") || undefined,
sortBy: reqUrl.searchParams.get("sortBy") as
| "relevancy"
| "date"
| "price"
| "distance"
| undefined,
sortOrder: reqUrl.searchParams.get("sortOrder") as
| "desc"
| "asc"
| undefined,
maxPages,
priceMin,
priceMax,
cookies: reqUrl.searchParams.get("cookies") || undefined,
};
try {
const items = await fetchKijijiItems(
SEARCH_QUERY,
4, // 4 requests per second for faster scraping
"https://www.kijiji.ca",
searchOptions,
{},
);
if (!items)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Kijiji scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
}

View File

@@ -1,6 +0,0 @@
/**
* Health check endpoint
*/
export function statusRoute(): Response {
return new Response("OK", { status: 200 });
}

View File

@@ -1,13 +0,0 @@
{
"compilerOptions": {
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"paths": {
"@/*": ["./src/*"]
},
"strict": true,
"noEmit": true
}
}

View File

@@ -1,21 +0,0 @@
{
"name": "@marketplace-scrapers/core",
"version": "1.0.0",
"type": "module",
"main": "./src/index.ts",
"module": "./src/index.ts",
"private": true,
"dependencies": {
"cli-progress": "^3.12.0",
"linkedom": "^0.18.12",
"unidecode": "^1.1.0"
},
"devDependencies": {
"@types/bun": "latest",
"@types/unidecode": "^1.1.0",
"@types/cli-progress": "^3.11.6"
},
"peerDependencies": {
"typescript": "^5"
}
}

View File

@@ -1,43 +0,0 @@
// Export all scrapers
export type { EbayListingDetails } from "./scrapers/ebay";
export { default as fetchEbayItems } from "./scrapers/ebay";
export type { FacebookListingDetails } from "./scrapers/facebook";
export {
default as fetchFacebookItems,
ensureFacebookCookies,
extractFacebookItemData,
extractFacebookMarketplaceData,
fetchFacebookItem,
parseFacebookAds,
parseFacebookCookieString,
parseFacebookItem,
} from "./scrapers/facebook";
export type {
DetailedListing,
KijijiListingDetails,
ListingFetchOptions,
SearchOptions,
} from "./scrapers/kijiji";
export {
buildSearchUrl,
default as fetchKijijiItems,
extractApolloState,
HttpError,
NetworkError,
ParseError,
parseDetailedListing,
parseSearch,
RateLimitError,
resolveCategoryId,
resolveLocationId,
slugify,
ValidationError,
} from "./scrapers/kijiji";
// Export shared types
export * from "./types/common";
// Export shared utilities
export * from "./utils/cookies";
export * from "./utils/delay";
export * from "./utils/format";
export * from "./utils/http";

View File

@@ -1,20 +0,0 @@
/** HTML string alias for better type clarity */
export type HTMLString = string;
/** Currency price object with formatting options */
export interface Price {
amountFormatted: string;
cents: number;
currency: string;
}
/** Base listing details common across all marketplaces */
export interface ListingDetails {
url: string;
title: string;
listingPrice: Price;
listingType: string;
listingStatus: string;
address?: string | null;
creationDate?: string;
}

View File

@@ -1,227 +0,0 @@
/**
* Shared cookie handling utilities for marketplace scrapers
*/
export interface Cookie {
name: string;
value: string;
domain: string;
path: string;
secure?: boolean;
httpOnly?: boolean;
sameSite?: "strict" | "lax" | "none" | "unspecified";
session?: boolean;
expirationDate?: number;
partitionKey?: Record<string, unknown>;
storeId?: string;
}
export interface CookieConfig {
/** Name used in log messages (e.g., "Facebook", "Kijiji") */
name: string;
/** Domain for cookies (e.g., ".facebook.com", ".kijiji.ca") */
domain: string;
/** Environment variable name (e.g., "FACEBOOK_COOKIE") */
envVar: string;
/** Path to cookie file (e.g., "./cookies/facebook.json") */
filePath: string;
}
/**
* Parse cookie string format into Cookie array
* Supports format: "name1=value1; name2=value2"
*/
export function parseCookieString(
cookieString: string,
domain: string,
): Cookie[] {
if (!cookieString?.trim()) {
return [];
}
return cookieString
.split(";")
.map((pair) => pair.trim())
.filter((pair) => pair.includes("="))
.map((pair) => {
const [name, ...valueParts] = pair.split("=");
const trimmedName = name.trim();
const trimmedValue = valueParts.join("=").trim();
if (!trimmedName || !trimmedValue) {
return null;
}
return {
name: trimmedName,
value: decodeURIComponent(trimmedValue),
domain,
path: "/",
secure: true,
httpOnly: false,
sameSite: "lax" as const,
expirationDate: undefined,
};
})
.filter((cookie): cookie is Cookie => cookie !== null);
}
/**
* Parse JSON array format into Cookie array
* Supports format: [{"name": "foo", "value": "bar", ...}]
*/
export function parseJsonCookies(jsonString: string): Cookie[] {
const parsed = JSON.parse(jsonString);
if (!Array.isArray(parsed)) {
return [];
}
return parsed.filter(
(cookie): cookie is Cookie =>
cookie &&
typeof cookie.name === "string" &&
typeof cookie.value === "string",
);
}
/**
* Try to parse cookies from a string (tries JSON first, then cookie string format)
*/
export function parseCookiesAuto(
input: string,
defaultDomain: string,
): Cookie[] {
// Try JSON array format first
try {
const cookies = parseJsonCookies(input);
if (cookies.length > 0) {
return cookies;
}
} catch {
// JSON parse failed, try cookie string format
}
// Try cookie string format
return parseCookieString(input, defaultDomain);
}
/**
* Load cookies from file (supports both JSON array and cookie string formats)
*/
export async function loadCookiesFromFile(
filePath: string,
defaultDomain: string,
): Promise<Cookie[]> {
const file = Bun.file(filePath);
if (!(await file.exists())) {
return [];
}
const content = await file.text();
return parseCookiesAuto(content.trim(), defaultDomain);
}
/**
* Format cookies array into Cookie header string for HTTP requests
*/
export function formatCookiesForHeader(
cookies: Cookie[],
targetDomain: string,
): string {
const validCookies = cookies
.filter((cookie) => {
// Check if cookie applies to this domain
if (cookie.domain.startsWith(".")) {
// Domain cookie (applies to subdomains)
return (
targetDomain.endsWith(cookie.domain.slice(1)) ||
targetDomain === cookie.domain.slice(1)
);
}
// Host-only cookie
return cookie.domain === targetDomain;
})
.filter((cookie) => {
// Check expiration
if (cookie.expirationDate && cookie.expirationDate < Date.now() / 1000) {
return false;
}
return true;
});
return validCookies
.map((cookie) => `${cookie.name}=${cookie.value}`)
.join("; ");
}
/**
* Load cookies with priority: URL param > ENV var > file
* Supports both JSON array and cookie string formats for all sources
*/
export async function ensureCookies(
config: CookieConfig,
cookiesSource?: string,
): Promise<Cookie[]> {
// Priority 1: URL/API parameter (if provided)
if (cookiesSource) {
const cookies = parseCookiesAuto(cookiesSource, config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from parameter`,
);
return cookies;
}
console.warn(
`${config.name} cookies parameter provided but no valid cookies extracted`,
);
}
// Priority 2: Environment variable
const envValue = process.env[config.envVar];
if (envValue?.trim()) {
const cookies = parseCookiesAuto(envValue, config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
);
return cookies;
}
console.warn(`${config.envVar} env var contains no valid cookies`);
}
// Priority 3: Cookie file (fallback)
try {
const cookies = await loadCookiesFromFile(config.filePath, config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.filePath}`,
);
return cookies;
}
} catch (e) {
console.warn(`Could not load cookies from ${config.filePath}: ${e}`);
}
// No cookies found from any source
throw new Error(
`No valid ${config.name} cookies found. Provide cookies via (in priority order):\n` +
` 1. 'cookies' parameter (highest priority), or\n` +
` 2. ${config.envVar} environment variable, or\n` +
` 3. ${config.filePath} file (lowest priority)\n` +
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
);
}
/**
* Try to load cookies, return empty array if none found (non-throwing version)
*/
export async function loadCookiesOptional(
config: CookieConfig,
cookiesSource?: string,
): Promise<Cookie[]> {
try {
return await ensureCookies(config, cookiesSource);
} catch {
return [];
}
}

View File

@@ -1,8 +0,0 @@
/**
* Delay execution for a specified number of milliseconds
* @param ms - Milliseconds to delay
* @returns A promise that resolves after the specified delay
*/
export function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View File

@@ -1,24 +0,0 @@
/**
* Format cents to a human-readable currency string
* @param cents - Amount in cents (integer)
* @param locale - Locale string for formatting (e.g., 'en-CA', 'en-US')
* @returns Formatted currency string
*/
export function formatCentsToCurrency(
cents: number,
locale: string = "en-CA",
): string {
try {
const formatter = new Intl.NumberFormat(locale, {
style: "currency",
currency: "CAD",
minimumFractionDigits: 2,
maximumFractionDigits: 2,
});
return formatter.format(cents / 100);
} catch {
// Fallback if locale is not supported
const dollars = (cents / 100).toFixed(2);
return `$${dollars}`;
}
}

View File

@@ -1,200 +0,0 @@
/** Custom error class for HTTP-related failures */
export class HttpError extends Error {
constructor(
message: string,
public readonly statusCode: number,
public readonly url?: string,
) {
super(message);
this.name = "HttpError";
}
}
/** Error class for network failures (timeouts, connection issues) */
export class NetworkError extends Error {
constructor(
message: string,
public readonly url: string,
public readonly cause?: Error,
) {
super(message);
this.name = "NetworkError";
}
}
/** Error class for parsing failures */
export class ParseError extends Error {
constructor(
message: string,
public readonly data?: unknown,
) {
super(message);
this.name = "ParseError";
}
}
/** Error class for rate limiting */
export class RateLimitError extends Error {
constructor(
message: string,
public readonly url: string,
public readonly resetTime?: number,
) {
super(message);
this.name = "RateLimitError";
}
}
/** Error class for validation failures */
export class ValidationError extends Error {
constructor(message: string) {
super(message);
this.name = "ValidationError";
}
}
/** Type guard to check if a value is a record (object) */
export function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
/**
* Calculate exponential backoff delay with jitter
*/
function calculateBackoffDelay(attempt: number, baseMs: number): number {
const exponentialDelay = baseMs * 2 ** attempt;
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
}
/** Options for fetchHtml */
export interface FetchHtmlOptions {
maxRetries?: number;
retryBaseMs?: number;
timeoutMs?: number;
onRateInfo?: (remaining: string | null, reset: string | null) => void;
headers?: Record<string, string>;
}
/**
* Fetch HTML content from a URL with automatic retries, timeout, and exponential backoff
* @param url - The URL to fetch
* @param delayMs - Delay in milliseconds between requests (rate limiting)
* @param opts - Optional fetch options
* @returns The HTML content as a string
* @throws HttpError, NetworkError, or RateLimitError on failure
*/
export async function fetchHtml(
url: string,
delayMs: number,
opts?: FetchHtmlOptions,
): Promise<string> {
const maxRetries = opts?.maxRetries ?? 3;
const retryBaseMs = opts?.retryBaseMs ?? 1000;
const timeoutMs = opts?.timeoutMs ?? 30000;
const defaultHeaders: Record<string, string> = {
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
"cache-control": "no-cache",
"upgrade-insecure-requests": "1",
"user-agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
};
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
const res = await fetch(url, {
method: "GET",
headers: { ...defaultHeaders, ...opts?.headers },
signal: controller.signal,
});
clearTimeout(timeoutId);
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
if (!res.ok) {
// Handle rate limiting
if (res.status === 429) {
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: calculateBackoffDelay(attempt, retryBaseMs);
if (attempt < maxRetries) {
await new Promise((resolve) => setTimeout(resolve, waitMs));
continue;
}
throw new RateLimitError(
`Rate limit exceeded for ${url}`,
url,
resetSeconds,
);
}
// Retry on server errors
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
await new Promise((resolve) =>
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
);
continue;
}
throw new HttpError(
`Request failed with status ${res.status}`,
res.status,
url,
);
}
const html = await res.text();
// Respect per-request delay to maintain rate limiting
await new Promise((resolve) => setTimeout(resolve, delayMs));
return html;
} catch (err) {
// Re-throw known errors
if (
err instanceof RateLimitError ||
err instanceof HttpError ||
err instanceof NetworkError
) {
throw err;
}
if (err instanceof Error && err.name === "AbortError") {
if (attempt < maxRetries) {
await new Promise((resolve) =>
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
);
continue;
}
throw new NetworkError(`Request timeout for ${url}`, url, err);
}
// Network or other errors
if (attempt < maxRetries) {
await new Promise((resolve) =>
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
);
continue;
}
throw new NetworkError(
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
url,
err instanceof Error ? err : undefined,
);
}
}
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
}

View File

@@ -1,13 +0,0 @@
{
"compilerOptions": {
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"paths": {
"@/*": ["./src/*"]
},
"strict": true,
"noEmit": true
}
}

View File

@@ -1,21 +0,0 @@
{
"name": "@marketplace-scrapers/mcp-server",
"version": "1.0.0",
"type": "module",
"module": "./src/index.ts",
"private": true,
"scripts": {
"start": "bun ./src/index.ts",
"dev": "bun --watch ./src/index.ts",
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp"
},
"dependencies": {
"@marketplace-scrapers/core": "workspace:*"
},
"devDependencies": {
"@types/bun": "latest"
},
"peerDependencies": {
"typescript": "^5"
}
}

View File

@@ -1,36 +0,0 @@
import { handleMcpRequest } from "./protocol/handler";
import { serverCard } from "./protocol/metadata";
const PORT = process.env.MCP_PORT || 4006;
const server = Bun.serve({
port: PORT as number | string,
idleTimeout: 255, // 255 seconds (max allowed)
routes: {
// MCP metadata discovery endpoint
"/.well-known/mcp/server-card.json": new Response(
JSON.stringify(serverCard),
{
headers: { "Content-Type": "application/json" },
},
),
// MCP JSON-RPC 2.0 protocol endpoint
"/mcp": async (req: Request) => {
if (req.method === "POST") {
return await handleMcpRequest(req);
}
return Response.json(
{ message: "MCP endpoint requires POST request" },
{ status: 405 },
);
},
},
// Fallback for all other routes
fetch(_req: Request) {
return new Response("Not Found", { status: 404 });
},
});
console.log(`MCP Server running on ${server.hostname}:${server.port}`);

View File

@@ -1,292 +0,0 @@
import { tools } from "./tools";
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; // 3 minutes default
/**
* Handle MCP JSON-RPC 2.0 protocol requests
*/
export async function handleMcpRequest(req: Request): Promise<Response> {
try {
const body = await req.json();
// Validate JSON-RPC 2.0 format
if (!body.jsonrpc || body.jsonrpc !== "2.0" || !body.method) {
return Response.json(
{
jsonrpc: "2.0",
error: { code: -32600, message: "Invalid Request" },
id: body.id,
},
{ status: 400 },
);
}
const { method, params, id } = body;
// Handle initialize method
if (method === "initialize") {
return Response.json({
jsonrpc: "2.0",
id,
result: {
protocolVersion: "2025-06-18",
capabilities: {
tools: {
listChanged: true,
},
},
serverInfo: {
name: "marketplace-scrapers",
version: "1.0.0",
},
instructions:
"Use search_kijiji, search_facebook, or search_ebay tools to find listings across Canadian marketplaces",
},
});
}
// Handle tools/list method
if (method === "tools/list") {
return Response.json({
jsonrpc: "2.0",
id,
result: {
tools,
},
});
}
// Handle notifications (messages without id field should not get a response)
if (!id) {
// Notifications don't require a response
if (method === "notifications/initialized") {
// Client initialized successfully, no response needed
return new Response(null, { status: 204 });
}
if (method === "notifications/progress") {
// Progress notifications, no response needed
return new Response(null, { status: 204 });
}
// Unknown notification - still no response for notifications
return new Response(null, { status: 204 });
}
// Handle tools/call method
if (method === "tools/call") {
const { name, arguments: args } = params || {};
if (!name || !args) {
return Response.json(
{
jsonrpc: "2.0",
id,
error: {
code: -32602,
message: "Invalid params: name and arguments required",
},
},
{ status: 400 },
);
}
// Route tool calls to appropriate handlers
try {
let result: unknown;
if (name === "search_kijiji") {
const query = args.query;
if (!query) {
return Response.json({
jsonrpc: "2.0",
id,
error: { code: -32602, message: "query parameter is required" },
});
}
const params = new URLSearchParams({ q: query });
if (args.location) params.append("location", args.location);
if (args.category) params.append("category", args.category);
if (args.keywords) params.append("keywords", args.keywords);
if (args.sortBy) params.append("sortBy", args.sortBy);
if (args.sortOrder) params.append("sortOrder", args.sortOrder);
if (args.maxPages)
params.append("maxPages", args.maxPages.toString());
if (args.priceMin)
params.append("priceMin", args.priceMin.toString());
if (args.priceMax)
params.append("priceMax", args.priceMax.toString());
if (args.cookies) params.append("cookies", args.cookies);
console.log(
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
);
const response = await Promise.race([
fetch(`${API_BASE_URL}/kijiji?${params.toString()}`),
new Promise<Response>((_, reject) =>
setTimeout(
() =>
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
API_TIMEOUT,
),
),
]);
if (!response.ok) {
const errorText = await response.text();
console.error(
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else if (name === "search_facebook") {
const query = args.query;
if (!query) {
return Response.json({
jsonrpc: "2.0",
id,
error: { code: -32602, message: "query parameter is required" },
});
}
const params = new URLSearchParams({ q: query });
if (args.location) params.append("location", args.location);
if (args.maxItems)
params.append("maxItems", args.maxItems.toString());
if (args.cookiesSource) params.append("cookies", args.cookiesSource);
console.log(
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
);
const response = await Promise.race([
fetch(`${API_BASE_URL}/facebook?${params.toString()}`),
new Promise<Response>((_, reject) =>
setTimeout(
() =>
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
API_TIMEOUT,
),
),
]);
if (!response.ok) {
const errorText = await response.text();
console.error(
`[MCP] Facebook API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else if (name === "search_ebay") {
const query = args.query;
if (!query) {
return Response.json({
jsonrpc: "2.0",
id,
error: { code: -32602, message: "query parameter is required" },
});
}
const params = new URLSearchParams({ q: query });
if (args.minPrice)
params.append("minPrice", args.minPrice.toString());
if (args.maxPrice)
params.append("maxPrice", args.maxPrice.toString());
if (args.strictMode !== undefined)
params.append("strictMode", args.strictMode.toString());
if (args.exclusions?.length)
params.append("exclusions", args.exclusions.join(","));
if (args.keywords?.length)
params.append("keywords", args.keywords.join(","));
if (args.buyItNowOnly !== undefined)
params.append("buyItNowOnly", args.buyItNowOnly.toString());
if (args.canadaOnly !== undefined)
params.append("canadaOnly", args.canadaOnly.toString());
if (args.maxItems)
params.append("maxItems", args.maxItems.toString());
if (args.cookies) params.append("cookies", args.cookies);
console.log(
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
);
const response = await Promise.race([
fetch(`${API_BASE_URL}/ebay?${params.toString()}`),
new Promise<Response>((_, reject) =>
setTimeout(
() =>
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
API_TIMEOUT,
),
),
]);
if (!response.ok) {
const errorText = await response.text();
console.error(
`[MCP] eBay API error ${response.status}: ${errorText}`,
);
throw new Error(`API returned ${response.status}: ${errorText}`);
}
result = await response.json();
console.log(
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
);
} else {
return Response.json({
jsonrpc: "2.0",
id,
error: { code: -32601, message: `Unknown tool: ${name}` },
});
}
return Response.json({
jsonrpc: "2.0",
id,
result: {
content: [
{
type: "text",
text: JSON.stringify(result, null, 2),
},
],
},
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Unknown error";
return Response.json({
jsonrpc: "2.0",
id,
error: {
code: -32603,
message: `Tool execution failed: ${errorMessage}`,
},
});
}
}
// Method not found
return Response.json(
{
jsonrpc: "2.0",
id,
error: { code: -32601, message: `Method not found: ${method}` },
},
{ status: 404 },
);
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Unknown error";
return Response.json(
{
jsonrpc: "2.0",
error: { code: -32700, message: `Parse error: ${errorMessage}` },
},
{ status: 400 },
);
}
}

View File

@@ -1,27 +0,0 @@
/**
* MCP Server metadata for discovery
*/
export const serverCard = {
$schema:
"https://static.modelcontextprotocol.io/schemas/mcp-server-card/v1.json",
version: "1.0",
protocolVersion: "2025-06-18",
serverInfo: {
name: "marketplace-scrapers",
title: "Marketplace Scrapers MCP Server",
version: "1.0.0",
},
transport: {
type: "streamable-http",
endpoint: "/mcp",
},
capabilities: {
tools: {
listChanged: true,
},
},
description:
"Scrapes marketplace listings from Kijiji, Facebook Marketplace, and eBay",
tools: "dynamic",
};

View File

@@ -1,150 +0,0 @@
/**
* MCP tool definitions for marketplace scrapers
*/
export const tools = [
{
name: "search_kijiji",
description: "Search Kijiji marketplace for listings matching a query",
inputSchema: {
type: "object",
properties: {
query: {
type: "string",
description: "Search query for Kijiji listings",
},
location: {
type: "string",
description:
"Location name or ID (e.g., 'toronto', 'gta', 'ontario')",
},
category: {
type: "string",
description:
"Category name or ID (e.g., 'computers', 'furniture', 'bikes')",
},
keywords: {
type: "string",
description: "Additional keywords to filter results",
},
sortBy: {
type: "string",
description: "Sort results by field",
enum: ["relevancy", "date", "price", "distance"],
default: "relevancy",
},
sortOrder: {
type: "string",
description: "Sort order",
enum: ["asc", "desc"],
default: "desc",
},
maxPages: {
type: "number",
description: "Maximum pages to fetch (~40 items per page)",
default: 5,
},
priceMin: {
type: "number",
description: "Minimum price in cents",
},
priceMax: {
type: "number",
description: "Maximum price in cents",
},
cookies: {
type: "string",
description:
"Optional: Kijiji session cookies to bypass bot detection (JSON array or 'name1=value1; name2=value2')",
},
},
required: ["query"],
},
},
{
name: "search_facebook",
description: "Search Facebook Marketplace for listings matching a query",
inputSchema: {
type: "object",
properties: {
query: {
type: "string",
description: "Search query for Facebook Marketplace listings",
},
location: {
type: "string",
description: "Location for search (e.g., 'toronto')",
default: "toronto",
},
maxItems: {
type: "number",
description: "Maximum number of items to return",
default: 5,
},
cookiesSource: {
type: "string",
description: "Optional Facebook session cookies source",
},
},
required: ["query"],
},
},
{
name: "search_ebay",
description:
"Search eBay for listings matching a query (default: Buy It Now only, Canada only)",
inputSchema: {
type: "object",
properties: {
query: {
type: "string",
description: "Search query for eBay listings",
},
minPrice: {
type: "number",
description: "Minimum price filter",
},
maxPrice: {
type: "number",
description: "Maximum price filter",
},
strictMode: {
type: "boolean",
description: "Enable strict search mode",
default: false,
},
exclusions: {
type: "array",
items: { type: "string" },
description: "Terms to exclude from results",
},
keywords: {
type: "array",
items: { type: "string" },
description: "Keywords to include in search",
},
buyItNowOnly: {
type: "boolean",
description: "Include only Buy It Now listings (exclude auctions)",
default: true,
},
canadaOnly: {
type: "boolean",
description: "Include only Canadian sellers/listings",
default: true,
},
maxItems: {
type: "number",
description: "Maximum number of items to return",
default: 5,
},
cookies: {
type: "string",
description:
"Optional: eBay session cookies to bypass bot detection (format: 'name1=value1; name2=value2')",
},
},
required: ["query"],
},
},
];

View File

@@ -1,13 +0,0 @@
{
"compilerOptions": {
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"paths": {
"@/*": ["./src/*"]
},
"strict": true,
"noEmit": true
}
}

View File

@@ -1,26 +0,0 @@
#!/usr/bin/env bash
# Get the path to the system biome executable
BIOME_PATH=$(which biome)
if [ -z "$BIOME_PATH" ]; then
echo "Error: biome executable not found in PATH"
exit 1
fi
# Find all biome executables in node_modules
files=$(fd biome node_modules --type executable --no-ignore --follow)
if [ -z "$files" ]; then
echo "No biome executables found in node_modules"
exit 0
fi
# Replace each with a symlink to the system biome
for file in $files; do
echo "Replacing $file with symlink to $BIOME_PATH"
rm "$file"
ln -s "$BIOME_PATH" "$file"
done
echo "Done."

View File

@@ -1,30 +0,0 @@
#!/usr/bin/env bash
PATTERN="eslint"
FILES="$(fd .)" # Or use 'find .' to search recursively
for file in $FILES; do
if [[ -f "$file" ]]; then
# 1. Use rg with line numbers (-n) and only the matched line (-o)
# 2. Use awk to print ONLY the line number (field 1)
# 3. Use xargs to pass multiple line numbers to a single sed command
LINE_NUMBERS=$(rg --line-number --no-filename "$PATTERN" "$file" | awk -F':' '{print $1}' | tr '\n' ',')
# Remove trailing comma if any
LINE_NUMBERS=${LINE_NUMBERS%,}
if [[ -n "$LINE_NUMBERS" ]]; then
echo "Deleting lines $LINE_NUMBERS from $file..."
# Use sed to delete the specified comma-separated line numbers in-place (-i)
# NOTE: The syntax for -i might vary slightly between GNU sed (Linux) and BSD sed (macOS).
sed -i.bak "${LINE_NUMBERS}d" "$file"
# Optional: Remove the backup file created by sed -i.bak
# rm "${file}.bak"
else
echo "$file: No lines matching pattern found."
fi
fi
done

View File

@@ -1,25 +0,0 @@
#!/usr/bin/env bash
set -e
# Trap SIGTERM and SIGINT for graceful shutdown
trap 'echo "Received shutdown signal, stopping services..."; kill -TERM $API_PID $MCP_PID 2>/dev/null; wait' TERM INT
# Start API Server in background
echo "Starting API Server on port ${API_PORT:-4005}..."
bun dist/api/index.js &
API_PID=$!
# Give API server a moment to initialize
sleep 1
# Start MCP Server in background
echo "Starting MCP Server on port ${API_PORT:-4006}..."
bun dist/mcp/index.js &
MCP_PID=$!
echo "Both services started successfully"
echo "API Server PID: $API_PID"
echo "MCP Server PID: $MCP_PID"
# Wait for both processes
wait $API_PID $MCP_PID

View File

@@ -1,22 +1,12 @@
import cliProgress from "cli-progress";
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import {
type CookieConfig,
formatCookiesForHeader,
loadCookiesOptional,
} from "../utils/cookies";
import { delay } from "../utils/delay";
// eBay cookie configuration
const EBAY_COOKIE_CONFIG: CookieConfig = {
name: "eBay",
domain: ".ebay.ca",
envVar: "EBAY_COOKIE",
filePath: "./cookies/ebay.json",
};
// ----------------------------- Types ----------------------------- // ----------------------------- Types -----------------------------
export interface EbayListingDetails { type HTMLString = string;
type ListingDetails = {
url: string; url: string;
title: string; title: string;
description?: string; description?: string;
@@ -31,10 +21,37 @@ export interface EbayListingDetails {
endDate?: string; endDate?: string;
numberOfViews?: number; numberOfViews?: number;
address?: string | null; address?: string | null;
} };
// ----------------------------- Utilities ----------------------------- // ----------------------------- Utilities -----------------------------
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
async function delay(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Turns cents to localized currency string.
*/
function formatCentsToCurrency(
num: number | string | undefined,
locale = "en-US",
): string {
if (num == null) return "";
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
if (Number.isNaN(cents)) return "";
const dollars = cents / 100;
const formatter = new Intl.NumberFormat(locale, {
minimumFractionDigits: 2,
maximumFractionDigits: 2,
useGrouping: true,
});
return formatter.format(dollars);
}
/** /**
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents * Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
*/ */
@@ -51,7 +68,7 @@ function parseEbayPrice(
if (!numberMatches) return null; if (!numberMatches) return null;
const amountStr = numberMatches[0].replace(/,/g, ""); const amountStr = numberMatches[0].replace(/,/g, "");
const dollars = parseFloat(amountStr); const dollars = Number.parseFloat(amountStr);
if (Number.isNaN(dollars)) return null; if (Number.isNaN(dollars)) return null;
const cents = Math.round(dollars * 100); const cents = Math.round(dollars * 100);
@@ -83,6 +100,81 @@ class HttpError extends Error {
} }
} }
// ----------------------------- HTTP Client -----------------------------
/**
Fetch HTML with a basic retry strategy and simple rate-limit delay between calls.
- Retries on 429 and 5xx
- Respects X-RateLimit-Reset when present (seconds)
*/
async function fetchHtml(
url: string,
DELAY_MS: number,
opts?: {
maxRetries?: number;
retryBaseMs?: number;
onRateInfo?: (remaining: string | null, reset: string | null) => void;
},
): Promise<HTMLString> {
const maxRetries = opts?.maxRetries ?? 3;
const retryBaseMs = opts?.retryBaseMs ?? 500;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
const res = await fetch(url, {
method: "GET",
headers: {
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "en-CA,en-US;q=0.9,en;q=0.8",
"cache-control": "no-cache",
"upgrade-insecure-requests": "1",
"user-agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
},
});
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
if (!res.ok) {
// Respect 429 reset if provided
if (res.status === 429) {
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: (attempt + 1) * retryBaseMs;
await delay(waitMs);
continue;
}
// Retry on 5xx
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
await delay((attempt + 1) * retryBaseMs);
continue;
}
throw new HttpError(
`Request failed with status ${res.status}`,
res.status,
url,
);
}
const html = await res.text();
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS);
return html;
} catch (err) {
if (attempt >= maxRetries) throw err;
await delay((attempt + 1) * retryBaseMs);
}
}
throw new Error("Exhausted retries without response");
}
// ----------------------------- Parsing ----------------------------- // ----------------------------- Parsing -----------------------------
/** /**
@@ -93,9 +185,9 @@ function parseEbayListings(
keywords: string[], keywords: string[],
exclusions: string[], exclusions: string[],
strictMode: boolean, strictMode: boolean,
): EbayListingDetails[] { ): ListingDetails[] {
const { document } = parseHTML(htmlString); const { document } = parseHTML(htmlString);
const results: EbayListingDetails[] = []; const results: ListingDetails[] = [];
// Find all listing links by looking for eBay item URLs (/itm/) // Find all listing links by looking for eBay item URLs (/itm/)
const linkElements = document.querySelectorAll('a[href*="itm/"]'); const linkElements = document.querySelectorAll('a[href*="itm/"]');
@@ -114,26 +206,13 @@ function parseEbayListings(
} }
// Find the container - go up several levels to find the item container // Find the container - go up several levels to find the item container
// Modern eBay uses complex nested structures (often 5-10 levels deep) // Modern eBay uses complex nested structures
let container: Element | null = linkElement; let container = linkElement.parentElement?.parentElement?.parentElement;
let depth = 0; if (!container) {
const maxDepth = 15; // Try a different level
container = linkElement.parentElement?.parentElement;
// Walk up until we find a list item or results container
while (container && depth < maxDepth) {
const classes = container.className || "";
if (
classes.includes("s-item") ||
classes.includes("srp-results") ||
container.tagName === "LI"
) {
break;
} }
container = container.parentElement; if (!container) continue;
depth++;
}
if (!container || depth >= maxDepth) continue;
// Extract title - look for heading or title-related elements near the link // Extract title - look for heading or title-related elements near the link
// Modern eBay often uses h3, span, or div with text content near the link // Modern eBay often uses h3, span, or div with text content near the link
@@ -194,9 +273,8 @@ function parseEbayListings(
if (title === "Shop on eBay" || title.length < 3) continue; if (title === "Shop on eBay" || title.length < 3) continue;
// Extract price - look for eBay's price classes, preferring sale/discount prices // Extract price - look for eBay's price classes, preferring sale/discount prices
// Updated for 2026 eBay HTML structure
let priceElement = container.querySelector( let priceElement = container.querySelector(
'[class*="s-item__price"], .s-item__price, .s-card__attribute-row, [class*="price"]', '[class*="s-item__price"], .s-item__price, [class*="price"]',
); );
// If no direct price class, look for spans containing $ (but not titles) // If no direct price class, look for spans containing $ (but not titles)
@@ -245,7 +323,7 @@ function parseEbayListings(
const text = el.textContent?.trim(); const text = el.textContent?.trim();
if ( if (
text && text &&
/^\s*[$£¥]/u.test(text) && /^\s*[\$£¥]/u.test(text) &&
text.length < 50 && text.length < 50 &&
!/\d{4}/.test(text) !/\d{4}/.test(text)
) { ) {
@@ -302,15 +380,14 @@ function parseEbayListings(
// Apply strict mode filter (title must contain at least one keyword) // Apply strict mode filter (title must contain at least one keyword)
if ( if (
strictMode && strictMode &&
title &&
!keywords.some((keyword) => !keywords.some((keyword) =>
title.toLowerCase().includes(keyword.toLowerCase()), title?.toLowerCase().includes(keyword.toLowerCase()),
) )
) { ) {
continue; continue;
} }
const listing: EbayListingDetails = { const listing: ListingDetails = {
url: href, url: href,
title, title,
listingPrice: { listingPrice: {
@@ -332,32 +409,6 @@ function parseEbayListings(
return results; return results;
} }
// ----------------------------- Cookie Loading -----------------------------
/**
* Load eBay cookies with priority: URL param > ENV var > file
* Uses shared cookie utility for consistent handling across all scrapers
*/
async function loadEbayCookies(
cookiesSource?: string,
): Promise<string | undefined> {
const cookies = await loadCookiesOptional(EBAY_COOKIE_CONFIG, cookiesSource);
if (cookies.length === 0) {
console.warn(
"No eBay cookies found. eBay may block requests without valid session cookies.\n" +
"Provide cookies via (in priority order):\n" +
" 1. 'cookies' URL parameter (highest priority), or\n" +
" 2. EBAY_COOKIE environment variable, or\n" +
" 3. ./cookies/ebay.json file (lowest priority)\n" +
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
);
return undefined;
}
return formatCookiesForHeader(cookies, "www.ebay.ca");
}
// ----------------------------- Main ----------------------------- // ----------------------------- Main -----------------------------
export default async function fetchEbayItems( export default async function fetchEbayItems(
@@ -369,9 +420,6 @@ export default async function fetchEbayItems(
strictMode?: boolean; strictMode?: boolean;
exclusions?: string[]; exclusions?: string[];
keywords?: string[]; keywords?: string[];
buyItNowOnly?: boolean;
canadaOnly?: boolean;
cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection)
} = {}, } = {},
) { ) {
const { const {
@@ -380,30 +428,10 @@ export default async function fetchEbayItems(
strictMode = false, strictMode = false,
exclusions = [], exclusions = [],
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
buyItNowOnly = true,
canadaOnly = true,
cookies: cookiesSource,
} = opts; } = opts;
// Load eBay cookies with priority: URL param > ENV var > file // Build eBay search URL - use Canadian site and tracking parameters like real browser
const cookies = await loadEbayCookies(cookiesSource); const searchUrl = `https://www.ebay.ca/sch/i.html?_nkw=${encodeURIComponent(SEARCH_QUERY)}^&_sacat=0^&_from=R40^&_trksid=p4432023.m570.l1313`;
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
const urlParams = new URLSearchParams({
_nkw: SEARCH_QUERY,
_sacat: "0",
_from: "R40",
});
if (buyItNowOnly) {
urlParams.set("LH_BIN", "1");
}
if (canadaOnly) {
urlParams.set("LH_PrefLoc", "1");
}
const searchUrl = `https://www.ebay.ca/sch/i.html?${urlParams.toString()}`;
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND)); const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
@@ -416,7 +444,7 @@ export default async function fetchEbayItems(
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0", "Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5", "Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br, zstd", "Accept-Encoding": "gzip, deflate, br",
Referer: "https://www.ebay.ca/", Referer: "https://www.ebay.ca/",
Connection: "keep-alive", Connection: "keep-alive",
"Upgrade-Insecure-Requests": "1", "Upgrade-Insecure-Requests": "1",
@@ -427,11 +455,6 @@ export default async function fetchEbayItems(
Priority: "u=0, i", Priority: "u=0, i",
}; };
// Add cookies if available (helps bypass bot detection)
if (cookies) {
headers.Cookie = cookies;
}
const res = await fetch(searchUrl, { const res = await fetch(searchUrl, {
method: "GET", method: "GET",
headers, headers,
@@ -449,7 +472,7 @@ export default async function fetchEbayItems(
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND // Respect per-request delay to keep at or under REQUESTS_PER_SECOND
await delay(DELAY_MS); await delay(DELAY_MS);
console.log(`\nParsing eBay listings...`); console.log("\nParsing eBay listings...");
const listings = parseEbayListings( const listings = parseEbayListings(
searchHtml, searchHtml,

View File

@@ -1,16 +1,6 @@
import cliProgress from "cli-progress"; import cliProgress from "cli-progress";
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import type { HTMLString } from "../types/common";
import {
type Cookie,
type CookieConfig,
ensureCookies,
formatCookiesForHeader,
parseCookieString,
} from "../utils/cookies";
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http";
/** /**
* Facebook Marketplace Scraper * Facebook Marketplace Scraper
@@ -20,13 +10,23 @@ import { isRecord } from "../utils/http";
* This is by design to respect Facebook's authentication requirements. * This is by design to respect Facebook's authentication requirements.
*/ */
// Facebook cookie configuration // ----------------------------- Types -----------------------------
const FACEBOOK_COOKIE_CONFIG: CookieConfig = {
name: "Facebook", type HTMLString = string;
domain: ".facebook.com",
envVar: "FACEBOOK_COOKIE", interface Cookie {
filePath: "./cookies/facebook.json", name: string;
}; value: string;
domain: string;
path: string;
secure?: boolean;
httpOnly?: boolean;
sameSite?: "strict" | "lax" | "none" | "unspecified";
session?: boolean;
expirationDate?: number;
partitionKey?: Record<string, unknown>;
storeId?: string;
}
interface FacebookAdNode { interface FacebookAdNode {
node: { node: {
@@ -36,8 +36,6 @@ interface FacebookAdNode {
listing_price?: { listing_price?: {
amount?: string | number; amount?: string | number;
currency?: string; currency?: string;
amount_with_offset_in_currency?: string | number;
formatted_amount?: string;
}; };
location?: { location?: {
reverse_geocode?: { reverse_geocode?: {
@@ -47,24 +45,6 @@ interface FacebookAdNode {
}; };
}; };
creation_time?: number; creation_time?: number;
is_sold?: boolean;
is_pending?: boolean;
is_live?: boolean;
is_hidden?: boolean;
primary_listing_photo?: {
image?: {
uri?: string;
};
};
listing_video?: {
id?: string;
};
marketplace_listing_seller?: {
name?: string;
id?: string;
};
marketplace_listing_category_id?: string;
delivery_types?: string[];
[k: string]: unknown; [k: string]: unknown;
}; };
[k: string]: unknown; [k: string]: unknown;
@@ -83,6 +63,11 @@ interface FacebookMarketplaceSearch {
[k: string]: unknown; [k: string]: unknown;
} }
interface FacebookRequireData {
require?: [number, number, number, FacebookMarketplaceSearch, number][];
[k: string]: unknown;
}
interface FacebookMarketplaceItem { interface FacebookMarketplaceItem {
// Basic identification // Basic identification
id: string; id: string;
@@ -171,10 +156,25 @@ interface FacebookMarketplaceItem {
logging_id?: string; logging_id?: string;
reportable_ent_id?: string; reportable_ent_id?: string;
// Related listings (for part-out sellers)
marketplace_listing_sets?: {
edges: Array<{
node: {
canonical_listing: {
id: string;
marketplace_listing_title: string;
is_live: boolean;
is_sold: boolean;
formatted_price: { text: string };
};
};
}>;
};
[k: string]: unknown; [k: string]: unknown;
} }
export interface FacebookListingDetails { type ListingDetails = {
url: string; url: string;
title: string; title: string;
description?: string; description?: string;
@@ -198,28 +198,176 @@ export interface FacebookListingDetails {
}; };
categoryId?: string; categoryId?: string;
deliveryTypes?: string[]; deliveryTypes?: string[];
} };
// ----------------------------- Utilities ----------------------------- // ----------------------------- Utilities -----------------------------
/** function isRecord(value: unknown): value is Record<string, unknown> {
* Parse Facebook cookie string into Cookie array format return typeof value === "object" && value !== null;
* @deprecated Use parseCookieString from utils/cookies instead }
*/
export function parseFacebookCookieString(cookieString: string): Cookie[] { async function delay(ms: number): Promise<void> {
return parseCookieString(cookieString, FACEBOOK_COOKIE_CONFIG.domain); await new Promise((resolve) => setTimeout(resolve, ms));
} }
/** /**
* Load Facebook cookies with priority: URL param > ENV var > file * Load Facebook cookies from file or string
* @param cookiesSource - Optional cookie JSON string from URL parameter (highest priority)
* @param _cookiePath - Deprecated, uses default path from config
*/ */
export async function ensureFacebookCookies( async function loadFacebookCookies(
cookiesSource?: string, cookiesSource?: string,
_cookiePath?: string, cookiePath = "./cookies/facebook.json",
): Promise<Cookie[]> { ): Promise<Cookie[]> {
return ensureCookies(FACEBOOK_COOKIE_CONFIG, cookiesSource); // First try to load from provided string parameter
if (cookiesSource) {
try {
const cookies = JSON.parse(cookiesSource);
if (Array.isArray(cookies)) {
return cookies.filter(
(cookie): cookie is Cookie =>
cookie &&
typeof cookie.name === "string" &&
typeof cookie.value === "string",
);
}
} catch (e) {
throw new Error(`Invalid cookies JSON provided: ${e}`);
}
}
// Try to load from specified path
try {
const cookiesPath = cookiePath;
const file = Bun.file(cookiesPath);
if (await file.exists()) {
const content = await file.text();
const cookies = JSON.parse(content);
if (Array.isArray(cookies)) {
return cookies.filter(
(cookie): cookie is Cookie =>
cookie &&
typeof cookie.name === "string" &&
typeof cookie.value === "string",
);
}
}
} catch (e) {
console.warn(`Could not load cookies from ./cookies/facebook.json: ${e}`);
}
return [];
}
/**
* Parse Facebook cookie string into Cookie array format
*/
function parseFacebookCookieString(cookieString: string): Cookie[] {
if (!cookieString || !cookieString.trim()) {
return [];
}
return cookieString
.split(";")
.map((pair) => pair.trim())
.filter((pair) => pair.includes("="))
.map((pair) => {
const [name, value] = pair.split("=", 2);
const trimmedName = name.trim();
const trimmedValue = value.trim();
// Skip empty names or values
if (!trimmedName || !trimmedValue) {
return null;
}
return {
name: trimmedName,
value: decodeURIComponent(trimmedValue),
domain: ".facebook.com",
path: "/",
secure: true,
httpOnly: false,
sameSite: "lax" as const,
expirationDate: undefined, // Session cookies
};
})
.filter((cookie): cookie is Cookie => cookie !== null);
}
/**
* Ensure Facebook cookies are available, parsing from env var if needed
*/
async function ensureFacebookCookies(
cookiePath = "./cookies/facebook.json",
): Promise<Cookie[]> {
// First try to load existing cookies
try {
const existing = await loadFacebookCookies(undefined, cookiePath);
if (existing.length > 0) {
return existing;
}
} catch (error) {
// File doesn't exist or is invalid, continue to check env var
}
// Try to parse from environment variable
const cookieString = process.env.FACEBOOK_COOKIE;
if (!cookieString || !cookieString.trim()) {
throw new Error(
"No valid Facebook cookies found. Either:\n" +
" 1. Set FACEBOOK_COOKIE environment variable with cookie string, or\n" +
" 2. Create ./cookies/facebook.json manually with cookie array",
);
}
// Parse the cookie string
const cookies = parseFacebookCookieString(cookieString);
if (cookies.length === 0) {
throw new Error(
"FACEBOOK_COOKIE environment variable contains no valid cookies. " +
'Expected format: "name1=value1; name2=value2;"',
);
}
// Save to file for future use
try {
await Bun.write(cookiePath, JSON.stringify(cookies, null, 2));
console.log(`✅ Saved ${cookies.length} Facebook cookies to ${cookiePath}`);
} catch (error) {
console.warn(`! Could not save cookies to ${cookiePath}: ${error}`);
// Continue anyway, we have the cookies in memory
}
return cookies;
}
/**
* Format cookies array into Cookie header string
*/
function formatCookiesForHeader(cookies: Cookie[], domain: string): string {
const validCookies = cookies
.filter((cookie) => {
// Check if cookie applies to this domain
if (cookie.domain.startsWith(".")) {
// Domain cookie (applies to subdomains)
return (
domain.endsWith(cookie.domain.slice(1)) ||
domain === cookie.domain.slice(1)
);
}
// Host-only cookie
return cookie.domain === domain;
})
.filter((cookie) => {
// Check expiration
if (cookie.expirationDate && cookie.expirationDate < Date.now() / 1000) {
return false; // Expired
}
return true;
});
return validCookies
.map((cookie) => `${cookie.name}=${cookie.value}`)
.join("; ");
} }
class HttpError extends Error { class HttpError extends Error {
@@ -233,48 +381,6 @@ class HttpError extends Error {
} }
} }
// ----------------------------- Extraction Metrics -----------------------------
/**
* Monitor API extraction success/failure for detecting changes
*/
const extractionStats = {
totalExtractions: 0,
successfulExtractions: 0,
failedExtractions: 0,
lastApiChangeDetected: null as Date | null,
};
/**
* Log extraction metrics for monitoring API stability
*/
function logExtractionMetrics(success: boolean, itemId?: string) {
extractionStats.totalExtractions++;
if (success) {
extractionStats.successfulExtractions++;
} else {
extractionStats.failedExtractions++;
}
// Log warning if extraction success rate drops below 80%
const successRate =
extractionStats.successfulExtractions / extractionStats.totalExtractions;
if (
extractionStats.totalExtractions > 10 &&
successRate < 0.8 &&
!extractionStats.lastApiChangeDetected
) {
console.warn(
"Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
);
extractionStats.lastApiChangeDetected = new Date();
}
if (!success && itemId) {
console.warn(`Facebook API extraction failed for item ${itemId}`);
}
}
// ----------------------------- HTTP Client ----------------------------- // ----------------------------- HTTP Client -----------------------------
/** /**
@@ -378,7 +484,7 @@ async function fetchHtml(
/** /**
Extract marketplace search data from Facebook page script tags Extract marketplace search data from Facebook page script tags
*/ */
export function extractFacebookMarketplaceData( function extractFacebookMarketplaceData(
htmlString: HTMLString, htmlString: HTMLString,
): FacebookAdNode[] | null { ): FacebookAdNode[] | null {
const { document } = parseHTML(htmlString); const { document } = parseHTML(htmlString);
@@ -425,7 +531,7 @@ export function extractFacebookMarketplaceData(
if ( if (
result && result &&
isRecord(result) && isRecord(result) &&
(result as Record<string, unknown>).feed_units?.edges?.length > 0 result.feed_units?.edges?.length > 0
) { ) {
marketplaceData = result as FacebookMarketplaceSearch; marketplaceData = result as FacebookMarketplaceSearch;
break; break;
@@ -440,13 +546,14 @@ export function extractFacebookMarketplaceData(
if (parsed.marketplace_search && isRecord(parsed.marketplace_search)) { if (parsed.marketplace_search && isRecord(parsed.marketplace_search)) {
const searchData = const searchData =
parsed.marketplace_search as FacebookMarketplaceSearch; parsed.marketplace_search as FacebookMarketplaceSearch;
const feedLength = searchData.feed_units?.edges?.length ?? 0; if (searchData.feed_units?.edges?.length > 0) {
if (feedLength > 0) {
marketplaceData = searchData; marketplaceData = searchData;
break; break;
} }
} }
} catch {} } catch {
// Ignore parsing errors for other scripts
}
} }
if (!marketplaceData?.feed_units?.edges?.length) { if (!marketplaceData?.feed_units?.edges?.length) {
@@ -460,11 +567,78 @@ export function extractFacebookMarketplaceData(
return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node })); return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node }));
} }
/**
* Monitor API extraction success/failure for detecting changes
*/
const extractionStats = {
totalExtractions: 0,
successfulExtractions: 0,
failedExtractions: 0,
lastApiChangeDetected: null as Date | null,
};
/**
* Log extraction metrics for monitoring API stability
*/
function logExtractionMetrics(success: boolean, itemId?: string) {
extractionStats.totalExtractions++;
if (success) {
extractionStats.successfulExtractions++;
} else {
extractionStats.failedExtractions++;
}
// Log warning if extraction success rate drops below 80%
const successRate =
extractionStats.successfulExtractions / extractionStats.totalExtractions;
if (
extractionStats.totalExtractions > 10 &&
successRate < 0.8 &&
!extractionStats.lastApiChangeDetected
) {
console.warn(
"! Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
);
extractionStats.lastApiChangeDetected = new Date();
}
if (success) {
console.log(
`📊 Facebook API extraction stats: ${extractionStats.successfulExtractions}/${extractionStats.totalExtractions} successful`,
);
} else {
console.warn(
`❌ Facebook API extraction failed for item ${itemId || "unknown"}`,
);
}
}
/**
* Turns cents to localized currency string.
*/
function formatCentsToCurrency(
num: number | string | undefined,
locale = "en-US",
): string {
if (num == null) return "";
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
if (Number.isNaN(cents)) return "";
const dollars = cents / 100;
const formatter = new Intl.NumberFormat(locale, {
style: "currency",
currency: "USD",
minimumFractionDigits: 2,
maximumFractionDigits: 2,
useGrouping: true,
});
return formatter.format(dollars);
}
/** /**
Extract marketplace item details from Facebook item page HTML Extract marketplace item details from Facebook item page HTML
Updated for 2026 Facebook Marketplace API structure with multiple extraction paths Updated for 2026 Facebook Marketplace API structure with multiple extraction paths
*/ */
export function extractFacebookItemData( function extractFacebookItemData(
htmlString: HTMLString, htmlString: HTMLString,
): FacebookMarketplaceItem | null { ): FacebookMarketplaceItem | null {
const { document } = parseHTML(htmlString); const { document } = parseHTML(htmlString);
@@ -477,7 +651,7 @@ export function extractFacebookItemData(
try { try {
const parsed = JSON.parse(scriptText); const parsed = JSON.parse(scriptText);
// Check for the require structure with marketplace product details // Check for the 2026 require structure with marketplace product details
if (parsed.require && Array.isArray(parsed.require)) { if (parsed.require && Array.isArray(parsed.require)) {
// Try multiple extraction paths discovered from reverse engineering // Try multiple extraction paths discovered from reverse engineering
const extractionPaths = [ const extractionPaths = [
@@ -533,14 +707,13 @@ export function extractFacebookItemData(
if (depth > maxDepth) return null; // Prevent infinite recursion if (depth > maxDepth) return null; // Prevent infinite recursion
if (isRecord(obj)) { if (isRecord(obj)) {
// Check if this object matches the expected marketplace item structure // Check if this object matches the expected marketplace item structure
const candidate = obj as Record<string, unknown>;
if ( if (
candidate.marketplace_listing_title && obj.marketplace_listing_title &&
candidate.id && obj.id &&
candidate.__typename === "GroupCommerceProductItem" && obj.__typename === "GroupCommerceProductItem" &&
candidate.redacted_description obj.redacted_description
) { ) {
return candidate as unknown as FacebookMarketplaceItem; return obj as FacebookMarketplaceItem;
} }
// Recursively search nested objects and arrays // Recursively search nested objects and arrays
for (const key in obj) { for (const key in obj) {
@@ -591,7 +764,10 @@ export function extractFacebookItemData(
} }
} }
} }
} catch {} } catch (error) {
// Log parsing errors for debugging but continue to next script
console.debug(`Failed to parse script for Facebook item data: ${error}`);
}
} }
return null; return null;
@@ -600,10 +776,8 @@ export function extractFacebookItemData(
/** /**
Parse Facebook marketplace search results into ListingDetails[] Parse Facebook marketplace search results into ListingDetails[]
*/ */
export function parseFacebookAds( function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
ads: FacebookAdNode[], const results: ListingDetails[] = [];
): FacebookListingDetails[] {
const results: FacebookListingDetails[] = [];
for (const adJson of ads) { for (const adJson of ads) {
try { try {
@@ -664,7 +838,7 @@ export function parseFacebookAds(
const address = cityName || null; const address = cityName || null;
// Determine listing status from Facebook flags // Determine listing status from Facebook flags
let listingStatus: string | undefined; let listingStatus: string | undefined = undefined;
if (listing.is_sold) { if (listing.is_sold) {
listingStatus = "SOLD"; listingStatus = "SOLD";
} else if (listing.is_pending) { } else if (listing.is_pending) {
@@ -694,12 +868,12 @@ export function parseFacebookAds(
} }
: undefined; : undefined;
const listingDetails: FacebookListingDetails = { const listingDetails: ListingDetails = {
url, url,
title, title,
listingPrice: { listingPrice: {
amountFormatted: amountFormatted:
priceObj.formatted_amount || formatCentsToCurrency(cents, "en-CA"), priceObj.formatted_amount || formatCentsToCurrency(cents),
cents, cents,
currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD
}, },
@@ -725,9 +899,9 @@ export function parseFacebookAds(
Parse Facebook marketplace item details into ListingDetails format Parse Facebook marketplace item details into ListingDetails format
Updated for 2026 GroupCommerceProductItem structure Updated for 2026 GroupCommerceProductItem structure
*/ */
export function parseFacebookItem( function parseFacebookItem(
item: FacebookMarketplaceItem, item: FacebookMarketplaceItem,
): FacebookListingDetails | null { ): ListingDetails | null {
try { try {
const title = item.marketplace_listing_title || item.custom_title; const title = item.marketplace_listing_title || item.custom_title;
if (!title) return null; if (!title) return null;
@@ -746,7 +920,7 @@ export function parseFacebookItem(
if (!Number.isNaN(amount)) { if (!Number.isNaN(amount)) {
cents = Math.round(amount * 100); cents = Math.round(amount * 100);
amountFormatted = amountFormatted =
item.formatted_price?.text || formatCentsToCurrency(cents, "en-CA"); item.formatted_price?.text || formatCentsToCurrency(cents);
} }
} }
} }
@@ -786,9 +960,12 @@ export function parseFacebookItem(
let listingType = "item"; let listingType = "item";
if (item.vehicle_make_display_name || item.vehicle_odometer_data) { if (item.vehicle_make_display_name || item.vehicle_odometer_data) {
listingType = "vehicle"; listingType = "vehicle";
} else if (item.marketplace_listing_category_id) {
// Could map category IDs to types, but keeping simple for now
listingType = "item";
} }
const listingDetails: FacebookListingDetails = { const listingDetails: ListingDetails = {
url, url,
title, title,
description, description,
@@ -813,6 +990,20 @@ export function parseFacebookItem(
} }
} }
// ----------------------------- Exports for Testing -----------------------------
// Export internal functions for comprehensive testing
export {
extractFacebookItemData,
extractFacebookMarketplaceData,
parseFacebookItem,
parseFacebookAds,
formatCentsToCurrency,
loadFacebookCookies,
formatCookiesForHeader,
parseFacebookCookieString,
ensureFacebookCookies,
};
// ----------------------------- Main ----------------------------- // ----------------------------- Main -----------------------------
export default async function fetchFacebookItems( export default async function fetchFacebookItems(
@@ -823,8 +1014,22 @@ export default async function fetchFacebookItems(
cookiesSource?: string, cookiesSource?: string,
cookiePath?: string, cookiePath?: string,
) { ) {
// Load Facebook cookies with priority: URL param > ENV var > file // Load Facebook cookies - required for Facebook Marketplace access
const cookies = await ensureFacebookCookies(cookiesSource, cookiePath); let cookies: Cookie[];
if (cookiesSource) {
// Use provided cookie source (backward compatibility)
cookies = await loadFacebookCookies(cookiesSource);
} else {
// Auto-load from file or parse from env var
cookies = await ensureFacebookCookies(cookiePath);
}
if (cookies.length === 0) {
throw new Error(
"Facebook cookies are required for marketplace access. " +
"Please provide cookies via 'cookies' parameter or create ./cookies/facebook.json file with valid Facebook session cookies.",
);
}
// Format cookies for HTTP header // Format cookies for HTTP header
const domain = "www.facebook.com"; const domain = "www.facebook.com";
@@ -849,7 +1054,6 @@ export default async function fetchFacebookItems(
let searchHtml: string; let searchHtml: string;
try { try {
searchHtml = await fetchHtml(searchUrl, DELAY_MS, { searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
maxRetries: 3,
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
console.log( console.log(
@@ -910,13 +1114,28 @@ export default async function fetchFacebookItems(
export async function fetchFacebookItem( export async function fetchFacebookItem(
itemId: string, itemId: string,
cookiesSource?: string, cookiesSource?: string,
_cookiePath?: string, cookiePath?: string,
): Promise<FacebookListingDetails | null> { ): Promise<ListingDetails | null> {
// Load Facebook cookies - required for Facebook Marketplace access // Load Facebook cookies - required for Facebook Marketplace access
const cookies = await ensureFacebookCookies(cookiesSource); let cookies: Cookie[];
if (cookiesSource) {
// Use provided cookie source (backward compatibility)
cookies = await loadFacebookCookies(cookiesSource);
} else {
// Auto-load from file or parse from env var
cookies = await ensureFacebookCookies(cookiePath);
}
if (cookies.length === 0) {
throw new Error(
"Facebook cookies are required for marketplace access. " +
"Please provide cookies via 'cookies' parameter or create ./cookies/facebook.json file with valid Facebook session cookies.",
);
}
// Format cookies for HTTP header // Format cookies for HTTP header
const cookiesHeader = formatCookiesForHeader(cookies, "www.facebook.com"); const domain = "www.facebook.com";
const cookiesHeader = formatCookiesForHeader(cookies, domain);
if (!cookiesHeader) { if (!cookiesHeader) {
throw new Error( throw new Error(
"No valid Facebook cookies found. Please check that cookies are not expired and apply to facebook.com domain.", "No valid Facebook cookies found. Please check that cookies are not expired and apply to facebook.com domain.",

215
src/index.ts Normal file
View File

@@ -0,0 +1,215 @@
import fetchEbayItems from "@/ebay";
import fetchFacebookItems from "@/facebook";
import fetchKijijiItems from "@/kijiji";
const PORT = process.env.PORT || 4005;
const server = Bun.serve({
port: PORT,
idleTimeout: 0,
routes: {
// Static routes
"/api/status": new Response("OK"),
// Dynamic routes
"/api/kijiji": async (req: Request) => {
const reqUrl = new URL(req.url);
const SEARCH_QUERY =
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
if (!SEARCH_QUERY)
return Response.json(
{
message:
"Request didn't have 'query' header or 'q' search parameter!",
},
{ status: 400 },
);
// Parse optional parameters with enhanced defaults
const location = reqUrl.searchParams.get("location");
const category = reqUrl.searchParams.get("category");
const maxPagesParam = reqUrl.searchParams.get("maxPages");
const maxPages = maxPagesParam ? Number.parseInt(maxPagesParam, 10) : 5; // Default: 5 pages
const sortBy = reqUrl.searchParams.get("sortBy") as
| "relevancy"
| "date"
| "price"
| "distance"
| undefined;
const sortOrder = reqUrl.searchParams.get("sortOrder") as
| "asc"
| "desc"
| undefined;
// Build search options
const locationValue = location
? /^\d+$/.test(location)
? Number(location)
: location
: 1700272;
const categoryValue = category
? /^\d+$/.test(category)
? Number(category)
: category
: 0;
const searchOptions: import("@/kijiji").SearchOptions = {
location: locationValue,
category: categoryValue,
keywords: SEARCH_QUERY,
sortBy: sortBy || "relevancy",
sortOrder: sortOrder || "desc",
maxPages,
};
// Build listing fetch options with enhanced defaults
const listingOptions: import("@/kijiji").ListingFetchOptions = {
includeImages: true, // Always include full image arrays
sellerDataDepth: "detailed", // Default: detailed seller info
includeClientSideData: false, // GraphQL reviews disabled by default
};
try {
const items = await fetchKijijiItems(
SEARCH_QUERY,
1,
undefined,
searchOptions,
listingOptions,
);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Kijiji scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json(
{
message: `Scraping failed: ${errorMessage}`,
query: SEARCH_QUERY,
options: { searchOptions, listingOptions },
},
{ status: 500 },
);
}
},
"/api/facebook": async (req: Request) => {
const reqUrl = new URL(req.url);
const SEARCH_QUERY =
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
if (!SEARCH_QUERY)
return Response.json(
{
message:
"Request didn't have 'query' header or 'q' search parameter!",
},
{ status: 400 },
);
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
try {
const items = await fetchFacebookItems(
SEARCH_QUERY,
5,
LOCATION,
25,
COOKIES_SOURCE,
"./cookies/facebook.json",
);
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("Facebook scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
},
"/api/ebay": async (req: Request) => {
const reqUrl = new URL(req.url);
const SEARCH_QUERY =
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
if (!SEARCH_QUERY)
return Response.json(
{
message:
"Request didn't have 'query' header or 'q' search parameter!",
},
{ status: 400 },
);
// Parse optional parameters with defaults
const minPriceParam = reqUrl.searchParams.get("minPrice");
const minPrice = minPriceParam
? Number.parseInt(minPriceParam, 10)
: undefined;
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
const maxPrice = maxPriceParam
? Number.parseInt(maxPriceParam, 10)
: undefined;
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
const exclusionsParam = reqUrl.searchParams.get("exclusions");
const exclusions = exclusionsParam
? exclusionsParam.split(",").map((s) => s.trim())
: [];
const keywordsParam = reqUrl.searchParams.get("keywords");
const keywords = keywordsParam
? keywordsParam.split(",").map((s) => s.trim())
: [SEARCH_QUERY];
try {
const items = await fetchEbayItems(SEARCH_QUERY, 5, {
minPrice,
maxPrice,
strictMode,
exclusions,
keywords,
});
if (!items || items.length === 0)
return Response.json(
{ message: "Search didn't return any results!" },
{ status: 404 },
);
return Response.json(items, { status: 200 });
} catch (error) {
console.error("eBay scraping error:", error);
const errorMessage =
error instanceof Error ? error.message : "Unknown error occurred";
return Response.json({ message: errorMessage }, { status: 400 });
}
},
// Wildcard route for all routes that start with "/api/" and aren't otherwise matched
"/api/*": Response.json({ message: "Not found" }, { status: 404 }),
// // Serve a file by buffering it in memory
// "/favicon.ico": new Response(await Bun.file("./favicon.ico").bytes(), {
// headers: {
// "Content-Type": "image/x-icon",
// },
// }),
},
// (optional) fallback for unmatched routes:
// Required if Bun's version < 1.2.3
fetch(req: Request) {
return new Response("Not Found", { status: 404 });
},
});
console.log(`Serving on ${server.hostname}:${server.port}`);

View File

@@ -1,33 +1,14 @@
import cliProgress from "cli-progress"; import cliProgress from "cli-progress";
/* eslint-disable @typescript-eslint/no-explicit-any */
import { parseHTML } from "linkedom"; import { parseHTML } from "linkedom";
import unidecode from "unidecode"; import unidecode from "unidecode";
import type { HTMLString } from "../types/common";
import {
type CookieConfig,
formatCookiesForHeader,
loadCookiesOptional,
} from "../utils/cookies";
import { formatCentsToCurrency } from "../utils/format";
import {
fetchHtml,
HttpError,
isRecord,
NetworkError,
ParseError,
RateLimitError,
ValidationError,
} from "../utils/http";
// Kijiji cookie configuration // const unidecode = require("unidecode");
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
name: "Kijiji",
domain: ".kijiji.ca",
envVar: "KIJIJI_COOKIE",
filePath: "./cookies/kijiji.json",
};
// ----------------------------- Types ----------------------------- // ----------------------------- Types -----------------------------
type HTMLString = string;
type SearchListing = { type SearchListing = {
name: string; name: string;
listingLink: string; listingLink: string;
@@ -68,7 +49,7 @@ interface ApolloListingRoot {
} }
// Keep existing interface for backward compatibility // Keep existing interface for backward compatibility
export interface KijijiListingDetails { type ListingDetails = {
url: string; url: string;
title: string; title: string;
description?: string; description?: string;
@@ -83,10 +64,10 @@ export interface KijijiListingDetails {
endDate?: string; endDate?: string;
numberOfViews?: number; numberOfViews?: number;
address?: string | null; address?: string | null;
} };
// New comprehensive interface for detailed listings // New comprehensive interface for detailed listings
export interface DetailedListing extends KijijiListingDetails { interface DetailedListing extends ListingDetails {
images: string[]; images: string[];
categoryId: number; categoryId: number;
adSource: string; adSource: string;
@@ -114,7 +95,7 @@ export interface DetailedListing extends KijijiListingDetails {
} }
// Configuration interfaces // Configuration interfaces
export interface SearchOptions { interface SearchOptions {
location?: number | string; // Location ID or name location?: number | string; // Location ID or name
category?: number | string; // Category ID or name category?: number | string; // Category ID or name
keywords?: string; keywords?: string;
@@ -123,10 +104,9 @@ export interface SearchOptions {
maxPages?: number; // Default: 5 maxPages?: number; // Default: 5
priceMin?: number; priceMin?: number;
priceMax?: number; priceMax?: number;
cookies?: string; // Optional: Cookie string or JSON (helps bypass bot detection)
} }
export interface ListingFetchOptions { interface ListingFetchOptions {
includeImages?: boolean; // Default: true includeImages?: boolean; // Default: true
sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed' sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed'
includeClientSideData?: boolean; // Default: false includeClientSideData?: boolean; // Default: false
@@ -134,7 +114,7 @@ export interface ListingFetchOptions {
// ----------------------------- Constants & Mappings ----------------------------- // ----------------------------- Constants & Mappings -----------------------------
// Location mappings // Location mappings from KIJIJI.md
const LOCATION_MAPPINGS: Record<string, number> = { const LOCATION_MAPPINGS: Record<string, number> = {
canada: 0, canada: 0,
ontario: 9004, ontario: 9004,
@@ -154,7 +134,7 @@ const LOCATION_MAPPINGS: Record<string, number> = {
"prince edward island": 9011, "prince edward island": 9011,
}; };
// Category mappings (Buy & Sell main categories) // Category mappings from KIJIJI.md (Buy & Sell main categories)
const CATEGORY_MAPPINGS: Record<string, number> = { const CATEGORY_MAPPINGS: Record<string, number> = {
all: 0, all: 0,
"buy-sell": 10, "buy-sell": 10,
@@ -197,6 +177,14 @@ const SORT_MAPPINGS: Record<string, string> = {
distance: "DISTANCE", distance: "DISTANCE",
}; };
// ----------------------------- Exports for Testing -----------------------------
// Note: These are exported for testing purposes only
export { resolveLocationId, resolveCategoryId, buildSearchUrl };
export { extractApolloState, parseSearch };
export { parseDetailedListing };
export { HttpError, NetworkError, ParseError, RateLimitError, ValidationError };
// ----------------------------- Utilities ----------------------------- // ----------------------------- Utilities -----------------------------
const SEPS = new Set([" ", "", "—", "/", ":", ";", ",", ".", "-"]); const SEPS = new Set([" ", "", "—", "/", ":", ";", ",", ".", "-"]);
@@ -204,7 +192,7 @@ const SEPS = new Set([" ", "", "—", "/", ":", ";", ",", ".", "-"]);
/** /**
* Resolve location ID from name or return numeric ID * Resolve location ID from name or return numeric ID
*/ */
export function resolveLocationId(location?: number | string): number { function resolveLocationId(location?: number | string): number {
if (typeof location === "number") return location; if (typeof location === "number") return location;
if (typeof location === "string") { if (typeof location === "string") {
const normalized = location.toLowerCase().replace(/\s+/g, "-"); const normalized = location.toLowerCase().replace(/\s+/g, "-");
@@ -216,7 +204,7 @@ export function resolveLocationId(location?: number | string): number {
/** /**
* Resolve category ID from name or return numeric ID * Resolve category ID from name or return numeric ID
*/ */
export function resolveCategoryId(category?: number | string): number { function resolveCategoryId(category?: number | string): number {
if (typeof category === "number") return category; if (typeof category === "number") return category;
if (typeof category === "string") { if (typeof category === "string") {
const normalized = category.toLowerCase().replace(/\s+/g, "-"); const normalized = category.toLowerCase().replace(/\s+/g, "-");
@@ -228,7 +216,7 @@ export function resolveCategoryId(category?: number | string): number {
/** /**
* Build search URL with enhanced parameters * Build search URL with enhanced parameters
*/ */
export function buildSearchUrl( function buildSearchUrl(
keywords: string, keywords: string,
options: SearchOptions & { page?: number }, options: SearchOptions & { page?: number },
BASE_URL = "https://www.kijiji.ca", BASE_URL = "https://www.kijiji.ca",
@@ -236,8 +224,8 @@ export function buildSearchUrl(
const locationId = resolveLocationId(options.location); const locationId = resolveLocationId(options.location);
const categoryId = resolveCategoryId(options.category); const categoryId = resolveCategoryId(options.category);
const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell"; const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell"; // Could be enhanced
const locationSlug = locationId === 0 ? "canada" : "canada"; const locationSlug = locationId === 0 ? "canada" : "canada"; // Could be enhanced
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`; let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
@@ -254,7 +242,7 @@ export function buildSearchUrl(
} }
/** /**
* Slugifies a string for Kijiji search URLs * Slugifies a string for search
*/ */
export function slugify(input: string): string { export function slugify(input: string): string {
const s = unidecode(input).toLowerCase(); const s = unidecode(input).toLowerCase();
@@ -281,50 +269,211 @@ export function slugify(input: string): string {
return out.join(""); return out.join("");
} }
/**
* Turns cents to localized currency string.
*/
export function formatCentsToCurrency(
num: number | string | undefined,
locale = "en-US",
): string {
if (num == null) return "";
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
if (Number.isNaN(cents)) return "";
const dollars = cents / 100;
const formatter = new Intl.NumberFormat(locale, {
style: "currency",
currency: "USD",
minimumFractionDigits: 2,
maximumFractionDigits: 2,
});
return formatter.format(dollars);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
async function delay(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
// ----------------------------- Error Classes -----------------------------
class HttpError extends Error {
constructor(
message: string,
public readonly status: number,
public readonly url: string,
) {
super(message);
this.name = "HttpError";
}
}
class NetworkError extends Error {
constructor(
message: string,
public readonly url: string,
public readonly cause?: Error,
) {
super(message);
this.name = "NetworkError";
}
}
class ParseError extends Error {
constructor(
message: string,
public readonly data?: unknown,
) {
super(message);
this.name = "ParseError";
}
}
class RateLimitError extends Error {
constructor(
message: string,
public readonly url: string,
public readonly resetTime?: number,
) {
super(message);
this.name = "RateLimitError";
}
}
class ValidationError extends Error {
constructor(message: string) {
super(message);
this.name = "ValidationError";
}
}
// ----------------------------- HTTP Client -----------------------------
/**
Fetch HTML with enhanced retry strategy and exponential backoff.
- Retries on 429, 5xx, and network errors
- Respects X-RateLimit-Reset when present (seconds)
- Exponential backoff with jitter
*/
async function fetchHtml(
url: string,
DELAY_MS: number,
opts?: {
maxRetries?: number;
retryBaseMs?: number;
onRateInfo?: (remaining: string | null, reset: string | null) => void;
},
): Promise<HTMLString> {
const maxRetries = opts?.maxRetries ?? 3;
const retryBaseMs = opts?.retryBaseMs ?? 1000;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 30000); // 30s timeout
const res = await fetch(url, {
method: "GET",
headers: {
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
"cache-control": "no-cache",
"upgrade-insecure-requests": "1",
"user-agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
},
signal: controller.signal,
});
clearTimeout(timeoutId);
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
if (!res.ok) {
// Handle rate limiting
if (res.status === 429) {
const resetSeconds = rateLimitReset
? Number(rateLimitReset)
: Number.NaN;
const waitMs = Number.isFinite(resetSeconds)
? Math.max(0, resetSeconds * 1000)
: calculateBackoffDelay(attempt, retryBaseMs);
if (attempt < maxRetries) {
await delay(waitMs);
continue;
}
throw new RateLimitError(
`Rate limit exceeded for ${url}`,
url,
resetSeconds,
);
}
// Retry on server errors
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
await delay(calculateBackoffDelay(attempt, retryBaseMs));
continue;
}
throw new HttpError(
`Request failed with status ${res.status}`,
res.status,
url,
);
}
const html = await res.text();
// Respect per-request delay to maintain rate limiting
await delay(DELAY_MS);
return html;
} catch (err) {
// Handle different error types
if (err instanceof RateLimitError || err instanceof HttpError) {
throw err; // Re-throw known errors
}
if (err instanceof Error && err.name === "AbortError") {
if (attempt < maxRetries) {
await delay(calculateBackoffDelay(attempt, retryBaseMs));
continue;
}
throw new NetworkError(`Request timeout for ${url}`, url, err);
}
// Network or other errors
if (attempt < maxRetries) {
await delay(calculateBackoffDelay(attempt, retryBaseMs));
continue;
}
throw new NetworkError(
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
url,
err instanceof Error ? err : undefined,
);
}
}
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
}
/**
* Calculate exponential backoff delay with jitter
*/
function calculateBackoffDelay(attempt: number, baseMs: number): number {
const exponentialDelay = baseMs * 2 ** attempt;
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
}
// ----------------------------- GraphQL Client ----------------------------- // ----------------------------- GraphQL Client -----------------------------
// GraphQL response interfaces
interface GraphQLReviewResponse {
user?: {
reviewSummary?: {
count?: number;
score?: number;
};
};
}
interface GraphQLProfileResponse {
user?: {
memberSince?: string;
accountType?: string;
};
}
// GraphQL queries
const GRAPHQL_QUERIES = {
getReviewSummary: `
query GetReviewSummary($userId: String!) {
user(id: $userId) {
reviewSummary {
count
score
__typename
}
__typename
}
}
`,
getProfileMetrics: `
query GetProfileMetrics($profileId: String!) {
user(id: $profileId) {
memberSince
accountType
__typename
}
}
`,
} as const;
/** /**
* Fetch additional data via GraphQL API * Fetch additional data via GraphQL API
*/ */
@@ -378,6 +527,48 @@ async function fetchGraphQLData(
} }
} }
// GraphQL response interfaces
interface GraphQLReviewResponse {
user?: {
reviewSummary?: {
count?: number;
score?: number;
};
};
}
interface GraphQLProfileResponse {
user?: {
memberSince?: string;
accountType?: string;
};
}
// GraphQL queries from KIJIJI.md
const GRAPHQL_QUERIES = {
getReviewSummary: `
query GetReviewSummary($userId: String!) {
user(id: $userId) {
reviewSummary {
count
score
__typename
}
__typename
}
}
`,
getProfileMetrics: `
query GetProfileMetrics($profileId: String!) {
user(id: $profileId) {
memberSince
accountType
__typename
}
}
`,
} as const;
/** /**
* Fetch additional seller data via GraphQL * Fetch additional seller data via GraphQL
*/ */
@@ -428,9 +619,7 @@ async function fetchSellerDetails(
/** /**
Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML. Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML.
*/ */
export function extractApolloState( function extractApolloState(htmlString: HTMLString): ApolloRecord | null {
htmlString: HTMLString,
): ApolloRecord | null {
const { document } = parseHTML(htmlString); const { document } = parseHTML(htmlString);
const nextData = document.getElementById("__NEXT_DATA__"); const nextData = document.getElementById("__NEXT_DATA__");
if (!nextData || !nextData.textContent) return null; if (!nextData || !nextData.textContent) return null;
@@ -448,7 +637,7 @@ export function extractApolloState(
Parse search page apollo state into SearchListing[]. Parse search page apollo state into SearchListing[].
Filters keys likely to be listing entities and ensures url/title exist. Filters keys likely to be listing entities and ensures url/title exist.
*/ */
export function parseSearch( function parseSearch(
htmlString: HTMLString, htmlString: HTMLString,
BASE_URL: string, BASE_URL: string,
): SearchListing[] { ): SearchListing[] {
@@ -475,12 +664,12 @@ export function parseSearch(
} }
/** /**
Parse a listing page into a typed object (backward compatible). Parse a listing page into a typed object.
*/ */
function _parseListing( function parseListing(
htmlString: HTMLString, htmlString: HTMLString,
BASE_URL: string, BASE_URL: string,
): KijijiListingDetails | null { ): ListingDetails | null {
const apolloState = extractApolloState(htmlString); const apolloState = extractApolloState(htmlString);
if (!apolloState) return null; if (!apolloState) return null;
@@ -507,8 +696,7 @@ function _parseListing(
} = root as ApolloListingRoot; } = root as ApolloListingRoot;
const cents = price?.amount != null ? Number(price.amount) : undefined; const cents = price?.amount != null ? Number(price.amount) : undefined;
const amountFormatted = const amountFormatted = formatCentsToCurrency(cents);
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
const numberOfViews = const numberOfViews =
metrics?.views != null ? Number(metrics.views) : undefined; metrics?.views != null ? Number(metrics.views) : undefined;
@@ -549,7 +737,7 @@ function _parseListing(
/** /**
* Parse a listing page into a detailed object with all available fields * Parse a listing page into a detailed object with all available fields
*/ */
export async function parseDetailedListing( async function parseDetailedListing(
htmlString: HTMLString, htmlString: HTMLString,
BASE_URL: string, BASE_URL: string,
options: ListingFetchOptions = {}, options: ListingFetchOptions = {},
@@ -578,6 +766,7 @@ export async function parseDetailedListing(
metrics, metrics,
location, location,
imageUrls, imageUrls,
imageCount,
categoryId, categoryId,
adSource, adSource,
flags, flags,
@@ -586,8 +775,7 @@ export async function parseDetailedListing(
} = root as ApolloListingRoot; } = root as ApolloListingRoot;
const cents = price?.amount != null ? Number(price.amount) : undefined; const cents = price?.amount != null ? Number(price.amount) : undefined;
const amountFormatted = const amountFormatted = formatCentsToCurrency(cents);
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
const numberOfViews = const numberOfViews =
metrics?.views != null ? Number(metrics.views) : undefined; metrics?.views != null ? Number(metrics.views) : undefined;
@@ -645,7 +833,7 @@ export async function parseDetailedListing(
...sellerInfo, ...sellerInfo,
...additionalData, ...additionalData,
}; };
} catch { } catch (err) {
// Silently fail - GraphQL data is optional // Silently fail - GraphQL data is optional
console.warn( console.warn(
`Failed to fetch additional seller data for ${posterInfo.posterId}`, `Failed to fetch additional seller data for ${posterInfo.posterId}`,
@@ -705,16 +893,6 @@ export default async function fetchKijijiItems(
) { ) {
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND)); const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
// Load Kijiji cookies (optional - helps bypass bot detection)
const cookies = await loadCookiesOptional(
KIJIJI_COOKIE_CONFIG,
searchOptions.cookies,
);
const cookieHeader =
cookies.length > 0
? formatCookiesForHeader(cookies, "www.kijiji.ca")
: undefined;
// Set defaults for configuration // Set defaults for configuration
const finalSearchOptions: Required<SearchOptions> = { const finalSearchOptions: Required<SearchOptions> = {
location: searchOptions.location ?? 1700272, // Default to GTA location: searchOptions.location ?? 1700272, // Default to GTA
@@ -723,9 +901,8 @@ export default async function fetchKijijiItems(
sortBy: searchOptions.sortBy ?? "relevancy", sortBy: searchOptions.sortBy ?? "relevancy",
sortOrder: searchOptions.sortOrder ?? "desc", sortOrder: searchOptions.sortOrder ?? "desc",
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
priceMin: searchOptions.priceMin as number, priceMin: searchOptions.priceMin,
priceMax: searchOptions.priceMax as number, priceMax: searchOptions.priceMax,
cookies: searchOptions.cookies ?? "",
}; };
const finalListingOptions: Required<ListingFetchOptions> = { const finalListingOptions: Required<ListingFetchOptions> = {
@@ -758,7 +935,6 @@ export default async function fetchKijijiItems(
); );
} }
}, },
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
}); });
const searchResults = parseSearch(searchHtml, BASE_URL); const searchResults = parseSearch(searchHtml, BASE_URL);
@@ -782,25 +958,18 @@ export default async function fetchKijijiItems(
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`, `\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
); );
// Fetch details for this page's listings with controlled concurrency // Fetch details for this page's listings
const isTTY = process.stdout?.isTTY ?? false; const progressBar = new cliProgress.SingleBar(
const progressBar = isTTY {},
? new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic) cliProgress.Presets.shades_classic,
: null; );
const totalProgress = newListingLinks.length; const totalProgress = newListingLinks.length;
let currentProgress = 0; let currentProgress = 0;
progressBar?.start(totalProgress, currentProgress); progressBar.start(totalProgress, currentProgress);
// Process in batches for controlled concurrency for (const link of newListingLinks) {
const CONCURRENT_REQUESTS = REQUESTS_PER_SECOND * 2; // 2x rate for faster processing
const results: (DetailedListing | null)[] = [];
for (let i = 0; i < newListingLinks.length; i += CONCURRENT_REQUESTS) {
const batch = newListingLinks.slice(i, i + CONCURRENT_REQUESTS);
const batchPromises = batch.map(async (link) => {
try { try {
const html = await fetchHtml(link, 0, { const html = await fetchHtml(link, DELAY_MS, {
// No per-request delay, batch handles rate limit
onRateInfo: (remaining, reset) => { onRateInfo: (remaining, reset) => {
if (remaining && reset) { if (remaining && reset) {
console.log( console.log(
@@ -808,50 +977,32 @@ export default async function fetchKijijiItems(
); );
} }
}, },
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
}); });
const parsed = await parseDetailedListing( const parsed = await parseDetailedListing(
html, html,
BASE_URL, BASE_URL,
finalListingOptions, finalListingOptions,
); );
return parsed; if (parsed) {
allListings.push(parsed);
}
} catch (err) { } catch (err) {
if (err instanceof HttpError) { if (err instanceof HttpError) {
console.error( console.error(
`\nFailed to fetch ${link}\n - ${err.statusCode} ${err.message}`, `\nFailed to fetch ${link}\n - ${err.status} ${err.message}`,
); );
} else { } else {
console.error( console.error(
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`, `\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
); );
} }
return null;
} finally { } finally {
currentProgress++; currentProgress++;
progressBar?.update(currentProgress); progressBar.update(currentProgress);
if (!progressBar) {
console.log(`Progress: ${currentProgress}/${totalProgress}`);
}
}
});
const batchResults = await Promise.all(batchPromises);
results.push(...batchResults);
// Wait between batches to respect rate limit
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
await new Promise((resolve) =>
setTimeout(resolve, DELAY_MS * batch.length),
);
} }
} }
allListings.push( progressBar.stop();
...results.filter((r): r is DetailedListing => r !== null),
);
progressBar?.stop();
// If we got fewer results than expected (40 per page), we've reached the end // If we got fewer results than expected (40 per page), we've reached the end
if (searchResults.length < 40) { if (searchResults.length < 40) {
@@ -862,6 +1013,3 @@ export default async function fetchKijijiItems(
console.log(`\nParsed ${allListings.length} detailed listings.`); console.log(`\nParsed ${allListings.length} detailed listings.`);
return allListings; return allListings;
} }
// Re-export error classes for convenience
export { HttpError, NetworkError, ParseError, RateLimitError, ValidationError };

View File

@@ -5,10 +5,11 @@ import {
fetchFacebookItem, fetchFacebookItem,
formatCentsToCurrency, formatCentsToCurrency,
formatCookiesForHeader, formatCookiesForHeader,
loadFacebookCookies,
parseFacebookAds, parseFacebookAds,
parseFacebookCookieString, parseFacebookCookieString,
parseFacebookItem, parseFacebookItem,
} from "../src/scrapers/facebook"; } from "../src/facebook";
// Mock fetch globally // Mock fetch globally
const originalFetch = global.fetch; const originalFetch = global.fetch;
@@ -182,7 +183,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
}); });
}); });
const _result = await fetchFacebookItem("123", mockCookies); const result = await fetchFacebookItem("123", mockCookies);
expect(attempts).toBe(2); expect(attempts).toBe(2);
// Should eventually succeed after retry // Should eventually succeed after retry
}); });

View File

@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import { fetchFacebookItems } from "../src/scrapers/facebook"; import fetchFacebookItems, { fetchFacebookItem } from "../src/facebook";
// Mock fetch globally // Mock fetch globally
const originalFetch = global.fetch; const originalFetch = global.fetch;

View File

@@ -1,13 +1,14 @@
import { describe, expect, test } from "bun:test"; import { describe, expect, test } from "bun:test";
import { import {
buildSearchUrl, HttpError,
NetworkError, NetworkError,
ParseError, ParseError,
RateLimitError, RateLimitError,
ValidationError,
buildSearchUrl,
resolveCategoryId, resolveCategoryId,
resolveLocationId, resolveLocationId,
ValidationError, } from "../src/kijiji";
} from "../src/scrapers/kijiji";
describe("Location and Category Resolution", () => { describe("Location and Category Resolution", () => {
describe("resolveLocationId", () => { describe("resolveLocationId", () => {
@@ -120,6 +121,14 @@ describe("URL Construction", () => {
}); });
describe("Error Classes", () => { describe("Error Classes", () => {
test("HttpError should store status and URL", () => {
const error = new HttpError("Not found", 404, "https://example.com");
expect(error.message).toBe("Not found");
expect(error.status).toBe(404);
expect(error.url).toBe("https://example.com");
expect(error.name).toBe("HttpError");
});
test("NetworkError should store URL and cause", () => { test("NetworkError should store URL and cause", () => {
const cause = new Error("Connection failed"); const cause = new Error("Connection failed");
const error = new NetworkError( const error = new NetworkError(

View File

@@ -3,7 +3,7 @@ import {
extractApolloState, extractApolloState,
parseDetailedListing, parseDetailedListing,
parseSearch, parseSearch,
} from "../src/scrapers/kijiji"; } from "../src/kijiji";
// Mock fetch globally // Mock fetch globally
const originalFetch = global.fetch; const originalFetch = global.fetch;

View File

@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test"; import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { formatCentsToCurrency, slugify } from "../src/scrapers/kijiji"; import { formatCentsToCurrency, slugify } from "../src/kijiji";
describe("Utility Functions", () => { describe("Utility Functions", () => {
describe("slugify", () => { describe("slugify", () => {

View File

@@ -1,4 +1,7 @@
// Test setup for Bun test runner // Test setup for Bun test runner
import { expect } from "bun:test";
// Global test setup
// This file is loaded before any tests run due to bunfig.toml preload // This file is loaded before any tests run due to bunfig.toml preload
// Mock fetch globally for tests // Mock fetch globally for tests

31
tsconfig.json Normal file
View File

@@ -0,0 +1,31 @@
{
"compilerOptions": {
// Environment setup & latest features
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleDetection": "force",
"jsx": "react-jsx",
"allowJs": true,
// Bundler mode
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"noEmit": true,
// Best practices
"strict": true,
"skipLibCheck": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedIndexedAccess": true,
"noImplicitAny": true,
// Some stricter flags (disabled by default)
"noUnusedLocals": false,
"noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false,
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
}