Compare commits
34 Commits
main
...
f944d319c2
| Author | SHA1 | Date | |
|---|---|---|---|
| f944d319c2 | |||
| cf9784a565 | |||
| df0c528535 | |||
| 2f97d3eafd | |||
| 65eb8d1724 | |||
| f3839aba54 | |||
| 90b98bfb09 | |||
| eb6705df0f | |||
| 72525609ed | |||
| 8b0a65860c | |||
| f9b1c7e096 | |||
| 9edc74cbeb | |||
| ee0fca826d | |||
| f7372612fb | |||
| bce126664e | |||
| 8cbf11538e | |||
| 79f47fdaef | |||
| de5069bf2b | |||
| 637f1a4e75 | |||
| 441ff436c4 | |||
| 1f53ec912a | |||
| 053efd815b | |||
| d619fa5d77 | |||
| 050fd0adba | |||
| 7b106c91ce | |||
| 6e0487f8f3 | |||
| da23ca1c3f | |||
| c35aae4c95 | |||
| 02162c02f5 | |||
| 50d56201af | |||
| 497c7995a2 | |||
| 083b862552 | |||
| 0a32094e93 | |||
| a66b5b2362 |
181
.dockerignore
181
.dockerignore
@@ -1,145 +1,84 @@
|
|||||||
# Dependencies
|
# =============================================================================
|
||||||
|
# Dependencies & Build Output
|
||||||
|
# =============================================================================
|
||||||
node_modules/
|
node_modules/
|
||||||
npm-debug.log*
|
dist/
|
||||||
yarn-debug.log*
|
out/
|
||||||
yarn-error.log*
|
|
||||||
bun.sum
|
|
||||||
|
|
||||||
# Runtime data
|
|
||||||
pids
|
|
||||||
*.pid
|
|
||||||
*.seed
|
|
||||||
*.pid.lock
|
|
||||||
|
|
||||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
|
||||||
lib-cov
|
|
||||||
|
|
||||||
# Coverage directory used by tools like istanbul
|
|
||||||
coverage/
|
|
||||||
*.lcov
|
|
||||||
|
|
||||||
# nyc test coverage
|
|
||||||
.nyc_output
|
|
||||||
|
|
||||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
|
||||||
.grunt
|
|
||||||
|
|
||||||
# Bower dependency directory (https://bower.io/)
|
|
||||||
bower_components
|
|
||||||
|
|
||||||
# node-waf configuration
|
|
||||||
.lock-wscript
|
|
||||||
|
|
||||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
|
||||||
build/Release
|
|
||||||
|
|
||||||
# Dependency directories
|
|
||||||
jspm_packages/
|
|
||||||
|
|
||||||
# TypeScript cache
|
|
||||||
*.tsbuildinfo
|
|
||||||
|
|
||||||
# Optional npm cache directory
|
|
||||||
.npm
|
|
||||||
|
|
||||||
# Optional eslint cache
|
|
||||||
.eslintcache
|
|
||||||
|
|
||||||
# Microbundle cache
|
|
||||||
.rpt2_cache/
|
|
||||||
.rts2_cache_cjs/
|
|
||||||
.rts2_cache_es/
|
|
||||||
.rts2_cache_umd/
|
|
||||||
|
|
||||||
# Optional REPL history
|
|
||||||
.node_repl_history
|
|
||||||
|
|
||||||
# Output of 'npm pack'
|
|
||||||
*.tgz
|
*.tgz
|
||||||
|
|
||||||
# Yarn Integrity file
|
# =============================================================================
|
||||||
.yarn-integrity
|
# Sensitive Files
|
||||||
|
# =============================================================================
|
||||||
# dotenv environment variables file
|
|
||||||
.env
|
.env
|
||||||
.env.local
|
.env.*
|
||||||
.env.development.local
|
.envrc
|
||||||
.env.test.local
|
cookies/
|
||||||
.env.production.local
|
*.pem
|
||||||
|
*.key
|
||||||
|
*.cert
|
||||||
|
*secret*
|
||||||
|
*credential*
|
||||||
|
|
||||||
# parcel-bundler cache (https://parceljs.org/)
|
# =============================================================================
|
||||||
.cache
|
# Development Tools & Config
|
||||||
.parcel-cache
|
# =============================================================================
|
||||||
|
# Nix/Devenv
|
||||||
|
.devenv/
|
||||||
|
.devenv.flake.nix
|
||||||
|
devenv.*
|
||||||
|
.direnv/
|
||||||
|
|
||||||
# Next.js build output
|
# Linting/Formatting
|
||||||
.next
|
biome.json
|
||||||
|
.eslintcache
|
||||||
|
.pre-commit-config.yaml
|
||||||
|
|
||||||
# Nuxt.js build / generate output
|
# IDE/Editor
|
||||||
.nuxt
|
|
||||||
dist
|
|
||||||
|
|
||||||
# Gatsby files
|
|
||||||
.cache/
|
|
||||||
public
|
|
||||||
|
|
||||||
# Vuepress build output
|
|
||||||
.vuepress/dist
|
|
||||||
|
|
||||||
# Serverless directories
|
|
||||||
.serverless/
|
|
||||||
|
|
||||||
# FuseBox cache
|
|
||||||
.fusebox/
|
|
||||||
|
|
||||||
# DynamoDB Local files
|
|
||||||
.dynamodb/
|
|
||||||
|
|
||||||
# TernJS port file
|
|
||||||
.tern-port
|
|
||||||
|
|
||||||
# Stores VSCode versions used for testing VSCode extensions
|
|
||||||
.vscode-test
|
|
||||||
|
|
||||||
# IDE and editor files
|
|
||||||
.vscode/
|
.vscode/
|
||||||
.idea/
|
.idea/
|
||||||
*.swp
|
*.swp
|
||||||
*.swo
|
*.swo
|
||||||
*~
|
*~
|
||||||
|
|
||||||
# OS generated files
|
# AI Assistant Config
|
||||||
.DS_Store
|
.claude/
|
||||||
.DS_Store?
|
CLAUDE.md
|
||||||
._*
|
AGENTS.md
|
||||||
.Spotlight-V100
|
opencode.jsonc
|
||||||
.Trashes
|
|
||||||
ehthumbs.db
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# Git
|
# =============================================================================
|
||||||
.git
|
# Documentation (not needed at runtime)
|
||||||
|
# =============================================================================
|
||||||
|
README.md
|
||||||
|
*.md
|
||||||
|
docs/
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Git & Docker (avoid recursive inclusion)
|
||||||
|
# =============================================================================
|
||||||
|
.git/
|
||||||
.gitignore
|
.gitignore
|
||||||
|
|
||||||
# Docker
|
|
||||||
Dockerfile*
|
Dockerfile*
|
||||||
.dockerignore
|
.dockerignore
|
||||||
|
|
||||||
# Documentation
|
# =============================================================================
|
||||||
README.md
|
# Testing & Coverage
|
||||||
docs/
|
# =============================================================================
|
||||||
|
|
||||||
# Test files
|
|
||||||
test/
|
test/
|
||||||
tests/
|
tests/
|
||||||
*.test.js
|
|
||||||
*.test.ts
|
*.test.ts
|
||||||
*.spec.js
|
|
||||||
*.spec.ts
|
*.spec.ts
|
||||||
|
coverage/
|
||||||
|
*.lcov
|
||||||
|
.nyc_output/
|
||||||
|
|
||||||
# Development files
|
# =============================================================================
|
||||||
CLAUDE.md
|
# OS & Misc
|
||||||
devenv.*
|
# =============================================================================
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
*.log
|
*.log
|
||||||
|
*.pid
|
||||||
# Runtime cookies/config
|
.cache/
|
||||||
cookies/
|
examples/
|
||||||
|
scripts/
|
||||||
|
|||||||
157
AGENTS.md
157
AGENTS.md
@@ -1,9 +1,3 @@
|
|||||||
# AGENTS.md
|
|
||||||
|
|
||||||
This file provides guidance to coding agents when working with code in this repository.
|
|
||||||
|
|
||||||
The project uses TypeScript with path mapping (`@/*` to `src/*`). Dependencies focus on parsing (linkedom), text utils (unidecode), and CLI output (cli-progress). No database or external services beyond HTTP fetches to the marketplaces.
|
|
||||||
|
|
||||||
PRIORITIZE COMMUNICATION STYLE ABOVE ALL ELSE
|
PRIORITIZE COMMUNICATION STYLE ABOVE ALL ELSE
|
||||||
|
|
||||||
## Communication Style
|
## Communication Style
|
||||||
@@ -31,3 +25,154 @@ Examples of constructive pushback:
|
|||||||
- "That adds unnecessary complexity. We can achieve the same with..."
|
- "That adds unnecessary complexity. We can achieve the same with..."
|
||||||
|
|
||||||
This ensures: Better solutions through technical merit, not agreement | Learning through understanding tradeoffs | Avoiding over-engineering | Maintaining code quality
|
This ensures: Better solutions through technical merit, not agreement | Learning through understanding tradeoffs | Avoiding over-engineering | Maintaining code quality
|
||||||
|
|
||||||
|
## Project Structure
|
||||||
|
|
||||||
|
This is a **monorepo** with three packages:
|
||||||
|
|
||||||
|
```
|
||||||
|
packages/
|
||||||
|
├── core/ # Shared scraper logic (Kijiji, Facebook, eBay)
|
||||||
|
├── api-server/ # HTTP REST API server
|
||||||
|
└── mcp-server/ # MCP server for AI agent integration
|
||||||
|
```
|
||||||
|
|
||||||
|
## Common Commands
|
||||||
|
|
||||||
|
**Root level:**
|
||||||
|
- `bun ci`: Run Biome linting
|
||||||
|
|
||||||
|
**API Server (`packages/api-server/`):**
|
||||||
|
- `bun start`: Run the API server
|
||||||
|
- `bun dev`: Run with hot reloading
|
||||||
|
- `bun build`: Build to `dist/api/`
|
||||||
|
|
||||||
|
**MCP Server (`packages/mcp-server/`):**
|
||||||
|
- `bun start`: Run the MCP server
|
||||||
|
- `bun dev`: Run with hot reloading
|
||||||
|
- `bun build`: Build to `dist/mcp/`
|
||||||
|
|
||||||
|
## Code Architecture
|
||||||
|
|
||||||
|
### Core Package (`@marketplace-scrapers/core`)
|
||||||
|
Contains scraper implementations for three marketplaces:
|
||||||
|
|
||||||
|
- **`src/scrapers/kijiji.ts`**: Kijiji Marketplace scraper
|
||||||
|
- Parses Next.js Apollo state (`__APOLLO_STATE__`) from HTML
|
||||||
|
- Supports location/category filtering, sorting, pagination
|
||||||
|
- Fetches individual listing details with seller info
|
||||||
|
- Exports: `fetchKijijiItems()`, type interfaces
|
||||||
|
|
||||||
|
- **`src/scrapers/facebook.ts`**: Facebook Marketplace scraper
|
||||||
|
- Parses nested JSON from script tags (`require/__bbox` structure)
|
||||||
|
- Requires authentication cookies (file or env var `FACEBOOK_COOKIE`)
|
||||||
|
- Exports: `fetchFacebookItems()`, `fetchFacebookItem()`, cookie utilities
|
||||||
|
|
||||||
|
- **`src/scrapers/ebay.ts`**: eBay scraper
|
||||||
|
- DOM-based parsing of search results
|
||||||
|
- Supports Buy It Now filter, Canada-only, price ranges, exclusions
|
||||||
|
- Exports: `fetchEbayItems()`
|
||||||
|
|
||||||
|
- **`src/utils/`**: Shared utilities (HTTP, delay, formatting)
|
||||||
|
- **`src/types/`**: Common type definitions
|
||||||
|
|
||||||
|
### API Server (`@marketplace-scrapers/api-server`)
|
||||||
|
HTTP server using `Bun.serve()` on port 4005 (or `PORT` env var).
|
||||||
|
|
||||||
|
**Routes:**
|
||||||
|
- `GET /api/status` - Health check
|
||||||
|
- `GET /api/kijiji?q={query}` - Search Kijiji
|
||||||
|
- `GET /api/facebook?q={query}&location={location}&cookies={cookies}` - Search Facebook
|
||||||
|
- `GET /api/ebay?q={query}&minPrice=&maxPrice=&strictMode=&exclusions=&keywords=&buyItNowOnly=&canadaOnly=&cookies=` - Search eBay
|
||||||
|
- `GET /api/*` - 404 fallback
|
||||||
|
|
||||||
|
### MCP Server (`@marketplace-scrapers/mcp-server`)
|
||||||
|
MCP JSON-RPC 2.0 server on port 4006 (or `MCP_PORT` env var).
|
||||||
|
|
||||||
|
**Endpoints:**
|
||||||
|
- `GET /.well-known/mcp/server-card.json` - Server discovery metadata
|
||||||
|
- `POST /mcp` - JSON-RPC 2.0 protocol endpoint
|
||||||
|
|
||||||
|
**Tools:**
|
||||||
|
- `search_kijiji` - Search Kijiji (query, maxItems)
|
||||||
|
- `search_facebook` - Search Facebook (query, location, maxItems, cookiesSource)
|
||||||
|
- `search_ebay` - Search eBay (query, minPrice, maxPrice, strictMode, exclusions, keywords, buyItNowOnly, canadaOnly, maxItems, cookies)
|
||||||
|
|
||||||
|
## API Response Formats
|
||||||
|
|
||||||
|
All scrapers return arrays of listing objects with these common fields:
|
||||||
|
- `url`: Full listing URL
|
||||||
|
- `title`: Listing title
|
||||||
|
- `listingPrice`: `{ amountFormatted, cents, currency }`
|
||||||
|
- `address`: Location string (or null)
|
||||||
|
- `listingType`: Type of listing
|
||||||
|
- `listingStatus`: Status (ACTIVE, SOLD, etc.)
|
||||||
|
|
||||||
|
### Kijiji-specific fields
|
||||||
|
`description`, `creationDate`, `endDate`, `numberOfViews`, `images`, `categoryId`, `adSource`, `flags`, `attributes`, `location`, `sellerInfo`
|
||||||
|
|
||||||
|
### Facebook-specific fields
|
||||||
|
`creationDate`, `imageUrl`, `videoUrl`, `seller`, `categoryId`, `deliveryTypes`
|
||||||
|
|
||||||
|
### eBay-specific fields
|
||||||
|
Minimal - mainly the common fields
|
||||||
|
|
||||||
|
## Cookie Management
|
||||||
|
|
||||||
|
Both **Facebook Marketplace** and **eBay** require valid session cookies for reliable scraping.
|
||||||
|
|
||||||
|
### Cookie Priority Hierarchy (High → Low)
|
||||||
|
All scrapers follow this loading order:
|
||||||
|
1. **URL/API Parameter** - Passed directly via `cookies` parameter (highest priority)
|
||||||
|
2. **Environment Variable** - `FACEBOOK_COOKIE` or `EBAY_COOKIE`
|
||||||
|
3. **Cookie File** - `cookies/facebook.json` or `cookies/ebay.json` (fallback)
|
||||||
|
|
||||||
|
### Facebook Cookies
|
||||||
|
- **Required for**: Facebook Marketplace scraping
|
||||||
|
- **Format**: JSON array (see `cookies/README.md`)
|
||||||
|
- **Key cookies**: `c_user`, `xs`, `fr`, `datr`, `sb`
|
||||||
|
|
||||||
|
**Setup:**
|
||||||
|
```bash
|
||||||
|
# Option 1: File (fallback)
|
||||||
|
# Create cookies/facebook.json with cookie array
|
||||||
|
|
||||||
|
# Option 2: Environment variable
|
||||||
|
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request'
|
||||||
|
|
||||||
|
# Option 3: URL parameter (highest priority)
|
||||||
|
curl "http://localhost:4005/api/facebook?q=laptop&cookies=[{...}]"
|
||||||
|
```
|
||||||
|
|
||||||
|
### eBay Cookies
|
||||||
|
- **Required for**: Bypassing bot detection
|
||||||
|
- **Format**: Cookie string `"name=value; name2=value2"`
|
||||||
|
- **Key cookies**: `s`, `ds2`, `ebay`, `dp1`, `nonsession`
|
||||||
|
|
||||||
|
**Setup:**
|
||||||
|
```bash
|
||||||
|
# Option 1: File (fallback)
|
||||||
|
# Create cookies/ebay.json with cookie string
|
||||||
|
|
||||||
|
# Option 2: Environment variable
|
||||||
|
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
|
||||||
|
|
||||||
|
# Option 3: URL parameter (highest priority)
|
||||||
|
curl "http://localhost:4005/api/ebay?q=laptop&cookies=s=VALUE;ds2=VALUE"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Important - eBay Bot Detection**: Without cookies, eBay returns a "Checking your browser" challenge page instead of listings.
|
||||||
|
|
||||||
|
## Technical Details
|
||||||
|
|
||||||
|
- **TypeScript** with path mapping (`@/*` → `src/*`) per package
|
||||||
|
- **Dependencies**: linkedom (parsing), unidecode (text utils), cli-progress (CLI output)
|
||||||
|
- **No database** - stateless HTTP fetches to marketplaces
|
||||||
|
- **Rate limiting**: Respects `X-RateLimit-*` headers, configurable delays
|
||||||
|
|
||||||
|
## Development Notes
|
||||||
|
|
||||||
|
- **Cookie files** are git-ignored for security (see `cookies/README.md`)
|
||||||
|
- Kijiji parses Apollo state from Next.js hydration data
|
||||||
|
- All scrapers handle retries on 429/5xx errors
|
||||||
|
- Cookie priority ensures flexibility across different deployment environments
|
||||||
|
|||||||
129
Dockerfile
129
Dockerfile
@@ -1,32 +1,123 @@
|
|||||||
# Use the official Bun base image
|
# =============================================================================
|
||||||
FROM oven/bun:latest AS base
|
# Stage 1: Dependencies
|
||||||
|
# Install only production dependencies for optimal layer caching
|
||||||
|
# =============================================================================
|
||||||
|
FROM oven/bun:1-slim AS dependencies
|
||||||
|
|
||||||
# Set the working directory
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Copy package files
|
# Copy workspace configuration
|
||||||
COPY package.json bun.lock* ./
|
COPY package.json bun.lock ./
|
||||||
|
|
||||||
# Install dependencies
|
# Copy all package.json files to establish workspace structure
|
||||||
|
COPY packages/core/package.json ./packages/core/
|
||||||
|
COPY packages/api-server/package.json ./packages/api-server/
|
||||||
|
COPY packages/mcp-server/package.json ./packages/mcp-server/
|
||||||
|
|
||||||
|
# Install dependencies with frozen lockfile (production only)
|
||||||
|
RUN bun install --frozen-lockfile --production
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Stage 2: Build
|
||||||
|
# Build both services with minification for production
|
||||||
|
# =============================================================================
|
||||||
|
FROM oven/bun:1-slim AS build
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy workspace configuration
|
||||||
|
COPY package.json bun.lock ./
|
||||||
|
|
||||||
|
# Copy all package.json files
|
||||||
|
COPY packages/core/package.json ./packages/core/
|
||||||
|
COPY packages/api-server/package.json ./packages/api-server/
|
||||||
|
COPY packages/mcp-server/package.json ./packages/mcp-server/
|
||||||
|
|
||||||
|
# Install ALL dependencies (including devDependencies for TypeScript)
|
||||||
RUN bun install --frozen-lockfile
|
RUN bun install --frozen-lockfile
|
||||||
|
|
||||||
# Copy source code
|
# Copy source code for all packages
|
||||||
COPY src ./src
|
COPY packages/core ./packages/core
|
||||||
COPY tsconfig.json ./
|
COPY packages/api-server ./packages/api-server
|
||||||
|
COPY packages/mcp-server ./packages/mcp-server
|
||||||
|
|
||||||
# Build the application for production
|
# Build both services with minification
|
||||||
RUN bun build ./src/index.ts --outdir ./dist --minify --target=bun
|
# Output: dist/api/index.js and dist/mcp/index.js
|
||||||
|
RUN bun build ./packages/api-server/src/index.ts \
|
||||||
|
--target=bun \
|
||||||
|
--outdir=./dist/api \
|
||||||
|
--minify && \
|
||||||
|
bun build ./packages/mcp-server/src/index.ts \
|
||||||
|
--target=bun \
|
||||||
|
--outdir=./dist/mcp \
|
||||||
|
--minify
|
||||||
|
|
||||||
# Multi-stage build - runtime stage
|
# =============================================================================
|
||||||
FROM oven/bun:latest AS runtime
|
# Stage 3: Runtime
|
||||||
|
# Minimal production image with both services
|
||||||
|
# =============================================================================
|
||||||
|
FROM oven/bun:1-slim AS runtime
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Copy the built application from the base stage
|
# Copy production dependencies from dependencies stage
|
||||||
COPY --from=base /app/dist/ ./
|
COPY --from=dependencies /app/node_modules ./node_modules
|
||||||
|
|
||||||
# Expose the port the app runs on
|
# Copy built artifacts from build stage
|
||||||
EXPOSE 3000
|
COPY --from=build /app/dist ./dist
|
||||||
|
|
||||||
# Start the application
|
# Create cookies directory (will be mounted as volume at runtime)
|
||||||
CMD ["bun", "index.js"]
|
# This ensures the directory exists even if volume is not mounted
|
||||||
|
RUN mkdir -p /app/cookies && \
|
||||||
|
chown -R bun:bun /app/cookies
|
||||||
|
|
||||||
|
# Create startup script that runs both services
|
||||||
|
# Uses Bun's built-in capabilities for process management
|
||||||
|
RUN cat > /app/start.sh << 'EOF'
|
||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Trap SIGTERM and SIGINT for graceful shutdown
|
||||||
|
trap 'echo "Received shutdown signal, stopping services..."; kill -TERM $API_PID $MCP_PID 2>/dev/null; wait' TERM INT
|
||||||
|
|
||||||
|
# Start API Server in background
|
||||||
|
echo "Starting API Server on port ${API_PORT:-4005}..."
|
||||||
|
bun /app/dist/api/index.js &
|
||||||
|
API_PID=$!
|
||||||
|
|
||||||
|
# Give API server a moment to initialize
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Start MCP Server in background
|
||||||
|
echo "Starting MCP Server on port ${API_PORT:-4006}..."
|
||||||
|
bun /app/dist/mcp/index.js &
|
||||||
|
MCP_PID=$!
|
||||||
|
|
||||||
|
echo "Both services started successfully"
|
||||||
|
echo "API Server PID: $API_PID"
|
||||||
|
echo "MCP Server PID: $MCP_PID"
|
||||||
|
|
||||||
|
# Wait for both processes
|
||||||
|
wait $API_PID $MCP_PID
|
||||||
|
EOF
|
||||||
|
|
||||||
|
RUN chmod +x /app/start.sh
|
||||||
|
|
||||||
|
# Expose both service ports
|
||||||
|
# API Server: 4005 (default), MCP Server: 4006 (default)
|
||||||
|
EXPOSE 4005 4006
|
||||||
|
|
||||||
|
# Environment variables for port configuration
|
||||||
|
ENV PORT=4005
|
||||||
|
ENV MCP_PORT=4006
|
||||||
|
|
||||||
|
# Volume mount point for cookies
|
||||||
|
# Mount your cookies directory here: -v /path/to/cookies:/app/cookies
|
||||||
|
VOLUME ["/app/cookies"]
|
||||||
|
|
||||||
|
# Health check that verifies both services are responding
|
||||||
|
HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
|
||||||
|
CMD bun -e "Promise.all([fetch('http://localhost:${PORT}/api/status'),fetch('http://localhost:${MCP_PORT}/.well-known/mcp/server-card.json')]).then(r=>process.exit(0)).catch(()=>process.exit(1))"
|
||||||
|
|
||||||
|
# Run the startup script
|
||||||
|
CMD ["/app/start.sh"]
|
||||||
|
|||||||
20
biome.json
20
biome.json
@@ -1,21 +1,17 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
|
"$schema": "https://biomejs.dev/schemas/2.3.11/schema.json",
|
||||||
"vcs": {
|
"vcs": {
|
||||||
"enabled": false,
|
"enabled": true,
|
||||||
"clientKind": "git",
|
"clientKind": "git",
|
||||||
"useIgnoreFile": false
|
"useIgnoreFile": true
|
||||||
},
|
},
|
||||||
"files": {
|
"files": {
|
||||||
"ignoreUnknown": false,
|
"includes": ["**", "!!**/dist"]
|
||||||
"ignore": []
|
|
||||||
},
|
},
|
||||||
"formatter": {
|
"formatter": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"indentStyle": "space"
|
"indentStyle": "space"
|
||||||
},
|
},
|
||||||
"organizeImports": {
|
|
||||||
"enabled": true
|
|
||||||
},
|
|
||||||
"linter": {
|
"linter": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"rules": {
|
"rules": {
|
||||||
@@ -26,5 +22,13 @@
|
|||||||
"formatter": {
|
"formatter": {
|
||||||
"quoteStyle": "double"
|
"quoteStyle": "double"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"assist": {
|
||||||
|
"enabled": true,
|
||||||
|
"actions": {
|
||||||
|
"source": {
|
||||||
|
"organizeImports": "on"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
383
bun.lock
383
bun.lock
@@ -1,17 +1,35 @@
|
|||||||
{
|
{
|
||||||
"lockfileVersion": 1,
|
"lockfileVersion": 1,
|
||||||
"configVersion": 0,
|
"configVersion": 1,
|
||||||
"workspaces": {
|
"workspaces": {
|
||||||
"": {
|
"": {
|
||||||
"name": "sone4ka-tok",
|
"name": "marketplace-scrapers-monorepo",
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "2.3.11",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"packages/api-server": {
|
||||||
|
"name": "@marketplace-scrapers/api-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest",
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"packages/core": {
|
||||||
|
"name": "@marketplace-scrapers/core",
|
||||||
|
"version": "1.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"cli-progress": "^3.12.0",
|
"cli-progress": "^3.12.0",
|
||||||
"linkedom": "^0.18.12",
|
"linkedom": "^0.18.12",
|
||||||
"unidecode": "^1.1.0",
|
"unidecode": "^1.1.0",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@anthropic-ai/claude-code": "^2.0.1",
|
|
||||||
"@musistudio/claude-code-router": "^1.0.53",
|
|
||||||
"@types/bun": "latest",
|
"@types/bun": "latest",
|
||||||
"@types/cli-progress": "^3.11.6",
|
"@types/cli-progress": "^3.11.6",
|
||||||
"@types/unidecode": "^1.1.0",
|
"@types/unidecode": "^1.1.0",
|
||||||
@@ -20,136 +38,67 @@
|
|||||||
"typescript": "^5",
|
"typescript": "^5",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
"packages/mcp-server": {
|
||||||
|
"name": "@marketplace-scrapers/mcp-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest",
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5",
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"packages": {
|
"packages": {
|
||||||
"@anthropic-ai/claude-code": ["@anthropic-ai/claude-code@2.0.1", "", { "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.33.5", "@img/sharp-darwin-x64": "^0.33.5", "@img/sharp-linux-arm": "^0.33.5", "@img/sharp-linux-arm64": "^0.33.5", "@img/sharp-linux-x64": "^0.33.5", "@img/sharp-win32-x64": "^0.33.5" }, "bin": { "claude": "cli.js" } }, "sha512-2SboYcdJ+dsE2K784dbJ4ohVWlAkLZhU7mZG1lebyG6TvGLXLhjc2qTEfCxSeelCjJHhIh/YkNpe06veB4IgBw=="],
|
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="],
|
||||||
|
|
||||||
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.54.0", "", { "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-xyoCtHJnt/qg5GG6IgK+UJEndz8h8ljzt/caKXmq3LfBF81nC/BW6E4x2rOWCZcvsLyVW+e8U5mtIr6UCE/kJw=="],
|
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.3.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-/uXXkBcPKVQY7rc9Ys2CrlirBJYbpESEDme7RKiBD6MmqR2w3j0+ZZXRIL2xiaNPsIMMNhP1YnA+jRRxoOAFrA=="],
|
||||||
|
|
||||||
"@fastify/accept-negotiator": ["@fastify/accept-negotiator@2.0.1", "", {}, "sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ=="],
|
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.3.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-fh7nnvbweDPm2xEmFjfmq7zSUiox88plgdHF9OIW4i99WnXrAC3o2P3ag9judoUMv8FCSUnlwJCM1B64nO5Fbg=="],
|
||||||
|
|
||||||
"@fastify/ajv-compiler": ["@fastify/ajv-compiler@4.0.2", "", { "dependencies": { "ajv": "^8.12.0", "ajv-formats": "^3.0.1", "fast-uri": "^3.0.0" } }, "sha512-Rkiu/8wIjpsf46Rr+Fitd3HRP+VsxUFDDeag0hs9L0ksfnwx2g7SPQQTFL0E8Qv+rfXzQOxBJnjUB9ITUDjfWQ=="],
|
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-l4xkGa9E7Uc0/05qU2lMYfN1H+fzzkHgaJoy98wO+b/7Gl78srbCRRgwYSW+BTLixTBrM6Ede5NSBwt7rd/i6g=="],
|
||||||
|
|
||||||
"@fastify/cors": ["@fastify/cors@11.1.0", "", { "dependencies": { "fastify-plugin": "^5.0.0", "toad-cache": "^3.7.0" } }, "sha512-sUw8ed8wP2SouWZTIbA7V2OQtMNpLj2W6qJOYhNdcmINTu6gsxVYXjQiM9mdi8UUDlcoDDJ/W2syPo1WB2QjYA=="],
|
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.3.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-XPSQ+XIPZMLaZ6zveQdwNjbX+QdROEd1zPgMwD47zvHV+tCGB88VH+aynyGxAHdzL+Tm/+DtKST5SECs4iwCLg=="],
|
||||||
|
|
||||||
"@fastify/error": ["@fastify/error@4.2.0", "", {}, "sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ=="],
|
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-/1s9V/H3cSe0r0Mv/Z8JryF5x9ywRxywomqZVLHAoa/uN0eY7F8gEngWKNS5vbbN/BsfpCG5yeBT5ENh50Frxg=="],
|
||||||
|
|
||||||
"@fastify/fast-json-stringify-compiler": ["@fastify/fast-json-stringify-compiler@5.0.3", "", { "dependencies": { "fast-json-stringify": "^6.0.0" } }, "sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ=="],
|
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.3.11", "", { "os": "linux", "cpu": "x64" }, "sha512-vU7a8wLs5C9yJ4CB8a44r12aXYb8yYgBn+WeyzbMjaCMklzCv1oXr8x+VEyWodgJt9bDmhiaW/I0RHbn7rsNmw=="],
|
||||||
|
|
||||||
"@fastify/forwarded": ["@fastify/forwarded@3.0.1", "", {}, "sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw=="],
|
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.3.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-PZQ6ElCOnkYapSsysiTy0+fYX+agXPlWugh6+eQ6uPKI3vKAqNp6TnMhoM3oY2NltSB89hz59o8xIfOdyhi9Iw=="],
|
||||||
|
|
||||||
"@fastify/merge-json-schemas": ["@fastify/merge-json-schemas@0.2.1", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A=="],
|
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.3.11", "", { "os": "win32", "cpu": "x64" }, "sha512-43VrG813EW+b5+YbDbz31uUsheX+qFKCpXeY9kfdAx+ww3naKxeVkTD9zLIWxUPfJquANMHrmW3wbe/037G0Qg=="],
|
||||||
|
|
||||||
"@fastify/proxy-addr": ["@fastify/proxy-addr@5.1.0", "", { "dependencies": { "@fastify/forwarded": "^3.0.0", "ipaddr.js": "^2.1.0" } }, "sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw=="],
|
"@marketplace-scrapers/api-server": ["@marketplace-scrapers/api-server@workspace:packages/api-server"],
|
||||||
|
|
||||||
"@fastify/send": ["@fastify/send@4.1.0", "", { "dependencies": { "@lukeed/ms": "^2.0.2", "escape-html": "~1.0.3", "fast-decode-uri-component": "^1.0.1", "http-errors": "^2.0.0", "mime": "^3" } }, "sha512-TMYeQLCBSy2TOFmV95hQWkiTYgC/SEx7vMdV+wnZVX4tt8VBLKzmH8vV9OzJehV0+XBfg+WxPMt5wp+JBUKsVw=="],
|
"@marketplace-scrapers/core": ["@marketplace-scrapers/core@workspace:packages/core"],
|
||||||
|
|
||||||
"@fastify/static": ["@fastify/static@8.2.0", "", { "dependencies": { "@fastify/accept-negotiator": "^2.0.0", "@fastify/send": "^4.0.0", "content-disposition": "^0.5.4", "fastify-plugin": "^5.0.0", "fastq": "^1.17.1", "glob": "^11.0.0" } }, "sha512-PejC/DtT7p1yo3p+W7LiUtLMsV8fEvxAK15sozHy9t8kwo5r0uLYmhV/inURmGz1SkHZFz/8CNtHLPyhKcx4SQ=="],
|
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"],
|
||||||
|
|
||||||
"@google/genai": ["@google/genai@1.21.0", "", { "dependencies": { "google-auth-library": "^9.14.2", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.11.4" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-k47DECR8BF9z7IJxQd3reKuH2eUnOH5NlJWSe+CKM6nbXx+wH3hmtWQxUQR9M8gzWW1EvFuRVgjQssEIreNZsw=="],
|
"@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="],
|
||||||
|
|
||||||
"@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="],
|
|
||||||
|
|
||||||
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="],
|
|
||||||
|
|
||||||
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.0.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg=="],
|
|
||||||
|
|
||||||
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.0.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ=="],
|
|
||||||
|
|
||||||
"@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.0.5", "", { "os": "linux", "cpu": "arm" }, "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g=="],
|
|
||||||
|
|
||||||
"@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA=="],
|
|
||||||
|
|
||||||
"@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw=="],
|
|
||||||
|
|
||||||
"@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.0.5" }, "os": "linux", "cpu": "arm" }, "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ=="],
|
|
||||||
|
|
||||||
"@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.0.4" }, "os": "linux", "cpu": "arm64" }, "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA=="],
|
|
||||||
|
|
||||||
"@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.0.4" }, "os": "linux", "cpu": "x64" }, "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA=="],
|
|
||||||
|
|
||||||
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.33.5", "", { "os": "win32", "cpu": "x64" }, "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg=="],
|
|
||||||
|
|
||||||
"@isaacs/balanced-match": ["@isaacs/balanced-match@4.0.1", "", {}, "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ=="],
|
|
||||||
|
|
||||||
"@isaacs/brace-expansion": ["@isaacs/brace-expansion@5.0.0", "", { "dependencies": { "@isaacs/balanced-match": "^4.0.1" } }, "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA=="],
|
|
||||||
|
|
||||||
"@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="],
|
|
||||||
|
|
||||||
"@lukeed/ms": ["@lukeed/ms@2.0.2", "", {}, "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA=="],
|
|
||||||
|
|
||||||
"@musistudio/claude-code-router": ["@musistudio/claude-code-router@1.0.53", "", { "dependencies": { "@fastify/static": "^8.2.0", "@musistudio/llms": "^1.0.35", "dotenv": "^16.4.7", "find-process": "^2.0.0", "json5": "^2.2.3", "openurl": "^1.1.1", "rotating-file-stream": "^3.2.7", "shell-quote": "^1.8.3", "tiktoken": "^1.0.21", "uuid": "^11.1.0" }, "bin": { "ccr": "dist/cli.js" } }, "sha512-cNH3dOJu2ECUXHdTbuEyXq7sD12+ie4wqPD85mKz7yg6Xo1HmpFqQQvh4XAhQDBJAWZob6Fuavu+m5f2BwFT/g=="],
|
|
||||||
|
|
||||||
"@musistudio/llms": ["@musistudio/llms@1.0.35", "", { "dependencies": { "@anthropic-ai/sdk": "^0.54.0", "@fastify/cors": "^11.0.1", "@google/genai": "^1.7.0", "dotenv": "^16.5.0", "fastify": "^5.4.0", "google-auth-library": "^10.1.0", "json5": "^2.2.3", "jsonrepair": "^3.13.0", "openai": "^5.6.0", "undici": "^7.10.0", "uuid": "^11.1.0" } }, "sha512-fW7DCHrhzMNtQiaXlAAivSsn+4+vqOYWAURi1OfwESijRDfJk4Gpi0rhedI9o4e0ucr7ftVRO707sOeo/+TJNA=="],
|
|
||||||
|
|
||||||
"@types/bun": ["@types/bun@1.2.19", "", { "dependencies": { "bun-types": "1.2.19" } }, "sha512-d9ZCmrH3CJ2uYKXQIUuZ/pUnTqIvLDS0SK7pFmbx8ma+ziH/FRMoAq5bYpRG7y+w1gl+HgyNZbtqgMq4W4e2Lg=="],
|
|
||||||
|
|
||||||
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
|
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
|
||||||
|
|
||||||
"@types/node": ["@types/node@24.1.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w=="],
|
"@types/node": ["@types/node@25.0.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-gWEkeiyYE4vqjON/+Obqcoeffmk0NF15WSBwSs7zwVA2bAbTaE0SJ7P0WNGoJn8uE7fiaV5a7dKYIJriEqOrmA=="],
|
||||||
|
|
||||||
"@types/react": ["@types/react@19.1.9", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-WmdoynAX8Stew/36uTSVMcLJJ1KRh6L3IZRx1PZ7qJtBqT3dYTgyDTx8H1qoRghErydW7xw9mSJ3wS//tCRpFA=="],
|
|
||||||
|
|
||||||
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
||||||
|
|
||||||
"abstract-logging": ["abstract-logging@2.0.1", "", {}, "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA=="],
|
|
||||||
|
|
||||||
"agent-base": ["agent-base@7.1.4", "", {}, "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ=="],
|
|
||||||
|
|
||||||
"ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="],
|
|
||||||
|
|
||||||
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
|
||||||
|
|
||||||
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
|
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
|
||||||
|
|
||||||
"ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="],
|
|
||||||
|
|
||||||
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
|
|
||||||
|
|
||||||
"avvio": ["avvio@9.1.0", "", { "dependencies": { "@fastify/error": "^4.0.0", "fastq": "^1.17.1" } }, "sha512-fYASnYi600CsH/j9EQov7lECAniYiBFiiAtBNuZYLA2leLe9qOvZzqYHFjtIj6gD2VMoMLP14834LFWvr4IfDw=="],
|
|
||||||
|
|
||||||
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
|
|
||||||
|
|
||||||
"bignumber.js": ["bignumber.js@9.3.1", "", {}, "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ=="],
|
|
||||||
|
|
||||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||||
|
|
||||||
"buffer-equal-constant-time": ["buffer-equal-constant-time@1.0.1", "", {}, "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="],
|
"bun-types": ["bun-types@1.3.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-5ua817+BZPZOlNaRgGBpZJOSAQ9RQ17pkwPD0yR7CfJg+r8DgIILByFifDTa+IPDDxzf5VNhtNlcKqFzDgJvlQ=="],
|
||||||
|
|
||||||
"bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="],
|
|
||||||
|
|
||||||
"chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
|
|
||||||
|
|
||||||
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
||||||
|
|
||||||
"color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="],
|
|
||||||
|
|
||||||
"color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="],
|
|
||||||
|
|
||||||
"commander": ["commander@12.1.0", "", {}, "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA=="],
|
|
||||||
|
|
||||||
"content-disposition": ["content-disposition@0.5.4", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ=="],
|
|
||||||
|
|
||||||
"cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="],
|
|
||||||
|
|
||||||
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
|
||||||
|
|
||||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||||
|
|
||||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||||
|
|
||||||
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
"cssom": ["cssom@0.5.0", "", {}, "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="],
|
||||||
|
|
||||||
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
|
|
||||||
|
|
||||||
"data-uri-to-buffer": ["data-uri-to-buffer@4.0.1", "", {}, "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="],
|
|
||||||
|
|
||||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
|
||||||
|
|
||||||
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
|
|
||||||
|
|
||||||
"dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="],
|
|
||||||
|
|
||||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||||
|
|
||||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||||
@@ -158,260 +107,32 @@
|
|||||||
|
|
||||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||||
|
|
||||||
"dotenv": ["dotenv@16.6.1", "", {}, "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow=="],
|
|
||||||
|
|
||||||
"eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
|
|
||||||
|
|
||||||
"ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="],
|
|
||||||
|
|
||||||
"emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
|
"emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="],
|
||||||
|
|
||||||
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
"entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="],
|
||||||
|
|
||||||
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
|
|
||||||
|
|
||||||
"extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
|
|
||||||
|
|
||||||
"fast-decode-uri-component": ["fast-decode-uri-component@1.0.1", "", {}, "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg=="],
|
|
||||||
|
|
||||||
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
|
|
||||||
|
|
||||||
"fast-json-stringify": ["fast-json-stringify@6.1.1", "", { "dependencies": { "@fastify/merge-json-schemas": "^0.2.0", "ajv": "^8.12.0", "ajv-formats": "^3.0.1", "fast-uri": "^3.0.0", "json-schema-ref-resolver": "^3.0.0", "rfdc": "^1.2.0" } }, "sha512-DbgptncYEXZqDUOEl4krff4mUiVrTZZVI7BBrQR/T3BqMj/eM1flTC1Uk2uUoLcWCxjT95xKulV/Lc6hhOZsBQ=="],
|
|
||||||
|
|
||||||
"fast-querystring": ["fast-querystring@1.1.2", "", { "dependencies": { "fast-decode-uri-component": "^1.0.1" } }, "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg=="],
|
|
||||||
|
|
||||||
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
|
|
||||||
|
|
||||||
"fastify": ["fastify@5.6.1", "", { "dependencies": { "@fastify/ajv-compiler": "^4.0.0", "@fastify/error": "^4.0.0", "@fastify/fast-json-stringify-compiler": "^5.0.0", "@fastify/proxy-addr": "^5.0.0", "abstract-logging": "^2.0.1", "avvio": "^9.0.0", "fast-json-stringify": "^6.0.0", "find-my-way": "^9.0.0", "light-my-request": "^6.0.0", "pino": "^9.0.0", "process-warning": "^5.0.0", "rfdc": "^1.3.1", "secure-json-parse": "^4.0.0", "semver": "^7.6.0", "toad-cache": "^3.7.0" } }, "sha512-WjjlOciBF0K8pDUPZoGPhqhKrQJ02I8DKaDIfO51EL0kbSMwQFl85cRwhOvmSDWoukNOdTo27gLN549pLCcH7Q=="],
|
|
||||||
|
|
||||||
"fastify-plugin": ["fastify-plugin@5.1.0", "", {}, "sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw=="],
|
|
||||||
|
|
||||||
"fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="],
|
|
||||||
|
|
||||||
"fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
|
|
||||||
|
|
||||||
"find-my-way": ["find-my-way@9.3.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-querystring": "^1.0.0", "safe-regex2": "^5.0.0" } }, "sha512-eRoFWQw+Yv2tuYlK2pjFS2jGXSxSppAs3hSQjfxVKxM5amECzIgYYc1FEI8ZmhSh/Ig+FrKEz43NLRKJjYCZVg=="],
|
|
||||||
|
|
||||||
"find-process": ["find-process@2.0.0", "", { "dependencies": { "chalk": "~4.1.2", "commander": "^12.1.0", "loglevel": "^1.9.2" }, "bin": { "find-process": "dist/bin/find-process.js" } }, "sha512-YUBQnteWGASJoEVVsOXy6XtKAY2O1FCsWnnvQ8y0YwgY1rZiKeVptnFvMu6RSELZAJOGklqseTnUGGs5D0bKmg=="],
|
|
||||||
|
|
||||||
"foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="],
|
|
||||||
|
|
||||||
"formdata-polyfill": ["formdata-polyfill@4.0.10", "", { "dependencies": { "fetch-blob": "^3.1.2" } }, "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g=="],
|
|
||||||
|
|
||||||
"gaxios": ["gaxios@7.1.2", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "node-fetch": "^3.3.2" } }, "sha512-/Szrn8nr+2TsQT1Gp8iIe/BEytJmbyfrbFh419DfGQSkEgNEhbPi7JRJuughjkTzPWgU9gBQf5AVu3DbHt0OXA=="],
|
|
||||||
|
|
||||||
"gcp-metadata": ["gcp-metadata@7.0.1", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ=="],
|
|
||||||
|
|
||||||
"glob": ["glob@11.0.3", "", { "dependencies": { "foreground-child": "^3.3.1", "jackspeak": "^4.1.1", "minimatch": "^10.0.3", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA=="],
|
|
||||||
|
|
||||||
"google-auth-library": ["google-auth-library@10.4.0", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.0.0", "gcp-metadata": "^7.0.0", "google-logging-utils": "^1.0.0", "gtoken": "^8.0.0", "jws": "^4.0.0" } }, "sha512-CmIrSy1bqMQUsPmA9+hcSbAXL80cFhu40cGMUjCaLpNKVzzvi+0uAHq8GNZxkoGYIsTX4ZQ7e4aInAqWxgn4fg=="],
|
|
||||||
|
|
||||||
"google-logging-utils": ["google-logging-utils@1.1.1", "", {}, "sha512-rcX58I7nqpu4mbKztFeOAObbomBbHU2oIb/d3tJfF3dizGSApqtSwYJigGCooHdnMyQBIw8BrWyK96w3YXgr6A=="],
|
|
||||||
|
|
||||||
"gtoken": ["gtoken@8.0.0", "", { "dependencies": { "gaxios": "^7.0.0", "jws": "^4.0.0" } }, "sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw=="],
|
|
||||||
|
|
||||||
"has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="],
|
|
||||||
|
|
||||||
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
"html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="],
|
||||||
|
|
||||||
"htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
|
"htmlparser2": ["htmlparser2@10.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.2.1", "entities": "^6.0.0" } }, "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g=="],
|
||||||
|
|
||||||
"http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
|
|
||||||
|
|
||||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
|
||||||
|
|
||||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
|
||||||
|
|
||||||
"ipaddr.js": ["ipaddr.js@2.2.0", "", {}, "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA=="],
|
|
||||||
|
|
||||||
"is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
|
"is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
|
||||||
|
|
||||||
"is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="],
|
|
||||||
|
|
||||||
"isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
|
|
||||||
|
|
||||||
"jackspeak": ["jackspeak@4.1.1", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" } }, "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ=="],
|
|
||||||
|
|
||||||
"json-bigint": ["json-bigint@1.0.0", "", { "dependencies": { "bignumber.js": "^9.0.0" } }, "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ=="],
|
|
||||||
|
|
||||||
"json-schema-ref-resolver": ["json-schema-ref-resolver@3.0.0", "", { "dependencies": { "dequal": "^2.0.3" } }, "sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A=="],
|
|
||||||
|
|
||||||
"json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="],
|
|
||||||
|
|
||||||
"json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="],
|
|
||||||
|
|
||||||
"jsonrepair": ["jsonrepair@3.13.1", "", { "bin": { "jsonrepair": "bin/cli.js" } }, "sha512-WJeiE0jGfxYmtLwBTEk8+y/mYcaleyLXWaqp5bJu0/ZTSeG0KQq/wWQ8pmnkKenEdN6pdnn6QtcoSUkbqDHWNw=="],
|
|
||||||
|
|
||||||
"jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="],
|
|
||||||
|
|
||||||
"jws": ["jws@4.0.0", "", { "dependencies": { "jwa": "^2.0.0", "safe-buffer": "^5.0.1" } }, "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg=="],
|
|
||||||
|
|
||||||
"light-my-request": ["light-my-request@6.6.0", "", { "dependencies": { "cookie": "^1.0.1", "process-warning": "^4.0.0", "set-cookie-parser": "^2.6.0" } }, "sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A=="],
|
|
||||||
|
|
||||||
"linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="],
|
"linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="],
|
||||||
|
|
||||||
"loglevel": ["loglevel@1.9.2", "", {}, "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg=="],
|
|
||||||
|
|
||||||
"lru-cache": ["lru-cache@11.2.2", "", {}, "sha512-F9ODfyqML2coTIsQpSkRHnLSZMtkU8Q+mSfcaIyKwy58u+8k5nvAYeiNhsyMARvzNcXJ9QfWVrcPsC9e9rAxtg=="],
|
|
||||||
|
|
||||||
"mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="],
|
|
||||||
|
|
||||||
"minimatch": ["minimatch@10.0.3", "", { "dependencies": { "@isaacs/brace-expansion": "^5.0.0" } }, "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw=="],
|
|
||||||
|
|
||||||
"minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="],
|
|
||||||
|
|
||||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
|
||||||
|
|
||||||
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
|
|
||||||
|
|
||||||
"node-fetch": ["node-fetch@3.3.2", "", { "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", "formdata-polyfill": "^4.0.10" } }, "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA=="],
|
|
||||||
|
|
||||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||||
|
|
||||||
"on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="],
|
|
||||||
|
|
||||||
"openai": ["openai@5.23.2", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.23.8" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg=="],
|
|
||||||
|
|
||||||
"openurl": ["openurl@1.1.1", "", {}, "sha512-d/gTkTb1i1GKz5k3XE3XFV/PxQ1k45zDqGP2OA7YhgsaLoqm6qRvARAZOFer1fcXritWlGBRCu/UgeS4HAnXAA=="],
|
|
||||||
|
|
||||||
"package-json-from-dist": ["package-json-from-dist@1.0.1", "", {}, "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="],
|
|
||||||
|
|
||||||
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
|
|
||||||
|
|
||||||
"path-scurry": ["path-scurry@2.0.0", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg=="],
|
|
||||||
|
|
||||||
"pino": ["pino@9.12.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pino-std-serializers": "^7.0.0", "process-warning": "^5.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "slow-redact": "^0.3.0", "sonic-boom": "^4.0.1", "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "sha512-0Gd0OezGvqtqMwgYxpL7P0pSHHzTJ0Lx992h+mNlMtRVfNnqweWmf0JmRWk5gJzHalyd2mxTzKjhiNbGS2Ztfw=="],
|
|
||||||
|
|
||||||
"pino-abstract-transport": ["pino-abstract-transport@2.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw=="],
|
|
||||||
|
|
||||||
"pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="],
|
|
||||||
|
|
||||||
"process-warning": ["process-warning@5.0.0", "", {}, "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA=="],
|
|
||||||
|
|
||||||
"quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="],
|
|
||||||
|
|
||||||
"real-require": ["real-require@0.2.0", "", {}, "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="],
|
|
||||||
|
|
||||||
"require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
|
|
||||||
|
|
||||||
"ret": ["ret@0.5.0", "", {}, "sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw=="],
|
|
||||||
|
|
||||||
"reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="],
|
|
||||||
|
|
||||||
"rfdc": ["rfdc@1.4.1", "", {}, "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA=="],
|
|
||||||
|
|
||||||
"rotating-file-stream": ["rotating-file-stream@3.2.7", "", {}, "sha512-SVquhBEVvRFY+nWLUc791Y0MIlyZrEClRZwZFLLRgJKldHyV1z4e2e/dp9LPqCS3AM//uq/c3PnOFgjqnm5P+A=="],
|
|
||||||
|
|
||||||
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
|
|
||||||
|
|
||||||
"safe-regex2": ["safe-regex2@5.0.0", "", { "dependencies": { "ret": "~0.5.0" } }, "sha512-YwJwe5a51WlK7KbOJREPdjNrpViQBI3p4T50lfwPuDhZnE3XGVTlGvi+aolc5+RvxDD6bnUmjVsU9n1eboLUYw=="],
|
|
||||||
|
|
||||||
"safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="],
|
|
||||||
|
|
||||||
"secure-json-parse": ["secure-json-parse@4.0.0", "", {}, "sha512-dxtLJO6sc35jWidmLxo7ij+Eg48PM/kleBsxpC8QJE0qJICe+KawkDQmvCMZUr9u7WKVHgMW6vy3fQ7zMiFZMA=="],
|
|
||||||
|
|
||||||
"semver": ["semver@7.7.2", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA=="],
|
|
||||||
|
|
||||||
"set-cookie-parser": ["set-cookie-parser@2.7.1", "", {}, "sha512-IOc8uWeOZgnb3ptbCURJWNjWUPcO3ZnTTdzsurqERrP6nPyv+paC55vJM0LpOlT2ne+Ix+9+CRG1MNLlyZ4GjQ=="],
|
|
||||||
|
|
||||||
"setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
|
|
||||||
|
|
||||||
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
|
||||||
|
|
||||||
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
|
||||||
|
|
||||||
"shell-quote": ["shell-quote@1.8.3", "", {}, "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw=="],
|
|
||||||
|
|
||||||
"signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
|
|
||||||
|
|
||||||
"slow-redact": ["slow-redact@0.3.0", "", {}, "sha512-cf723wn9JeRIYP9tdtd86GuqoR5937u64Io+CYjlm2i7jvu7g0H+Cp0l0ShAf/4ZL+ISUTVT+8Qzz7RZmp9FjA=="],
|
|
||||||
|
|
||||||
"sonic-boom": ["sonic-boom@4.2.0", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww=="],
|
|
||||||
|
|
||||||
"split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
|
|
||||||
|
|
||||||
"statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="],
|
|
||||||
|
|
||||||
"string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
|
"string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
|
||||||
|
|
||||||
"string-width-cjs": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
|
|
||||||
|
|
||||||
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
||||||
|
|
||||||
"strip-ansi-cjs": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||||
|
|
||||||
"supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="],
|
|
||||||
|
|
||||||
"thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="],
|
|
||||||
|
|
||||||
"tiktoken": ["tiktoken@1.0.22", "", {}, "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA=="],
|
|
||||||
|
|
||||||
"toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="],
|
|
||||||
|
|
||||||
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
|
|
||||||
|
|
||||||
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
|
|
||||||
|
|
||||||
"typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
|
|
||||||
|
|
||||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||||
|
|
||||||
"undici": ["undici@7.16.0", "", {}, "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g=="],
|
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||||
|
|
||||||
"undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="],
|
|
||||||
|
|
||||||
"unidecode": ["unidecode@1.1.0", "", {}, "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA=="],
|
"unidecode": ["unidecode@1.1.0", "", {}, "sha512-GIp57N6DVVJi8dpeIU6/leJGdv7W65ZSXFLFiNmxvexXkc0nXdqUvhA/qL9KqBKsILxMwg5MnmYNOIDJLb5JVA=="],
|
||||||
|
|
||||||
"uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="],
|
|
||||||
|
|
||||||
"web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="],
|
|
||||||
|
|
||||||
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
|
|
||||||
|
|
||||||
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
|
|
||||||
|
|
||||||
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
|
||||||
|
|
||||||
"wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="],
|
|
||||||
|
|
||||||
"wrap-ansi-cjs": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="],
|
|
||||||
|
|
||||||
"ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library": ["google-auth-library@9.15.1", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^6.1.1", "gcp-metadata": "^6.1.0", "gtoken": "^7.0.0", "jws": "^4.0.0" } }, "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng=="],
|
|
||||||
|
|
||||||
"@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
|
|
||||||
|
|
||||||
"@isaacs/cliui/strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="],
|
|
||||||
|
|
||||||
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||||
|
|
||||||
"light-my-request/process-warning": ["process-warning@4.0.1", "", {}, "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q=="],
|
|
||||||
|
|
||||||
"wrap-ansi/ansi-styles": ["ansi-styles@6.2.3", "", {}, "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg=="],
|
|
||||||
|
|
||||||
"wrap-ansi/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
|
|
||||||
|
|
||||||
"wrap-ansi/strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library/gaxios": ["gaxios@6.7.1", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", "node-fetch": "^2.6.9", "uuid": "^9.0.1" } }, "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library/gcp-metadata": ["gcp-metadata@6.1.1", "", { "dependencies": { "gaxios": "^6.1.1", "google-logging-utils": "^0.0.2", "json-bigint": "^1.0.0" } }, "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library/gtoken": ["gtoken@7.1.0", "", { "dependencies": { "gaxios": "^6.0.0", "jws": "^4.0.0" } }, "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw=="],
|
|
||||||
|
|
||||||
"@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
|
|
||||||
|
|
||||||
"@isaacs/cliui/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
|
|
||||||
|
|
||||||
"wrap-ansi/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="],
|
|
||||||
|
|
||||||
"wrap-ansi/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library/gaxios/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library/gaxios/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
|
|
||||||
|
|
||||||
"@google/genai/google-auth-library/gcp-metadata/google-logging-utils": ["google-logging-utils@0.0.2", "", {}, "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ=="],
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
[test]
|
|
||||||
# Test configuration
|
|
||||||
preload = ["./test/setup.ts"]
|
|
||||||
@@ -1,24 +1,33 @@
|
|||||||
# Facebook Marketplace Cookies Setup
|
# Marketplace Cookies Setup
|
||||||
|
|
||||||
To use the Facebook Marketplace scraper, you need to provide valid Facebook session cookies.
|
Both Facebook Marketplace and eBay require valid session cookies to bypass bot detection and access listings.
|
||||||
|
|
||||||
## Option 1: Cookies File (`facebook.json`)
|
## Cookie Priority Hierarchy
|
||||||
|
|
||||||
1. Log into Facebook in your browser
|
All scrapers follow this priority order (highest to lowest):
|
||||||
2. Open Developer Tools → Network tab
|
1. **URL Parameter** - Passed directly in API/MCP request (overrides all)
|
||||||
3. Visit facebook.com/marketplace (ensure you're logged in)
|
2. **Environment Variable** - Set as `FACEBOOK_COOKIE` or `EBAY_COOKIE`
|
||||||
4. Look for any marketplace-related requests in the Network tab
|
3. **Cookie File** - Stored in `facebook.json` or `ebay.json` (fallback)
|
||||||
5. Export cookies from the browser's Application/Storage → Cookies section
|
|
||||||
6. Save the cookies as a JSON array to `facebook.json`
|
|
||||||
|
|
||||||
The `facebook.json` file should contain Facebook session cookies, particularly:
|
---
|
||||||
|
|
||||||
|
## Facebook Marketplace (`facebook.json`)
|
||||||
|
|
||||||
|
### Required Cookies
|
||||||
- `c_user`: Your Facebook user ID
|
- `c_user`: Your Facebook user ID
|
||||||
- `xs`: Facebook session token
|
- `xs`: Facebook session token
|
||||||
- `fr`: Facebook request token
|
- `fr`: Facebook request token
|
||||||
- `datr`: Data attribution token
|
- `datr`: Data attribution token
|
||||||
- `sb`: Session browser token
|
- `sb`: Session browser token
|
||||||
|
|
||||||
Example structure:
|
### Setup Methods
|
||||||
|
|
||||||
|
**Method 1: Cookie File (Lowest Priority)**
|
||||||
|
1. Log into Facebook in your browser
|
||||||
|
2. Open Developer Tools → Application/Storage → Cookies
|
||||||
|
3. Export cookies as JSON array to `facebook.json`
|
||||||
|
|
||||||
|
Example `facebook.json`:
|
||||||
```json
|
```json
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
@@ -27,26 +36,59 @@ Example structure:
|
|||||||
"domain": ".facebook.com",
|
"domain": ".facebook.com",
|
||||||
"path": "/",
|
"path": "/",
|
||||||
"secure": true
|
"secure": true
|
||||||
},
|
}
|
||||||
// ... other cookies
|
|
||||||
]
|
]
|
||||||
```
|
```
|
||||||
|
|
||||||
## Option 2: URL Parameter
|
**Method 2: Environment Variable**
|
||||||
|
```bash
|
||||||
You can pass cookies directly via the `cookies` URL parameter:
|
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request'
|
||||||
|
|
||||||
```
|
```
|
||||||
GET /api/facebook?q=laptop&cookies=[{"name":"c_user","value":"123","domain":".facebook.com",...}]
|
|
||||||
|
**Method 3: URL Parameter (Highest Priority)**
|
||||||
```
|
```
|
||||||
|
GET /api/facebook?q=laptop&cookies=[{"name":"c_user","value":"123",...}]
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## eBay (`ebay.json`)
|
||||||
|
|
||||||
|
eBay has aggressive bot detection that blocks requests without valid session cookies.
|
||||||
|
|
||||||
|
### Setup Methods
|
||||||
|
|
||||||
|
**Method 1: Cookie File (Lowest Priority)**
|
||||||
|
1. Log into eBay in your browser
|
||||||
|
2. Open Developer Tools → Network tab
|
||||||
|
3. Visit ebay.ca and inspect any request headers
|
||||||
|
4. Copy the full `Cookie` header value
|
||||||
|
5. Save as plain text to `ebay.json` (see `ebay.json.example`)
|
||||||
|
|
||||||
|
Example `ebay.json`:
|
||||||
|
```
|
||||||
|
s=VALUE; ds2=VALUE; ebay=VALUE; dp1=VALUE; nonsession=VALUE
|
||||||
|
```
|
||||||
|
|
||||||
|
**Method 2: Environment Variable**
|
||||||
|
```bash
|
||||||
|
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Method 3: URL Parameter (Highest Priority)**
|
||||||
|
```
|
||||||
|
GET /api/ebay?q=laptop&cookies=s=VALUE;ds2=VALUE;ebay=VALUE
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Important Notes
|
## Important Notes
|
||||||
|
|
||||||
- Cookies must be from an active Facebook session
|
- Cookies must be from active browser sessions
|
||||||
- Cookies expire, so you may need to refresh them periodically
|
- Cookies expire and need periodic refresh
|
||||||
- Never share real cookies or commit them to version control
|
- **NEVER** commit real cookies to version control
|
||||||
- Facebook may block automated scraping even with valid cookies
|
- Platforms may still block automated scraping despite valid cookies
|
||||||
|
|
||||||
## Security
|
## Security
|
||||||
|
|
||||||
The cookies file is intentionally left out of version control for security reasons.</content>
|
All `*.json` files in this directory are git-ignored for security.</content>
|
||||||
|
|||||||
1
cookies/ebay.json.example
Normal file
1
cookies/ebay.json.example
Normal file
@@ -0,0 +1 @@
|
|||||||
|
s=YOUR_VALUE; ds2=YOUR_VALUE; ebay=YOUR_VALUE; dp1=YOUR_VALUE; nonsession=YOUR_VALUE
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
services:
|
|
||||||
ca-marketplace-scraper:
|
|
||||||
container_name: ca-marketplace-scraper
|
|
||||||
build: .
|
|
||||||
ports:
|
|
||||||
- "4005:4005"
|
|
||||||
environment:
|
|
||||||
- NODE_ENV=production
|
|
||||||
- PORT=4005
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:4005/api/status"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 3
|
|
||||||
start_period: 5s
|
|
||||||
restart: unless-stopped
|
|
||||||
networks:
|
|
||||||
- internal
|
|
||||||
networks:
|
|
||||||
internal:
|
|
||||||
driver: bridge
|
|
||||||
name: ca-marketplace-scraper-network
|
|
||||||
@@ -1,27 +1,9 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://opencode.ai/config.json",
|
"$schema": "https://opencode.ai/config.json",
|
||||||
"mcp": {
|
"mcp": {
|
||||||
"chrome-devtools": {
|
"marketplace-scrape": {
|
||||||
"type": "local",
|
|
||||||
"command": [
|
|
||||||
"bunx",
|
|
||||||
"--bun",
|
|
||||||
"chrome-devtools-mcp@latest",
|
|
||||||
"--log-file",
|
|
||||||
"./debug.log",
|
|
||||||
"--headless=false",
|
|
||||||
"--isolated=false",
|
|
||||||
"-e",
|
|
||||||
"/nix/store/lz8ajxhnkkw2llj752bdz41wqr645h9c-google-chrome-dev-146.0.7635.0/bin/google-chrome-unstable",
|
|
||||||
"--ignore-default-chrome-arg='--disable-extensions'"
|
|
||||||
],
|
|
||||||
"enabled": false
|
|
||||||
},
|
|
||||||
"bun-docs": {
|
|
||||||
"type": "remote",
|
"type": "remote",
|
||||||
"url": "https://bun.com/docs/mcp",
|
"url": "http://localhost:4006/mcp"
|
||||||
"timeout": 3000,
|
|
||||||
"enabled": false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
33
package.json
33
package.json
@@ -1,26 +1,21 @@
|
|||||||
{
|
{
|
||||||
"name": "ca-marketplace-scraper",
|
"name": "marketplace-scrapers-monorepo",
|
||||||
"module": "./src/index.ts",
|
"version": "1.0.0",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "bun ./src/index.ts",
|
"ci": "biome ci",
|
||||||
"dev": "bun --watch ./src/index.ts",
|
"clean": "rm -rf dist",
|
||||||
"build": "bun build ./src/index.ts"
|
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
||||||
|
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
||||||
|
"build:all": "bun run build:api && bun run build:mcp",
|
||||||
|
"build": "bun run clean && bun run build:all",
|
||||||
|
"start": "./scripts/start.sh"
|
||||||
},
|
},
|
||||||
"type": "module",
|
|
||||||
"private": true,
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"workspaces": [
|
||||||
|
"packages/*"
|
||||||
|
],
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@anthropic-ai/claude-code": "^2.0.1",
|
"@biomejs/biome": "2.3.11"
|
||||||
"@musistudio/claude-code-router": "^1.0.53",
|
|
||||||
"@types/bun": "latest",
|
|
||||||
"@types/unidecode": "^1.1.0",
|
|
||||||
"@types/cli-progress": "^3.11.6"
|
|
||||||
},
|
|
||||||
"peerDependencies": {
|
|
||||||
"typescript": "^5"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"cli-progress": "^3.12.0",
|
|
||||||
"linkedom": "^0.18.12",
|
|
||||||
"unidecode": "^1.1.0"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
21
packages/api-server/package.json
Normal file
21
packages/api-server/package.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/api-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"module": "./src/index.ts",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun ./src/index.ts",
|
||||||
|
"dev": "bun --watch ./src/index.ts",
|
||||||
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
30
packages/api-server/src/index.ts
Normal file
30
packages/api-server/src/index.ts
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
import { ebayRoute } from "./routes/ebay";
|
||||||
|
import { facebookRoute } from "./routes/facebook";
|
||||||
|
import { kijijiRoute } from "./routes/kijiji";
|
||||||
|
import { statusRoute } from "./routes/status";
|
||||||
|
|
||||||
|
const PORT = process.env.PORT || 4005;
|
||||||
|
|
||||||
|
const server = Bun.serve({
|
||||||
|
port: PORT as number | string,
|
||||||
|
idleTimeout: 0,
|
||||||
|
routes: {
|
||||||
|
// Health check endpoint
|
||||||
|
"/api/status": statusRoute,
|
||||||
|
|
||||||
|
// Marketplace search endpoints
|
||||||
|
"/api/kijiji": kijijiRoute,
|
||||||
|
"/api/facebook": facebookRoute,
|
||||||
|
"/api/ebay": ebayRoute,
|
||||||
|
|
||||||
|
// Fallback for unmatched /api routes
|
||||||
|
"/api/*": Response.json({ message: "Not found" }, { status: 404 }),
|
||||||
|
},
|
||||||
|
|
||||||
|
// Fallback for all other routes
|
||||||
|
fetch(_req: Request) {
|
||||||
|
return new Response("Not Found", { status: 404 });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`API Server running on ${server.hostname}:${server.port}`);
|
||||||
68
packages/api-server/src/routes/ebay.ts
Normal file
68
packages/api-server/src/routes/ebay.ts
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
import { fetchEbayItems } from "@marketplace-scrapers/core";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}&cookies={cookies}
|
||||||
|
* Search eBay for listings (default: Buy It Now only, Canada only)
|
||||||
|
* Optional: Pass cookies parameter to bypass bot detection
|
||||||
|
*/
|
||||||
|
export async function ebayRoute(req: Request): Promise<Response> {
|
||||||
|
try {
|
||||||
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
|
const SEARCH_QUERY =
|
||||||
|
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
||||||
|
if (!SEARCH_QUERY)
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message:
|
||||||
|
"Request didn't have 'query' header or 'q' search parameter!",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const minPriceParam = reqUrl.searchParams.get("minPrice");
|
||||||
|
const minPrice = minPriceParam ? parseInt(minPriceParam, 10) : undefined;
|
||||||
|
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
|
||||||
|
const maxPrice = maxPriceParam ? parseInt(maxPriceParam, 10) : undefined;
|
||||||
|
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
||||||
|
const buyItNowOnly = reqUrl.searchParams.get("buyItNowOnly") !== "false";
|
||||||
|
const canadaOnly = reqUrl.searchParams.get("canadaOnly") !== "false";
|
||||||
|
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
||||||
|
const exclusions = exclusionsParam
|
||||||
|
? exclusionsParam.split(",").map((s) => s.trim())
|
||||||
|
: [];
|
||||||
|
const keywordsParam = reqUrl.searchParams.get("keywords");
|
||||||
|
const keywords = keywordsParam
|
||||||
|
? keywordsParam.split(",").map((s) => s.trim())
|
||||||
|
: [SEARCH_QUERY];
|
||||||
|
|
||||||
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : undefined;
|
||||||
|
const cookies = reqUrl.searchParams.get("cookies") || undefined;
|
||||||
|
|
||||||
|
const items = await fetchEbayItems(SEARCH_QUERY, 1, {
|
||||||
|
minPrice,
|
||||||
|
maxPrice,
|
||||||
|
strictMode,
|
||||||
|
exclusions,
|
||||||
|
keywords,
|
||||||
|
buyItNowOnly,
|
||||||
|
canadaOnly,
|
||||||
|
cookies,
|
||||||
|
});
|
||||||
|
|
||||||
|
const results = maxItems ? items.slice(0, maxItems) : items;
|
||||||
|
|
||||||
|
if (!results || results.length === 0)
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
return Response.json(results, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("eBay scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
}
|
||||||
|
}
|
||||||
46
packages/api-server/src/routes/facebook.ts
Normal file
46
packages/api-server/src/routes/facebook.ts
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
import { fetchFacebookItems } from "@marketplace-scrapers/core";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/facebook?q={query}&location={location}&cookies={cookies}
|
||||||
|
* Search Facebook Marketplace for listings
|
||||||
|
*/
|
||||||
|
export async function facebookRoute(req: Request): Promise<Response> {
|
||||||
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
|
const SEARCH_QUERY =
|
||||||
|
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
||||||
|
if (!SEARCH_QUERY)
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message: "Request didn't have 'query' header or 'q' search parameter!",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
|
||||||
|
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
||||||
|
const maxItemsParam = reqUrl.searchParams.get("maxItems");
|
||||||
|
const maxItems = maxItemsParam ? parseInt(maxItemsParam, 10) : 25;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const items = await fetchFacebookItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
1,
|
||||||
|
LOCATION,
|
||||||
|
maxItems,
|
||||||
|
COOKIES_SOURCE,
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
if (!items || items.length === 0)
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Facebook scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
}
|
||||||
|
}
|
||||||
66
packages/api-server/src/routes/kijiji.ts
Normal file
66
packages/api-server/src/routes/kijiji.ts
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
import { fetchKijijiItems } from "@marketplace-scrapers/core";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/kijiji?q={query}
|
||||||
|
* Search Kijiji marketplace for listings
|
||||||
|
*/
|
||||||
|
export async function kijijiRoute(req: Request): Promise<Response> {
|
||||||
|
const reqUrl = new URL(req.url);
|
||||||
|
|
||||||
|
const SEARCH_QUERY =
|
||||||
|
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
||||||
|
if (!SEARCH_QUERY)
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
message: "Request didn't have 'query' header or 'q' search parameter!",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
|
||||||
|
const maxPagesParam = reqUrl.searchParams.get("maxPages");
|
||||||
|
const maxPages = maxPagesParam ? parseInt(maxPagesParam, 10) : 5;
|
||||||
|
const priceMinParam = reqUrl.searchParams.get("priceMin");
|
||||||
|
const priceMin = priceMinParam ? parseInt(priceMinParam, 10) : undefined;
|
||||||
|
const priceMaxParam = reqUrl.searchParams.get("priceMax");
|
||||||
|
const priceMax = priceMaxParam ? parseInt(priceMaxParam, 10) : undefined;
|
||||||
|
|
||||||
|
const searchOptions = {
|
||||||
|
location: reqUrl.searchParams.get("location") || undefined,
|
||||||
|
category: reqUrl.searchParams.get("category") || undefined,
|
||||||
|
keywords: reqUrl.searchParams.get("keywords") || undefined,
|
||||||
|
sortBy: reqUrl.searchParams.get("sortBy") as
|
||||||
|
| "relevancy"
|
||||||
|
| "date"
|
||||||
|
| "price"
|
||||||
|
| "distance"
|
||||||
|
| undefined,
|
||||||
|
sortOrder: reqUrl.searchParams.get("sortOrder") as
|
||||||
|
| "desc"
|
||||||
|
| "asc"
|
||||||
|
| undefined,
|
||||||
|
maxPages,
|
||||||
|
priceMin,
|
||||||
|
priceMax,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const items = await fetchKijijiItems(
|
||||||
|
SEARCH_QUERY,
|
||||||
|
4, // 4 requests per second for faster scraping
|
||||||
|
"https://www.kijiji.ca",
|
||||||
|
searchOptions,
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
if (!items)
|
||||||
|
return Response.json(
|
||||||
|
{ message: "Search didn't return any results!" },
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
return Response.json(items, { status: 200 });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Kijiji scraping error:", error);
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error occurred";
|
||||||
|
return Response.json({ message: errorMessage }, { status: 400 });
|
||||||
|
}
|
||||||
|
}
|
||||||
6
packages/api-server/src/routes/status.ts
Normal file
6
packages/api-server/src/routes/status.ts
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
/**
|
||||||
|
* Health check endpoint
|
||||||
|
*/
|
||||||
|
export function statusRoute(): Response {
|
||||||
|
return new Response("OK", { status: 200 });
|
||||||
|
}
|
||||||
13
packages/api-server/tsconfig.json
Normal file
13
packages/api-server/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
},
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true
|
||||||
|
}
|
||||||
|
}
|
||||||
21
packages/core/package.json
Normal file
21
packages/core/package.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/core",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"main": "./src/index.ts",
|
||||||
|
"module": "./src/index.ts",
|
||||||
|
"private": true,
|
||||||
|
"dependencies": {
|
||||||
|
"cli-progress": "^3.12.0",
|
||||||
|
"linkedom": "^0.18.12",
|
||||||
|
"unidecode": "^1.1.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest",
|
||||||
|
"@types/unidecode": "^1.1.0",
|
||||||
|
"@types/cli-progress": "^3.11.6"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
42
packages/core/src/index.ts
Normal file
42
packages/core/src/index.ts
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
// Export all scrapers
|
||||||
|
|
||||||
|
export type { EbayListingDetails } from "./scrapers/ebay";
|
||||||
|
export { default as fetchEbayItems } from "./scrapers/ebay";
|
||||||
|
export type { FacebookListingDetails } from "./scrapers/facebook";
|
||||||
|
export {
|
||||||
|
default as fetchFacebookItems,
|
||||||
|
ensureFacebookCookies,
|
||||||
|
extractFacebookItemData,
|
||||||
|
extractFacebookMarketplaceData,
|
||||||
|
fetchFacebookItem,
|
||||||
|
parseFacebookAds,
|
||||||
|
parseFacebookCookieString,
|
||||||
|
parseFacebookItem,
|
||||||
|
} from "./scrapers/facebook";
|
||||||
|
export type {
|
||||||
|
DetailedListing,
|
||||||
|
KijijiListingDetails,
|
||||||
|
ListingFetchOptions,
|
||||||
|
SearchOptions,
|
||||||
|
} from "./scrapers/kijiji";
|
||||||
|
export {
|
||||||
|
buildSearchUrl,
|
||||||
|
default as fetchKijijiItems,
|
||||||
|
extractApolloState,
|
||||||
|
HttpError,
|
||||||
|
NetworkError,
|
||||||
|
ParseError,
|
||||||
|
parseDetailedListing,
|
||||||
|
parseSearch,
|
||||||
|
RateLimitError,
|
||||||
|
resolveCategoryId,
|
||||||
|
resolveLocationId,
|
||||||
|
slugify,
|
||||||
|
ValidationError,
|
||||||
|
} from "./scrapers/kijiji";
|
||||||
|
// Export shared types
|
||||||
|
export * from "./types/common";
|
||||||
|
export * from "./utils/delay";
|
||||||
|
export * from "./utils/format";
|
||||||
|
// Export shared utilities
|
||||||
|
export * from "./utils/http";
|
||||||
@@ -1,12 +1,9 @@
|
|||||||
import cliProgress from "cli-progress";
|
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
|
import { delay } from "../utils/delay";
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
type HTMLString = string;
|
export interface EbayListingDetails {
|
||||||
|
|
||||||
type ListingDetails = {
|
|
||||||
url: string;
|
url: string;
|
||||||
title: string;
|
title: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
@@ -21,37 +18,10 @@ type ListingDetails = {
|
|||||||
endDate?: string;
|
endDate?: string;
|
||||||
numberOfViews?: number;
|
numberOfViews?: number;
|
||||||
address?: string | null;
|
address?: string | null;
|
||||||
};
|
}
|
||||||
|
|
||||||
// ----------------------------- Utilities -----------------------------
|
// ----------------------------- Utilities -----------------------------
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
||||||
return typeof value === "object" && value !== null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function delay(ms: number): Promise<void> {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Turns cents to localized currency string.
|
|
||||||
*/
|
|
||||||
function formatCentsToCurrency(
|
|
||||||
num: number | string | undefined,
|
|
||||||
locale = "en-US",
|
|
||||||
): string {
|
|
||||||
if (num == null) return "";
|
|
||||||
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
|
|
||||||
if (Number.isNaN(cents)) return "";
|
|
||||||
const dollars = cents / 100;
|
|
||||||
const formatter = new Intl.NumberFormat(locale, {
|
|
||||||
minimumFractionDigits: 2,
|
|
||||||
maximumFractionDigits: 2,
|
|
||||||
useGrouping: true,
|
|
||||||
});
|
|
||||||
return formatter.format(dollars);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
* Parse eBay currency string like "$1.50 CAD" or "CA $1.50" into cents
|
||||||
*/
|
*/
|
||||||
@@ -68,7 +38,7 @@ function parseEbayPrice(
|
|||||||
if (!numberMatches) return null;
|
if (!numberMatches) return null;
|
||||||
|
|
||||||
const amountStr = numberMatches[0].replace(/,/g, "");
|
const amountStr = numberMatches[0].replace(/,/g, "");
|
||||||
const dollars = Number.parseFloat(amountStr);
|
const dollars = parseFloat(amountStr);
|
||||||
if (Number.isNaN(dollars)) return null;
|
if (Number.isNaN(dollars)) return null;
|
||||||
|
|
||||||
const cents = Math.round(dollars * 100);
|
const cents = Math.round(dollars * 100);
|
||||||
@@ -100,81 +70,6 @@ class HttpError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----------------------------- HTTP Client -----------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
Fetch HTML with a basic retry strategy and simple rate-limit delay between calls.
|
|
||||||
- Retries on 429 and 5xx
|
|
||||||
- Respects X-RateLimit-Reset when present (seconds)
|
|
||||||
*/
|
|
||||||
async function fetchHtml(
|
|
||||||
url: string,
|
|
||||||
DELAY_MS: number,
|
|
||||||
opts?: {
|
|
||||||
maxRetries?: number;
|
|
||||||
retryBaseMs?: number;
|
|
||||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
|
||||||
},
|
|
||||||
): Promise<HTMLString> {
|
|
||||||
const maxRetries = opts?.maxRetries ?? 3;
|
|
||||||
const retryBaseMs = opts?.retryBaseMs ?? 500;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
||||||
try {
|
|
||||||
const res = await fetch(url, {
|
|
||||||
method: "GET",
|
|
||||||
headers: {
|
|
||||||
accept:
|
|
||||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
||||||
"accept-language": "en-CA,en-US;q=0.9,en;q=0.8",
|
|
||||||
"cache-control": "no-cache",
|
|
||||||
"upgrade-insecure-requests": "1",
|
|
||||||
"user-agent":
|
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
|
||||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
|
||||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
// Respect 429 reset if provided
|
|
||||||
if (res.status === 429) {
|
|
||||||
const resetSeconds = rateLimitReset
|
|
||||||
? Number(rateLimitReset)
|
|
||||||
: Number.NaN;
|
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
|
||||||
? Math.max(0, resetSeconds * 1000)
|
|
||||||
: (attempt + 1) * retryBaseMs;
|
|
||||||
await delay(waitMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Retry on 5xx
|
|
||||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
|
||||||
await delay((attempt + 1) * retryBaseMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${res.status}`,
|
|
||||||
res.status,
|
|
||||||
url,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await res.text();
|
|
||||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
|
||||||
await delay(DELAY_MS);
|
|
||||||
return html;
|
|
||||||
} catch (err) {
|
|
||||||
if (attempt >= maxRetries) throw err;
|
|
||||||
await delay((attempt + 1) * retryBaseMs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new Error("Exhausted retries without response");
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- Parsing -----------------------------
|
// ----------------------------- Parsing -----------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -185,9 +80,9 @@ function parseEbayListings(
|
|||||||
keywords: string[],
|
keywords: string[],
|
||||||
exclusions: string[],
|
exclusions: string[],
|
||||||
strictMode: boolean,
|
strictMode: boolean,
|
||||||
): ListingDetails[] {
|
): EbayListingDetails[] {
|
||||||
const { document } = parseHTML(htmlString);
|
const { document } = parseHTML(htmlString);
|
||||||
const results: ListingDetails[] = [];
|
const results: EbayListingDetails[] = [];
|
||||||
|
|
||||||
// Find all listing links by looking for eBay item URLs (/itm/)
|
// Find all listing links by looking for eBay item URLs (/itm/)
|
||||||
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
const linkElements = document.querySelectorAll('a[href*="itm/"]');
|
||||||
@@ -206,13 +101,26 @@ function parseEbayListings(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Find the container - go up several levels to find the item container
|
// Find the container - go up several levels to find the item container
|
||||||
// Modern eBay uses complex nested structures
|
// Modern eBay uses complex nested structures (often 5-10 levels deep)
|
||||||
let container = linkElement.parentElement?.parentElement?.parentElement;
|
let container: Element | null = linkElement;
|
||||||
if (!container) {
|
let depth = 0;
|
||||||
// Try a different level
|
const maxDepth = 15;
|
||||||
container = linkElement.parentElement?.parentElement;
|
|
||||||
|
// Walk up until we find a list item or results container
|
||||||
|
while (container && depth < maxDepth) {
|
||||||
|
const classes = container.className || "";
|
||||||
|
if (
|
||||||
|
classes.includes("s-item") ||
|
||||||
|
classes.includes("srp-results") ||
|
||||||
|
container.tagName === "LI"
|
||||||
|
) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
container = container.parentElement;
|
||||||
|
depth++;
|
||||||
}
|
}
|
||||||
if (!container) continue;
|
|
||||||
|
if (!container || depth >= maxDepth) continue;
|
||||||
|
|
||||||
// Extract title - look for heading or title-related elements near the link
|
// Extract title - look for heading or title-related elements near the link
|
||||||
// Modern eBay often uses h3, span, or div with text content near the link
|
// Modern eBay often uses h3, span, or div with text content near the link
|
||||||
@@ -273,8 +181,9 @@ function parseEbayListings(
|
|||||||
if (title === "Shop on eBay" || title.length < 3) continue;
|
if (title === "Shop on eBay" || title.length < 3) continue;
|
||||||
|
|
||||||
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
// Extract price - look for eBay's price classes, preferring sale/discount prices
|
||||||
|
// Updated for 2026 eBay HTML structure
|
||||||
let priceElement = container.querySelector(
|
let priceElement = container.querySelector(
|
||||||
'[class*="s-item__price"], .s-item__price, [class*="price"]',
|
'[class*="s-item__price"], .s-item__price, .s-card__attribute-row, [class*="price"]',
|
||||||
);
|
);
|
||||||
|
|
||||||
// If no direct price class, look for spans containing $ (but not titles)
|
// If no direct price class, look for spans containing $ (but not titles)
|
||||||
@@ -323,7 +232,7 @@ function parseEbayListings(
|
|||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
if (
|
if (
|
||||||
text &&
|
text &&
|
||||||
/^\s*[\$£€¥]/u.test(text) &&
|
/^\s*[$£€¥]/u.test(text) &&
|
||||||
text.length < 50 &&
|
text.length < 50 &&
|
||||||
!/\d{4}/.test(text)
|
!/\d{4}/.test(text)
|
||||||
) {
|
) {
|
||||||
@@ -380,14 +289,15 @@ function parseEbayListings(
|
|||||||
// Apply strict mode filter (title must contain at least one keyword)
|
// Apply strict mode filter (title must contain at least one keyword)
|
||||||
if (
|
if (
|
||||||
strictMode &&
|
strictMode &&
|
||||||
|
title &&
|
||||||
!keywords.some((keyword) =>
|
!keywords.some((keyword) =>
|
||||||
title?.toLowerCase().includes(keyword.toLowerCase()),
|
title.toLowerCase().includes(keyword.toLowerCase()),
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const listing: ListingDetails = {
|
const listing: EbayListingDetails = {
|
||||||
url: href,
|
url: href,
|
||||||
title,
|
title,
|
||||||
listingPrice: {
|
listingPrice: {
|
||||||
@@ -409,6 +319,58 @@ function parseEbayListings(
|
|||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Cookie Loading -----------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load eBay cookies with priority: URL param > ENV var > file
|
||||||
|
* @param cookiesSource - Optional cookie string from URL parameter (highest priority)
|
||||||
|
* @param cookiePath - Path to cookie file (default: ./cookies/ebay.json) (lowest priority)
|
||||||
|
* @returns Cookie string for HTTP header or undefined if no cookies found
|
||||||
|
*/
|
||||||
|
async function loadEbayCookies(
|
||||||
|
cookiesSource?: string,
|
||||||
|
cookiePath = "./cookies/ebay.json",
|
||||||
|
): Promise<string | undefined> {
|
||||||
|
// Priority 1: URL parameter (if provided)
|
||||||
|
if (cookiesSource?.trim()) {
|
||||||
|
console.log("Loaded eBay cookies from URL parameter");
|
||||||
|
return cookiesSource.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Environment variable
|
||||||
|
const envCookies = process.env.EBAY_COOKIE;
|
||||||
|
if (envCookies?.trim()) {
|
||||||
|
console.log("Loaded eBay cookies from EBAY_COOKIE env var");
|
||||||
|
return envCookies.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: Cookie file (fallback)
|
||||||
|
try {
|
||||||
|
const file = Bun.file(cookiePath);
|
||||||
|
if (await file.exists()) {
|
||||||
|
const content = await file.text();
|
||||||
|
const trimmed = content.trim();
|
||||||
|
if (trimmed) {
|
||||||
|
console.log(`Loaded eBay cookies from ${cookiePath}`);
|
||||||
|
return trimmed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(`Could not load cookies from ${cookiePath}: ${e}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// No cookies found (eBay cookies are optional, just warn)
|
||||||
|
console.warn(
|
||||||
|
"No eBay cookies found. eBay may block requests without valid session cookies.\n" +
|
||||||
|
"Provide cookies via (in priority order):\n" +
|
||||||
|
" 1. 'cookies' URL parameter (highest priority), or\n" +
|
||||||
|
" 2. EBAY_COOKIE environment variable, or\n" +
|
||||||
|
" 3. ./cookies/ebay.json file (lowest priority)\n" +
|
||||||
|
'Format: Cookie string like "name1=value1; name2=value2"',
|
||||||
|
);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
// ----------------------------- Main -----------------------------
|
// ----------------------------- Main -----------------------------
|
||||||
|
|
||||||
export default async function fetchEbayItems(
|
export default async function fetchEbayItems(
|
||||||
@@ -420,6 +382,10 @@ export default async function fetchEbayItems(
|
|||||||
strictMode?: boolean;
|
strictMode?: boolean;
|
||||||
exclusions?: string[];
|
exclusions?: string[];
|
||||||
keywords?: string[];
|
keywords?: string[];
|
||||||
|
buyItNowOnly?: boolean;
|
||||||
|
canadaOnly?: boolean;
|
||||||
|
cookies?: string; // Optional: Cookie string from URL parameter (highest priority)
|
||||||
|
cookiePath?: string; // Optional: Path to cookie file (default: ./cookies/ebay.json)
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
const {
|
const {
|
||||||
@@ -428,10 +394,31 @@ export default async function fetchEbayItems(
|
|||||||
strictMode = false,
|
strictMode = false,
|
||||||
exclusions = [],
|
exclusions = [],
|
||||||
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
keywords = [SEARCH_QUERY], // Default to search query if no keywords provided
|
||||||
|
buyItNowOnly = true,
|
||||||
|
canadaOnly = true,
|
||||||
|
cookies: cookiesSource,
|
||||||
|
cookiePath,
|
||||||
} = opts;
|
} = opts;
|
||||||
|
|
||||||
// Build eBay search URL - use Canadian site and tracking parameters like real browser
|
// Load eBay cookies with priority: URL param > ENV var > file
|
||||||
const searchUrl = `https://www.ebay.ca/sch/i.html?_nkw=${encodeURIComponent(SEARCH_QUERY)}^&_sacat=0^&_from=R40^&_trksid=p4432023.m570.l1313`;
|
const cookies = await loadEbayCookies(cookiesSource, cookiePath);
|
||||||
|
|
||||||
|
// Build eBay search URL - use Canadian site, Buy It Now filter, and Canada-only preference
|
||||||
|
const urlParams = new URLSearchParams({
|
||||||
|
_nkw: SEARCH_QUERY,
|
||||||
|
_sacat: "0",
|
||||||
|
_from: "R40",
|
||||||
|
});
|
||||||
|
|
||||||
|
if (buyItNowOnly) {
|
||||||
|
urlParams.set("LH_BIN", "1");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (canadaOnly) {
|
||||||
|
urlParams.set("LH_PrefLoc", "1");
|
||||||
|
}
|
||||||
|
|
||||||
|
const searchUrl = `https://www.ebay.ca/sch/i.html?${urlParams.toString()}`;
|
||||||
|
|
||||||
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
|
const DELAY_MS = Math.max(1, Math.floor(1000 / REQUESTS_PER_SECOND));
|
||||||
|
|
||||||
@@ -444,7 +431,7 @@ export default async function fetchEbayItems(
|
|||||||
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
|
"Mozilla/5.0 (X11; Linux x86_64; rv:141.0) Gecko/20100101 Firefox/141.0",
|
||||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
"Accept-Language": "en-US,en;q=0.5",
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
"Accept-Encoding": "gzip, deflate, br",
|
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||||
Referer: "https://www.ebay.ca/",
|
Referer: "https://www.ebay.ca/",
|
||||||
Connection: "keep-alive",
|
Connection: "keep-alive",
|
||||||
"Upgrade-Insecure-Requests": "1",
|
"Upgrade-Insecure-Requests": "1",
|
||||||
@@ -455,6 +442,11 @@ export default async function fetchEbayItems(
|
|||||||
Priority: "u=0, i",
|
Priority: "u=0, i",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Add cookies if available (helps bypass bot detection)
|
||||||
|
if (cookies) {
|
||||||
|
headers.Cookie = cookies;
|
||||||
|
}
|
||||||
|
|
||||||
const res = await fetch(searchUrl, {
|
const res = await fetch(searchUrl, {
|
||||||
method: "GET",
|
method: "GET",
|
||||||
headers,
|
headers,
|
||||||
@@ -472,7 +464,7 @@ export default async function fetchEbayItems(
|
|||||||
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
// Respect per-request delay to keep at or under REQUESTS_PER_SECOND
|
||||||
await delay(DELAY_MS);
|
await delay(DELAY_MS);
|
||||||
|
|
||||||
console.log("\nParsing eBay listings...");
|
console.log(`\nParsing eBay listings...`);
|
||||||
|
|
||||||
const listings = parseEbayListings(
|
const listings = parseEbayListings(
|
||||||
searchHtml,
|
searchHtml,
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
import cliProgress from "cli-progress";
|
import cliProgress from "cli-progress";
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
|
import type { HTMLString } from "../types/common";
|
||||||
|
import { delay } from "../utils/delay";
|
||||||
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
|
import { isRecord } from "../utils/http";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Facebook Marketplace Scraper
|
* Facebook Marketplace Scraper
|
||||||
@@ -12,8 +15,6 @@ import { parseHTML } from "linkedom";
|
|||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
type HTMLString = string;
|
|
||||||
|
|
||||||
interface Cookie {
|
interface Cookie {
|
||||||
name: string;
|
name: string;
|
||||||
value: string;
|
value: string;
|
||||||
@@ -36,6 +37,8 @@ interface FacebookAdNode {
|
|||||||
listing_price?: {
|
listing_price?: {
|
||||||
amount?: string | number;
|
amount?: string | number;
|
||||||
currency?: string;
|
currency?: string;
|
||||||
|
amount_with_offset_in_currency?: string | number;
|
||||||
|
formatted_amount?: string;
|
||||||
};
|
};
|
||||||
location?: {
|
location?: {
|
||||||
reverse_geocode?: {
|
reverse_geocode?: {
|
||||||
@@ -45,6 +48,24 @@ interface FacebookAdNode {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
creation_time?: number;
|
creation_time?: number;
|
||||||
|
is_sold?: boolean;
|
||||||
|
is_pending?: boolean;
|
||||||
|
is_live?: boolean;
|
||||||
|
is_hidden?: boolean;
|
||||||
|
primary_listing_photo?: {
|
||||||
|
image?: {
|
||||||
|
uri?: string;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
listing_video?: {
|
||||||
|
id?: string;
|
||||||
|
};
|
||||||
|
marketplace_listing_seller?: {
|
||||||
|
name?: string;
|
||||||
|
id?: string;
|
||||||
|
};
|
||||||
|
marketplace_listing_category_id?: string;
|
||||||
|
delivery_types?: string[];
|
||||||
[k: string]: unknown;
|
[k: string]: unknown;
|
||||||
};
|
};
|
||||||
[k: string]: unknown;
|
[k: string]: unknown;
|
||||||
@@ -63,11 +84,6 @@ interface FacebookMarketplaceSearch {
|
|||||||
[k: string]: unknown;
|
[k: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface FacebookRequireData {
|
|
||||||
require?: [number, number, number, FacebookMarketplaceSearch, number][];
|
|
||||||
[k: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface FacebookMarketplaceItem {
|
interface FacebookMarketplaceItem {
|
||||||
// Basic identification
|
// Basic identification
|
||||||
id: string;
|
id: string;
|
||||||
@@ -156,25 +172,10 @@ interface FacebookMarketplaceItem {
|
|||||||
logging_id?: string;
|
logging_id?: string;
|
||||||
reportable_ent_id?: string;
|
reportable_ent_id?: string;
|
||||||
|
|
||||||
// Related listings (for part-out sellers)
|
|
||||||
marketplace_listing_sets?: {
|
|
||||||
edges: Array<{
|
|
||||||
node: {
|
|
||||||
canonical_listing: {
|
|
||||||
id: string;
|
|
||||||
marketplace_listing_title: string;
|
|
||||||
is_live: boolean;
|
|
||||||
is_sold: boolean;
|
|
||||||
formatted_price: { text: string };
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}>;
|
|
||||||
};
|
|
||||||
|
|
||||||
[k: string]: unknown;
|
[k: string]: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
type ListingDetails = {
|
export interface FacebookListingDetails {
|
||||||
url: string;
|
url: string;
|
||||||
title: string;
|
title: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
@@ -198,18 +199,10 @@ type ListingDetails = {
|
|||||||
};
|
};
|
||||||
categoryId?: string;
|
categoryId?: string;
|
||||||
deliveryTypes?: string[];
|
deliveryTypes?: string[];
|
||||||
};
|
}
|
||||||
|
|
||||||
// ----------------------------- Utilities -----------------------------
|
// ----------------------------- Utilities -----------------------------
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
||||||
return typeof value === "object" && value !== null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function delay(ms: number): Promise<void> {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load Facebook cookies from file or string
|
* Load Facebook cookies from file or string
|
||||||
*/
|
*/
|
||||||
@@ -251,7 +244,7 @@ async function loadFacebookCookies(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.warn(`Could not load cookies from ./cookies/facebook.json: ${e}`);
|
console.warn(`Could not load cookies from ${cookiePath}: ${e}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return [];
|
return [];
|
||||||
@@ -260,7 +253,7 @@ async function loadFacebookCookies(
|
|||||||
/**
|
/**
|
||||||
* Parse Facebook cookie string into Cookie array format
|
* Parse Facebook cookie string into Cookie array format
|
||||||
*/
|
*/
|
||||||
function parseFacebookCookieString(cookieString: string): Cookie[] {
|
export function parseFacebookCookieString(cookieString: string): Cookie[] {
|
||||||
if (!cookieString || !cookieString.trim()) {
|
if (!cookieString || !cookieString.trim()) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
@@ -294,50 +287,65 @@ function parseFacebookCookieString(cookieString: string): Cookie[] {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensure Facebook cookies are available, parsing from env var if needed
|
* Load Facebook cookies with priority: URL param > ENV var > file
|
||||||
|
* @param cookiesSource - Optional cookie JSON string from URL parameter (highest priority)
|
||||||
|
* @param cookiePath - Path to cookie file (default: ./cookies/facebook.json) (lowest priority)
|
||||||
*/
|
*/
|
||||||
async function ensureFacebookCookies(
|
export async function ensureFacebookCookies(
|
||||||
|
cookiesSource?: string,
|
||||||
cookiePath = "./cookies/facebook.json",
|
cookiePath = "./cookies/facebook.json",
|
||||||
): Promise<Cookie[]> {
|
): Promise<Cookie[]> {
|
||||||
// First try to load existing cookies
|
// Priority 1: URL parameter (if provided)
|
||||||
|
if (cookiesSource) {
|
||||||
|
try {
|
||||||
|
const cookies = await loadFacebookCookies(cookiesSource);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
console.log(
|
||||||
|
`Loaded ${cookies.length} Facebook cookies from URL parameter`,
|
||||||
|
);
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn(`Failed to parse cookies from URL parameter: ${e}`);
|
||||||
|
// Continue to next priority
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Environment variable
|
||||||
|
const cookieString = process.env.FACEBOOK_COOKIE;
|
||||||
|
if (cookieString?.trim()) {
|
||||||
|
const cookies = parseFacebookCookieString(cookieString);
|
||||||
|
if (cookies.length > 0) {
|
||||||
|
console.log(
|
||||||
|
`Loaded ${cookies.length} Facebook cookies from FACEBOOK_COOKIE env var`,
|
||||||
|
);
|
||||||
|
return cookies;
|
||||||
|
}
|
||||||
|
console.warn("FACEBOOK_COOKIE env var contains no valid cookies");
|
||||||
|
// Continue to next priority
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 3: Cookie file (fallback)
|
||||||
try {
|
try {
|
||||||
const existing = await loadFacebookCookies(undefined, cookiePath);
|
const existing = await loadFacebookCookies(undefined, cookiePath);
|
||||||
if (existing.length > 0) {
|
if (existing.length > 0) {
|
||||||
|
console.log(
|
||||||
|
`Loaded ${existing.length} Facebook cookies from ${cookiePath}`,
|
||||||
|
);
|
||||||
return existing;
|
return existing;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (e) {
|
||||||
// File doesn't exist or is invalid, continue to check env var
|
console.warn(`Could not load cookies from ${cookiePath}: ${e}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to parse from environment variable
|
// No cookies found from any source
|
||||||
const cookieString = process.env.FACEBOOK_COOKIE;
|
throw new Error(
|
||||||
if (!cookieString || !cookieString.trim()) {
|
"No valid Facebook cookies found. Provide cookies via (in priority order):\n" +
|
||||||
throw new Error(
|
" 1. 'cookies' URL parameter (highest priority), or\n" +
|
||||||
"No valid Facebook cookies found. Either:\n" +
|
" 2. FACEBOOK_COOKIE environment variable, or\n" +
|
||||||
" 1. Set FACEBOOK_COOKIE environment variable with cookie string, or\n" +
|
" 3. ./cookies/facebook.json file (lowest priority)\n" +
|
||||||
" 2. Create ./cookies/facebook.json manually with cookie array",
|
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
|
||||||
);
|
);
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the cookie string
|
|
||||||
const cookies = parseFacebookCookieString(cookieString);
|
|
||||||
if (cookies.length === 0) {
|
|
||||||
throw new Error(
|
|
||||||
"FACEBOOK_COOKIE environment variable contains no valid cookies. " +
|
|
||||||
'Expected format: "name1=value1; name2=value2;"',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save to file for future use
|
|
||||||
try {
|
|
||||||
await Bun.write(cookiePath, JSON.stringify(cookies, null, 2));
|
|
||||||
console.log(`✅ Saved ${cookies.length} Facebook cookies to ${cookiePath}`);
|
|
||||||
} catch (error) {
|
|
||||||
console.warn(`! Could not save cookies to ${cookiePath}: ${error}`);
|
|
||||||
// Continue anyway, we have the cookies in memory
|
|
||||||
}
|
|
||||||
|
|
||||||
return cookies;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -381,6 +389,48 @@ class HttpError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----------------------------- Extraction Metrics -----------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Monitor API extraction success/failure for detecting changes
|
||||||
|
*/
|
||||||
|
const extractionStats = {
|
||||||
|
totalExtractions: 0,
|
||||||
|
successfulExtractions: 0,
|
||||||
|
failedExtractions: 0,
|
||||||
|
lastApiChangeDetected: null as Date | null,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log extraction metrics for monitoring API stability
|
||||||
|
*/
|
||||||
|
function logExtractionMetrics(success: boolean, itemId?: string) {
|
||||||
|
extractionStats.totalExtractions++;
|
||||||
|
if (success) {
|
||||||
|
extractionStats.successfulExtractions++;
|
||||||
|
} else {
|
||||||
|
extractionStats.failedExtractions++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log warning if extraction success rate drops below 80%
|
||||||
|
const successRate =
|
||||||
|
extractionStats.successfulExtractions / extractionStats.totalExtractions;
|
||||||
|
if (
|
||||||
|
extractionStats.totalExtractions > 10 &&
|
||||||
|
successRate < 0.8 &&
|
||||||
|
!extractionStats.lastApiChangeDetected
|
||||||
|
) {
|
||||||
|
console.warn(
|
||||||
|
"Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
|
||||||
|
);
|
||||||
|
extractionStats.lastApiChangeDetected = new Date();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!success && itemId) {
|
||||||
|
console.warn(`Facebook API extraction failed for item ${itemId}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ----------------------------- HTTP Client -----------------------------
|
// ----------------------------- HTTP Client -----------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -484,7 +534,7 @@ async function fetchHtml(
|
|||||||
/**
|
/**
|
||||||
Extract marketplace search data from Facebook page script tags
|
Extract marketplace search data from Facebook page script tags
|
||||||
*/
|
*/
|
||||||
function extractFacebookMarketplaceData(
|
export function extractFacebookMarketplaceData(
|
||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
): FacebookAdNode[] | null {
|
): FacebookAdNode[] | null {
|
||||||
const { document } = parseHTML(htmlString);
|
const { document } = parseHTML(htmlString);
|
||||||
@@ -531,7 +581,7 @@ function extractFacebookMarketplaceData(
|
|||||||
if (
|
if (
|
||||||
result &&
|
result &&
|
||||||
isRecord(result) &&
|
isRecord(result) &&
|
||||||
result.feed_units?.edges?.length > 0
|
(result as Record<string, unknown>).feed_units?.edges?.length > 0
|
||||||
) {
|
) {
|
||||||
marketplaceData = result as FacebookMarketplaceSearch;
|
marketplaceData = result as FacebookMarketplaceSearch;
|
||||||
break;
|
break;
|
||||||
@@ -546,14 +596,13 @@ function extractFacebookMarketplaceData(
|
|||||||
if (parsed.marketplace_search && isRecord(parsed.marketplace_search)) {
|
if (parsed.marketplace_search && isRecord(parsed.marketplace_search)) {
|
||||||
const searchData =
|
const searchData =
|
||||||
parsed.marketplace_search as FacebookMarketplaceSearch;
|
parsed.marketplace_search as FacebookMarketplaceSearch;
|
||||||
if (searchData.feed_units?.edges?.length > 0) {
|
const feedLength = searchData.feed_units?.edges?.length ?? 0;
|
||||||
|
if (feedLength > 0) {
|
||||||
marketplaceData = searchData;
|
marketplaceData = searchData;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {}
|
||||||
// Ignore parsing errors for other scripts
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!marketplaceData?.feed_units?.edges?.length) {
|
if (!marketplaceData?.feed_units?.edges?.length) {
|
||||||
@@ -567,78 +616,11 @@ function extractFacebookMarketplaceData(
|
|||||||
return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node }));
|
return marketplaceData.feed_units.edges.map((edge) => ({ node: edge.node }));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Monitor API extraction success/failure for detecting changes
|
|
||||||
*/
|
|
||||||
const extractionStats = {
|
|
||||||
totalExtractions: 0,
|
|
||||||
successfulExtractions: 0,
|
|
||||||
failedExtractions: 0,
|
|
||||||
lastApiChangeDetected: null as Date | null,
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Log extraction metrics for monitoring API stability
|
|
||||||
*/
|
|
||||||
function logExtractionMetrics(success: boolean, itemId?: string) {
|
|
||||||
extractionStats.totalExtractions++;
|
|
||||||
if (success) {
|
|
||||||
extractionStats.successfulExtractions++;
|
|
||||||
} else {
|
|
||||||
extractionStats.failedExtractions++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Log warning if extraction success rate drops below 80%
|
|
||||||
const successRate =
|
|
||||||
extractionStats.successfulExtractions / extractionStats.totalExtractions;
|
|
||||||
if (
|
|
||||||
extractionStats.totalExtractions > 10 &&
|
|
||||||
successRate < 0.8 &&
|
|
||||||
!extractionStats.lastApiChangeDetected
|
|
||||||
) {
|
|
||||||
console.warn(
|
|
||||||
"! Facebook Marketplace API extraction success rate dropped below 80%. This may indicate API changes.",
|
|
||||||
);
|
|
||||||
extractionStats.lastApiChangeDetected = new Date();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (success) {
|
|
||||||
console.log(
|
|
||||||
`📊 Facebook API extraction stats: ${extractionStats.successfulExtractions}/${extractionStats.totalExtractions} successful`,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
console.warn(
|
|
||||||
`❌ Facebook API extraction failed for item ${itemId || "unknown"}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Turns cents to localized currency string.
|
|
||||||
*/
|
|
||||||
function formatCentsToCurrency(
|
|
||||||
num: number | string | undefined,
|
|
||||||
locale = "en-US",
|
|
||||||
): string {
|
|
||||||
if (num == null) return "";
|
|
||||||
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
|
|
||||||
if (Number.isNaN(cents)) return "";
|
|
||||||
const dollars = cents / 100;
|
|
||||||
const formatter = new Intl.NumberFormat(locale, {
|
|
||||||
style: "currency",
|
|
||||||
currency: "USD",
|
|
||||||
minimumFractionDigits: 2,
|
|
||||||
maximumFractionDigits: 2,
|
|
||||||
useGrouping: true,
|
|
||||||
});
|
|
||||||
return formatter.format(dollars);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Extract marketplace item details from Facebook item page HTML
|
Extract marketplace item details from Facebook item page HTML
|
||||||
Updated for 2026 Facebook Marketplace API structure with multiple extraction paths
|
Updated for 2026 Facebook Marketplace API structure with multiple extraction paths
|
||||||
*/
|
*/
|
||||||
function extractFacebookItemData(
|
export function extractFacebookItemData(
|
||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
): FacebookMarketplaceItem | null {
|
): FacebookMarketplaceItem | null {
|
||||||
const { document } = parseHTML(htmlString);
|
const { document } = parseHTML(htmlString);
|
||||||
@@ -651,7 +633,7 @@ function extractFacebookItemData(
|
|||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(scriptText);
|
const parsed = JSON.parse(scriptText);
|
||||||
|
|
||||||
// Check for the 2026 require structure with marketplace product details
|
// Check for the require structure with marketplace product details
|
||||||
if (parsed.require && Array.isArray(parsed.require)) {
|
if (parsed.require && Array.isArray(parsed.require)) {
|
||||||
// Try multiple extraction paths discovered from reverse engineering
|
// Try multiple extraction paths discovered from reverse engineering
|
||||||
const extractionPaths = [
|
const extractionPaths = [
|
||||||
@@ -707,13 +689,14 @@ function extractFacebookItemData(
|
|||||||
if (depth > maxDepth) return null; // Prevent infinite recursion
|
if (depth > maxDepth) return null; // Prevent infinite recursion
|
||||||
if (isRecord(obj)) {
|
if (isRecord(obj)) {
|
||||||
// Check if this object matches the expected marketplace item structure
|
// Check if this object matches the expected marketplace item structure
|
||||||
|
const candidate = obj as Record<string, unknown>;
|
||||||
if (
|
if (
|
||||||
obj.marketplace_listing_title &&
|
candidate.marketplace_listing_title &&
|
||||||
obj.id &&
|
candidate.id &&
|
||||||
obj.__typename === "GroupCommerceProductItem" &&
|
candidate.__typename === "GroupCommerceProductItem" &&
|
||||||
obj.redacted_description
|
candidate.redacted_description
|
||||||
) {
|
) {
|
||||||
return obj as FacebookMarketplaceItem;
|
return candidate as unknown as FacebookMarketplaceItem;
|
||||||
}
|
}
|
||||||
// Recursively search nested objects and arrays
|
// Recursively search nested objects and arrays
|
||||||
for (const key in obj) {
|
for (const key in obj) {
|
||||||
@@ -764,20 +747,19 @@ function extractFacebookItemData(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch {}
|
||||||
// Log parsing errors for debugging but continue to next script
|
|
||||||
console.debug(`Failed to parse script for Facebook item data: ${error}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Parse Facebook marketplace search results into ListingDetails[]
|
Parse Facebook marketplace search results into ListingDetails[]
|
||||||
*/
|
*/
|
||||||
function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
export function parseFacebookAds(
|
||||||
const results: ListingDetails[] = [];
|
ads: FacebookAdNode[],
|
||||||
|
): FacebookListingDetails[] {
|
||||||
|
const results: FacebookListingDetails[] = [];
|
||||||
|
|
||||||
for (const adJson of ads) {
|
for (const adJson of ads) {
|
||||||
try {
|
try {
|
||||||
@@ -838,7 +820,7 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
const address = cityName || null;
|
const address = cityName || null;
|
||||||
|
|
||||||
// Determine listing status from Facebook flags
|
// Determine listing status from Facebook flags
|
||||||
let listingStatus: string | undefined = undefined;
|
let listingStatus: string | undefined;
|
||||||
if (listing.is_sold) {
|
if (listing.is_sold) {
|
||||||
listingStatus = "SOLD";
|
listingStatus = "SOLD";
|
||||||
} else if (listing.is_pending) {
|
} else if (listing.is_pending) {
|
||||||
@@ -868,12 +850,12 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
}
|
}
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
const listingDetails: ListingDetails = {
|
const listingDetails: FacebookListingDetails = {
|
||||||
url,
|
url,
|
||||||
title,
|
title,
|
||||||
listingPrice: {
|
listingPrice: {
|
||||||
amountFormatted:
|
amountFormatted:
|
||||||
priceObj.formatted_amount || formatCentsToCurrency(cents),
|
priceObj.formatted_amount || formatCentsToCurrency(cents, "en-CA"),
|
||||||
cents,
|
cents,
|
||||||
currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD
|
currency: priceObj.currency || "CAD", // Facebook marketplace often uses CAD
|
||||||
},
|
},
|
||||||
@@ -899,9 +881,9 @@ function parseFacebookAds(ads: FacebookAdNode[]): ListingDetails[] {
|
|||||||
Parse Facebook marketplace item details into ListingDetails format
|
Parse Facebook marketplace item details into ListingDetails format
|
||||||
Updated for 2026 GroupCommerceProductItem structure
|
Updated for 2026 GroupCommerceProductItem structure
|
||||||
*/
|
*/
|
||||||
function parseFacebookItem(
|
export function parseFacebookItem(
|
||||||
item: FacebookMarketplaceItem,
|
item: FacebookMarketplaceItem,
|
||||||
): ListingDetails | null {
|
): FacebookListingDetails | null {
|
||||||
try {
|
try {
|
||||||
const title = item.marketplace_listing_title || item.custom_title;
|
const title = item.marketplace_listing_title || item.custom_title;
|
||||||
if (!title) return null;
|
if (!title) return null;
|
||||||
@@ -920,7 +902,7 @@ function parseFacebookItem(
|
|||||||
if (!Number.isNaN(amount)) {
|
if (!Number.isNaN(amount)) {
|
||||||
cents = Math.round(amount * 100);
|
cents = Math.round(amount * 100);
|
||||||
amountFormatted =
|
amountFormatted =
|
||||||
item.formatted_price?.text || formatCentsToCurrency(cents);
|
item.formatted_price?.text || formatCentsToCurrency(cents, "en-CA");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -960,12 +942,9 @@ function parseFacebookItem(
|
|||||||
let listingType = "item";
|
let listingType = "item";
|
||||||
if (item.vehicle_make_display_name || item.vehicle_odometer_data) {
|
if (item.vehicle_make_display_name || item.vehicle_odometer_data) {
|
||||||
listingType = "vehicle";
|
listingType = "vehicle";
|
||||||
} else if (item.marketplace_listing_category_id) {
|
|
||||||
// Could map category IDs to types, but keeping simple for now
|
|
||||||
listingType = "item";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const listingDetails: ListingDetails = {
|
const listingDetails: FacebookListingDetails = {
|
||||||
url,
|
url,
|
||||||
title,
|
title,
|
||||||
description,
|
description,
|
||||||
@@ -990,20 +969,6 @@ function parseFacebookItem(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----------------------------- Exports for Testing -----------------------------
|
|
||||||
// Export internal functions for comprehensive testing
|
|
||||||
export {
|
|
||||||
extractFacebookItemData,
|
|
||||||
extractFacebookMarketplaceData,
|
|
||||||
parseFacebookItem,
|
|
||||||
parseFacebookAds,
|
|
||||||
formatCentsToCurrency,
|
|
||||||
loadFacebookCookies,
|
|
||||||
formatCookiesForHeader,
|
|
||||||
parseFacebookCookieString,
|
|
||||||
ensureFacebookCookies,
|
|
||||||
};
|
|
||||||
|
|
||||||
// ----------------------------- Main -----------------------------
|
// ----------------------------- Main -----------------------------
|
||||||
|
|
||||||
export default async function fetchFacebookItems(
|
export default async function fetchFacebookItems(
|
||||||
@@ -1014,22 +979,8 @@ export default async function fetchFacebookItems(
|
|||||||
cookiesSource?: string,
|
cookiesSource?: string,
|
||||||
cookiePath?: string,
|
cookiePath?: string,
|
||||||
) {
|
) {
|
||||||
// Load Facebook cookies - required for Facebook Marketplace access
|
// Load Facebook cookies with priority: URL param > ENV var > file
|
||||||
let cookies: Cookie[];
|
const cookies = await ensureFacebookCookies(cookiesSource, cookiePath);
|
||||||
if (cookiesSource) {
|
|
||||||
// Use provided cookie source (backward compatibility)
|
|
||||||
cookies = await loadFacebookCookies(cookiesSource);
|
|
||||||
} else {
|
|
||||||
// Auto-load from file or parse from env var
|
|
||||||
cookies = await ensureFacebookCookies(cookiePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (cookies.length === 0) {
|
|
||||||
throw new Error(
|
|
||||||
"Facebook cookies are required for marketplace access. " +
|
|
||||||
"Please provide cookies via 'cookies' parameter or create ./cookies/facebook.json file with valid Facebook session cookies.",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Format cookies for HTTP header
|
// Format cookies for HTTP header
|
||||||
const domain = "www.facebook.com";
|
const domain = "www.facebook.com";
|
||||||
@@ -1054,6 +1005,7 @@ export default async function fetchFacebookItems(
|
|||||||
let searchHtml: string;
|
let searchHtml: string;
|
||||||
try {
|
try {
|
||||||
searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
searchHtml = await fetchHtml(searchUrl, DELAY_MS, {
|
||||||
|
maxRetries: 3,
|
||||||
onRateInfo: (remaining, reset) => {
|
onRateInfo: (remaining, reset) => {
|
||||||
if (remaining && reset) {
|
if (remaining && reset) {
|
||||||
console.log(
|
console.log(
|
||||||
@@ -1115,7 +1067,7 @@ export async function fetchFacebookItem(
|
|||||||
itemId: string,
|
itemId: string,
|
||||||
cookiesSource?: string,
|
cookiesSource?: string,
|
||||||
cookiePath?: string,
|
cookiePath?: string,
|
||||||
): Promise<ListingDetails | null> {
|
): Promise<FacebookListingDetails | null> {
|
||||||
// Load Facebook cookies - required for Facebook Marketplace access
|
// Load Facebook cookies - required for Facebook Marketplace access
|
||||||
let cookies: Cookie[];
|
let cookies: Cookie[];
|
||||||
if (cookiesSource) {
|
if (cookiesSource) {
|
||||||
@@ -1,14 +1,20 @@
|
|||||||
import cliProgress from "cli-progress";
|
import cliProgress from "cli-progress";
|
||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
||||||
import { parseHTML } from "linkedom";
|
import { parseHTML } from "linkedom";
|
||||||
import unidecode from "unidecode";
|
import unidecode from "unidecode";
|
||||||
|
import type { HTMLString } from "../types/common";
|
||||||
// const unidecode = require("unidecode");
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
|
import {
|
||||||
|
fetchHtml,
|
||||||
|
HttpError,
|
||||||
|
isRecord,
|
||||||
|
NetworkError,
|
||||||
|
ParseError,
|
||||||
|
RateLimitError,
|
||||||
|
ValidationError,
|
||||||
|
} from "../utils/http";
|
||||||
|
|
||||||
// ----------------------------- Types -----------------------------
|
// ----------------------------- Types -----------------------------
|
||||||
|
|
||||||
type HTMLString = string;
|
|
||||||
|
|
||||||
type SearchListing = {
|
type SearchListing = {
|
||||||
name: string;
|
name: string;
|
||||||
listingLink: string;
|
listingLink: string;
|
||||||
@@ -49,7 +55,7 @@ interface ApolloListingRoot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Keep existing interface for backward compatibility
|
// Keep existing interface for backward compatibility
|
||||||
type ListingDetails = {
|
export interface KijijiListingDetails {
|
||||||
url: string;
|
url: string;
|
||||||
title: string;
|
title: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
@@ -64,10 +70,10 @@ type ListingDetails = {
|
|||||||
endDate?: string;
|
endDate?: string;
|
||||||
numberOfViews?: number;
|
numberOfViews?: number;
|
||||||
address?: string | null;
|
address?: string | null;
|
||||||
};
|
}
|
||||||
|
|
||||||
// New comprehensive interface for detailed listings
|
// New comprehensive interface for detailed listings
|
||||||
interface DetailedListing extends ListingDetails {
|
export interface DetailedListing extends KijijiListingDetails {
|
||||||
images: string[];
|
images: string[];
|
||||||
categoryId: number;
|
categoryId: number;
|
||||||
adSource: string;
|
adSource: string;
|
||||||
@@ -95,7 +101,7 @@ interface DetailedListing extends ListingDetails {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Configuration interfaces
|
// Configuration interfaces
|
||||||
interface SearchOptions {
|
export interface SearchOptions {
|
||||||
location?: number | string; // Location ID or name
|
location?: number | string; // Location ID or name
|
||||||
category?: number | string; // Category ID or name
|
category?: number | string; // Category ID or name
|
||||||
keywords?: string;
|
keywords?: string;
|
||||||
@@ -106,7 +112,7 @@ interface SearchOptions {
|
|||||||
priceMax?: number;
|
priceMax?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ListingFetchOptions {
|
export interface ListingFetchOptions {
|
||||||
includeImages?: boolean; // Default: true
|
includeImages?: boolean; // Default: true
|
||||||
sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed'
|
sellerDataDepth?: "basic" | "detailed" | "full"; // Default: 'detailed'
|
||||||
includeClientSideData?: boolean; // Default: false
|
includeClientSideData?: boolean; // Default: false
|
||||||
@@ -114,7 +120,7 @@ interface ListingFetchOptions {
|
|||||||
|
|
||||||
// ----------------------------- Constants & Mappings -----------------------------
|
// ----------------------------- Constants & Mappings -----------------------------
|
||||||
|
|
||||||
// Location mappings from KIJIJI.md
|
// Location mappings
|
||||||
const LOCATION_MAPPINGS: Record<string, number> = {
|
const LOCATION_MAPPINGS: Record<string, number> = {
|
||||||
canada: 0,
|
canada: 0,
|
||||||
ontario: 9004,
|
ontario: 9004,
|
||||||
@@ -134,7 +140,7 @@ const LOCATION_MAPPINGS: Record<string, number> = {
|
|||||||
"prince edward island": 9011,
|
"prince edward island": 9011,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Category mappings from KIJIJI.md (Buy & Sell main categories)
|
// Category mappings (Buy & Sell main categories)
|
||||||
const CATEGORY_MAPPINGS: Record<string, number> = {
|
const CATEGORY_MAPPINGS: Record<string, number> = {
|
||||||
all: 0,
|
all: 0,
|
||||||
"buy-sell": 10,
|
"buy-sell": 10,
|
||||||
@@ -177,14 +183,6 @@ const SORT_MAPPINGS: Record<string, string> = {
|
|||||||
distance: "DISTANCE",
|
distance: "DISTANCE",
|
||||||
};
|
};
|
||||||
|
|
||||||
// ----------------------------- Exports for Testing -----------------------------
|
|
||||||
// Note: These are exported for testing purposes only
|
|
||||||
|
|
||||||
export { resolveLocationId, resolveCategoryId, buildSearchUrl };
|
|
||||||
export { extractApolloState, parseSearch };
|
|
||||||
export { parseDetailedListing };
|
|
||||||
export { HttpError, NetworkError, ParseError, RateLimitError, ValidationError };
|
|
||||||
|
|
||||||
// ----------------------------- Utilities -----------------------------
|
// ----------------------------- Utilities -----------------------------
|
||||||
|
|
||||||
const SEPS = new Set([" ", "–", "—", "/", ":", ";", ",", ".", "-"]);
|
const SEPS = new Set([" ", "–", "—", "/", ":", ";", ",", ".", "-"]);
|
||||||
@@ -192,7 +190,7 @@ const SEPS = new Set([" ", "–", "—", "/", ":", ";", ",", ".", "-"]);
|
|||||||
/**
|
/**
|
||||||
* Resolve location ID from name or return numeric ID
|
* Resolve location ID from name or return numeric ID
|
||||||
*/
|
*/
|
||||||
function resolveLocationId(location?: number | string): number {
|
export function resolveLocationId(location?: number | string): number {
|
||||||
if (typeof location === "number") return location;
|
if (typeof location === "number") return location;
|
||||||
if (typeof location === "string") {
|
if (typeof location === "string") {
|
||||||
const normalized = location.toLowerCase().replace(/\s+/g, "-");
|
const normalized = location.toLowerCase().replace(/\s+/g, "-");
|
||||||
@@ -204,7 +202,7 @@ function resolveLocationId(location?: number | string): number {
|
|||||||
/**
|
/**
|
||||||
* Resolve category ID from name or return numeric ID
|
* Resolve category ID from name or return numeric ID
|
||||||
*/
|
*/
|
||||||
function resolveCategoryId(category?: number | string): number {
|
export function resolveCategoryId(category?: number | string): number {
|
||||||
if (typeof category === "number") return category;
|
if (typeof category === "number") return category;
|
||||||
if (typeof category === "string") {
|
if (typeof category === "string") {
|
||||||
const normalized = category.toLowerCase().replace(/\s+/g, "-");
|
const normalized = category.toLowerCase().replace(/\s+/g, "-");
|
||||||
@@ -216,7 +214,7 @@ function resolveCategoryId(category?: number | string): number {
|
|||||||
/**
|
/**
|
||||||
* Build search URL with enhanced parameters
|
* Build search URL with enhanced parameters
|
||||||
*/
|
*/
|
||||||
function buildSearchUrl(
|
export function buildSearchUrl(
|
||||||
keywords: string,
|
keywords: string,
|
||||||
options: SearchOptions & { page?: number },
|
options: SearchOptions & { page?: number },
|
||||||
BASE_URL = "https://www.kijiji.ca",
|
BASE_URL = "https://www.kijiji.ca",
|
||||||
@@ -224,8 +222,8 @@ function buildSearchUrl(
|
|||||||
const locationId = resolveLocationId(options.location);
|
const locationId = resolveLocationId(options.location);
|
||||||
const categoryId = resolveCategoryId(options.category);
|
const categoryId = resolveCategoryId(options.category);
|
||||||
|
|
||||||
const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell"; // Could be enhanced
|
const categorySlug = categoryId === 0 ? "buy-sell" : "buy-sell";
|
||||||
const locationSlug = locationId === 0 ? "canada" : "canada"; // Could be enhanced
|
const locationSlug = locationId === 0 ? "canada" : "canada";
|
||||||
|
|
||||||
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
|
let url = `${BASE_URL}/b-${categorySlug}/${locationSlug}/${slugify(keywords)}/k0c${categoryId}l${locationId}`;
|
||||||
|
|
||||||
@@ -242,7 +240,7 @@ function buildSearchUrl(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Slugifies a string for search
|
* Slugifies a string for Kijiji search URLs
|
||||||
*/
|
*/
|
||||||
export function slugify(input: string): string {
|
export function slugify(input: string): string {
|
||||||
const s = unidecode(input).toLowerCase();
|
const s = unidecode(input).toLowerCase();
|
||||||
@@ -269,211 +267,50 @@ export function slugify(input: string): string {
|
|||||||
return out.join("");
|
return out.join("");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Turns cents to localized currency string.
|
|
||||||
*/
|
|
||||||
export function formatCentsToCurrency(
|
|
||||||
num: number | string | undefined,
|
|
||||||
locale = "en-US",
|
|
||||||
): string {
|
|
||||||
if (num == null) return "";
|
|
||||||
const cents = typeof num === "string" ? Number.parseInt(num, 10) : num;
|
|
||||||
if (Number.isNaN(cents)) return "";
|
|
||||||
const dollars = cents / 100;
|
|
||||||
const formatter = new Intl.NumberFormat(locale, {
|
|
||||||
style: "currency",
|
|
||||||
currency: "USD",
|
|
||||||
minimumFractionDigits: 2,
|
|
||||||
maximumFractionDigits: 2,
|
|
||||||
});
|
|
||||||
return formatter.format(dollars);
|
|
||||||
}
|
|
||||||
|
|
||||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
||||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function delay(ms: number): Promise<void> {
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- Error Classes -----------------------------
|
|
||||||
|
|
||||||
class HttpError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly status: number,
|
|
||||||
public readonly url: string,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "HttpError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class NetworkError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly url: string,
|
|
||||||
public readonly cause?: Error,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "NetworkError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class ParseError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly data?: unknown,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "ParseError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class RateLimitError extends Error {
|
|
||||||
constructor(
|
|
||||||
message: string,
|
|
||||||
public readonly url: string,
|
|
||||||
public readonly resetTime?: number,
|
|
||||||
) {
|
|
||||||
super(message);
|
|
||||||
this.name = "RateLimitError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class ValidationError extends Error {
|
|
||||||
constructor(message: string) {
|
|
||||||
super(message);
|
|
||||||
this.name = "ValidationError";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- HTTP Client -----------------------------
|
|
||||||
|
|
||||||
/**
|
|
||||||
Fetch HTML with enhanced retry strategy and exponential backoff.
|
|
||||||
- Retries on 429, 5xx, and network errors
|
|
||||||
- Respects X-RateLimit-Reset when present (seconds)
|
|
||||||
- Exponential backoff with jitter
|
|
||||||
*/
|
|
||||||
async function fetchHtml(
|
|
||||||
url: string,
|
|
||||||
DELAY_MS: number,
|
|
||||||
opts?: {
|
|
||||||
maxRetries?: number;
|
|
||||||
retryBaseMs?: number;
|
|
||||||
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
|
||||||
},
|
|
||||||
): Promise<HTMLString> {
|
|
||||||
const maxRetries = opts?.maxRetries ?? 3;
|
|
||||||
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
|
||||||
|
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
||||||
try {
|
|
||||||
const controller = new AbortController();
|
|
||||||
const timeoutId = setTimeout(() => controller.abort(), 30000); // 30s timeout
|
|
||||||
|
|
||||||
const res = await fetch(url, {
|
|
||||||
method: "GET",
|
|
||||||
headers: {
|
|
||||||
accept:
|
|
||||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
||||||
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
|
||||||
"cache-control": "no-cache",
|
|
||||||
"upgrade-insecure-requests": "1",
|
|
||||||
"user-agent":
|
|
||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
|
||||||
},
|
|
||||||
signal: controller.signal,
|
|
||||||
});
|
|
||||||
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
|
|
||||||
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
|
||||||
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
|
||||||
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
// Handle rate limiting
|
|
||||||
if (res.status === 429) {
|
|
||||||
const resetSeconds = rateLimitReset
|
|
||||||
? Number(rateLimitReset)
|
|
||||||
: Number.NaN;
|
|
||||||
const waitMs = Number.isFinite(resetSeconds)
|
|
||||||
? Math.max(0, resetSeconds * 1000)
|
|
||||||
: calculateBackoffDelay(attempt, retryBaseMs);
|
|
||||||
|
|
||||||
if (attempt < maxRetries) {
|
|
||||||
await delay(waitMs);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw new RateLimitError(
|
|
||||||
`Rate limit exceeded for ${url}`,
|
|
||||||
url,
|
|
||||||
resetSeconds,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Retry on server errors
|
|
||||||
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
|
||||||
await delay(calculateBackoffDelay(attempt, retryBaseMs));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new HttpError(
|
|
||||||
`Request failed with status ${res.status}`,
|
|
||||||
res.status,
|
|
||||||
url,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const html = await res.text();
|
|
||||||
|
|
||||||
// Respect per-request delay to maintain rate limiting
|
|
||||||
await delay(DELAY_MS);
|
|
||||||
return html;
|
|
||||||
} catch (err) {
|
|
||||||
// Handle different error types
|
|
||||||
if (err instanceof RateLimitError || err instanceof HttpError) {
|
|
||||||
throw err; // Re-throw known errors
|
|
||||||
}
|
|
||||||
|
|
||||||
if (err instanceof Error && err.name === "AbortError") {
|
|
||||||
if (attempt < maxRetries) {
|
|
||||||
await delay(calculateBackoffDelay(attempt, retryBaseMs));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw new NetworkError(`Request timeout for ${url}`, url, err);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Network or other errors
|
|
||||||
if (attempt < maxRetries) {
|
|
||||||
await delay(calculateBackoffDelay(attempt, retryBaseMs));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
throw new NetworkError(
|
|
||||||
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
|
||||||
url,
|
|
||||||
err instanceof Error ? err : undefined,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculate exponential backoff delay with jitter
|
|
||||||
*/
|
|
||||||
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
|
||||||
const exponentialDelay = baseMs * 2 ** attempt;
|
|
||||||
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
|
||||||
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------- GraphQL Client -----------------------------
|
// ----------------------------- GraphQL Client -----------------------------
|
||||||
|
|
||||||
|
// GraphQL response interfaces
|
||||||
|
interface GraphQLReviewResponse {
|
||||||
|
user?: {
|
||||||
|
reviewSummary?: {
|
||||||
|
count?: number;
|
||||||
|
score?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GraphQLProfileResponse {
|
||||||
|
user?: {
|
||||||
|
memberSince?: string;
|
||||||
|
accountType?: string;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraphQL queries
|
||||||
|
const GRAPHQL_QUERIES = {
|
||||||
|
getReviewSummary: `
|
||||||
|
query GetReviewSummary($userId: String!) {
|
||||||
|
user(id: $userId) {
|
||||||
|
reviewSummary {
|
||||||
|
count
|
||||||
|
score
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
getProfileMetrics: `
|
||||||
|
query GetProfileMetrics($profileId: String!) {
|
||||||
|
user(id: $profileId) {
|
||||||
|
memberSince
|
||||||
|
accountType
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`,
|
||||||
|
} as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch additional data via GraphQL API
|
* Fetch additional data via GraphQL API
|
||||||
*/
|
*/
|
||||||
@@ -527,48 +364,6 @@ async function fetchGraphQLData(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// GraphQL response interfaces
|
|
||||||
interface GraphQLReviewResponse {
|
|
||||||
user?: {
|
|
||||||
reviewSummary?: {
|
|
||||||
count?: number;
|
|
||||||
score?: number;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
interface GraphQLProfileResponse {
|
|
||||||
user?: {
|
|
||||||
memberSince?: string;
|
|
||||||
accountType?: string;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// GraphQL queries from KIJIJI.md
|
|
||||||
const GRAPHQL_QUERIES = {
|
|
||||||
getReviewSummary: `
|
|
||||||
query GetReviewSummary($userId: String!) {
|
|
||||||
user(id: $userId) {
|
|
||||||
reviewSummary {
|
|
||||||
count
|
|
||||||
score
|
|
||||||
__typename
|
|
||||||
}
|
|
||||||
__typename
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`,
|
|
||||||
getProfileMetrics: `
|
|
||||||
query GetProfileMetrics($profileId: String!) {
|
|
||||||
user(id: $profileId) {
|
|
||||||
memberSince
|
|
||||||
accountType
|
|
||||||
__typename
|
|
||||||
}
|
|
||||||
}
|
|
||||||
`,
|
|
||||||
} as const;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch additional seller data via GraphQL
|
* Fetch additional seller data via GraphQL
|
||||||
*/
|
*/
|
||||||
@@ -617,9 +412,11 @@ async function fetchSellerDetails(
|
|||||||
// ----------------------------- Parsing -----------------------------
|
// ----------------------------- Parsing -----------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML.
|
Extracts json.props.pageProps.__APOLLO_STATE__ safely from a Kijiji page HTML.
|
||||||
*/
|
*/
|
||||||
function extractApolloState(htmlString: HTMLString): ApolloRecord | null {
|
export function extractApolloState(
|
||||||
|
htmlString: HTMLString,
|
||||||
|
): ApolloRecord | null {
|
||||||
const { document } = parseHTML(htmlString);
|
const { document } = parseHTML(htmlString);
|
||||||
const nextData = document.getElementById("__NEXT_DATA__");
|
const nextData = document.getElementById("__NEXT_DATA__");
|
||||||
if (!nextData || !nextData.textContent) return null;
|
if (!nextData || !nextData.textContent) return null;
|
||||||
@@ -637,7 +434,7 @@ function extractApolloState(htmlString: HTMLString): ApolloRecord | null {
|
|||||||
Parse search page apollo state into SearchListing[].
|
Parse search page apollo state into SearchListing[].
|
||||||
Filters keys likely to be listing entities and ensures url/title exist.
|
Filters keys likely to be listing entities and ensures url/title exist.
|
||||||
*/
|
*/
|
||||||
function parseSearch(
|
export function parseSearch(
|
||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
BASE_URL: string,
|
BASE_URL: string,
|
||||||
): SearchListing[] {
|
): SearchListing[] {
|
||||||
@@ -664,12 +461,12 @@ function parseSearch(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
Parse a listing page into a typed object.
|
Parse a listing page into a typed object (backward compatible).
|
||||||
*/
|
*/
|
||||||
function parseListing(
|
function _parseListing(
|
||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
BASE_URL: string,
|
BASE_URL: string,
|
||||||
): ListingDetails | null {
|
): KijijiListingDetails | null {
|
||||||
const apolloState = extractApolloState(htmlString);
|
const apolloState = extractApolloState(htmlString);
|
||||||
if (!apolloState) return null;
|
if (!apolloState) return null;
|
||||||
|
|
||||||
@@ -696,7 +493,8 @@ function parseListing(
|
|||||||
} = root as ApolloListingRoot;
|
} = root as ApolloListingRoot;
|
||||||
|
|
||||||
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
||||||
const amountFormatted = formatCentsToCurrency(cents);
|
const amountFormatted =
|
||||||
|
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
|
||||||
|
|
||||||
const numberOfViews =
|
const numberOfViews =
|
||||||
metrics?.views != null ? Number(metrics.views) : undefined;
|
metrics?.views != null ? Number(metrics.views) : undefined;
|
||||||
@@ -737,7 +535,7 @@ function parseListing(
|
|||||||
/**
|
/**
|
||||||
* Parse a listing page into a detailed object with all available fields
|
* Parse a listing page into a detailed object with all available fields
|
||||||
*/
|
*/
|
||||||
async function parseDetailedListing(
|
export async function parseDetailedListing(
|
||||||
htmlString: HTMLString,
|
htmlString: HTMLString,
|
||||||
BASE_URL: string,
|
BASE_URL: string,
|
||||||
options: ListingFetchOptions = {},
|
options: ListingFetchOptions = {},
|
||||||
@@ -766,7 +564,6 @@ async function parseDetailedListing(
|
|||||||
metrics,
|
metrics,
|
||||||
location,
|
location,
|
||||||
imageUrls,
|
imageUrls,
|
||||||
imageCount,
|
|
||||||
categoryId,
|
categoryId,
|
||||||
adSource,
|
adSource,
|
||||||
flags,
|
flags,
|
||||||
@@ -775,7 +572,8 @@ async function parseDetailedListing(
|
|||||||
} = root as ApolloListingRoot;
|
} = root as ApolloListingRoot;
|
||||||
|
|
||||||
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
const cents = price?.amount != null ? Number(price.amount) : undefined;
|
||||||
const amountFormatted = formatCentsToCurrency(cents);
|
const amountFormatted =
|
||||||
|
cents != null ? formatCentsToCurrency(cents, "en-CA") : undefined;
|
||||||
|
|
||||||
const numberOfViews =
|
const numberOfViews =
|
||||||
metrics?.views != null ? Number(metrics.views) : undefined;
|
metrics?.views != null ? Number(metrics.views) : undefined;
|
||||||
@@ -833,7 +631,7 @@ async function parseDetailedListing(
|
|||||||
...sellerInfo,
|
...sellerInfo,
|
||||||
...additionalData,
|
...additionalData,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch {
|
||||||
// Silently fail - GraphQL data is optional
|
// Silently fail - GraphQL data is optional
|
||||||
console.warn(
|
console.warn(
|
||||||
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
|
`Failed to fetch additional seller data for ${posterInfo.posterId}`,
|
||||||
@@ -901,8 +699,8 @@ export default async function fetchKijijiItems(
|
|||||||
sortBy: searchOptions.sortBy ?? "relevancy",
|
sortBy: searchOptions.sortBy ?? "relevancy",
|
||||||
sortOrder: searchOptions.sortOrder ?? "desc",
|
sortOrder: searchOptions.sortOrder ?? "desc",
|
||||||
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
|
maxPages: searchOptions.maxPages ?? 5, // Default to 5 pages
|
||||||
priceMin: searchOptions.priceMin,
|
priceMin: searchOptions.priceMin as number,
|
||||||
priceMax: searchOptions.priceMax,
|
priceMax: searchOptions.priceMax as number,
|
||||||
};
|
};
|
||||||
|
|
||||||
const finalListingOptions: Required<ListingFetchOptions> = {
|
const finalListingOptions: Required<ListingFetchOptions> = {
|
||||||
@@ -958,51 +756,75 @@ export default async function fetchKijijiItems(
|
|||||||
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
|
`\nFound ${newListingLinks.length} new listing links on page ${page}. Total unique: ${seenUrls.size}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Fetch details for this page's listings
|
// Fetch details for this page's listings with controlled concurrency
|
||||||
const progressBar = new cliProgress.SingleBar(
|
const isTTY = process.stdout?.isTTY ?? false;
|
||||||
{},
|
const progressBar = isTTY
|
||||||
cliProgress.Presets.shades_classic,
|
? new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
|
||||||
);
|
: null;
|
||||||
const totalProgress = newListingLinks.length;
|
const totalProgress = newListingLinks.length;
|
||||||
let currentProgress = 0;
|
let currentProgress = 0;
|
||||||
progressBar.start(totalProgress, currentProgress);
|
progressBar?.start(totalProgress, currentProgress);
|
||||||
|
|
||||||
for (const link of newListingLinks) {
|
// Process in batches for controlled concurrency
|
||||||
try {
|
const CONCURRENT_REQUESTS = REQUESTS_PER_SECOND * 2; // 2x rate for faster processing
|
||||||
const html = await fetchHtml(link, DELAY_MS, {
|
const results: (DetailedListing | null)[] = [];
|
||||||
onRateInfo: (remaining, reset) => {
|
|
||||||
if (remaining && reset) {
|
for (let i = 0; i < newListingLinks.length; i += CONCURRENT_REQUESTS) {
|
||||||
console.log(
|
const batch = newListingLinks.slice(i, i + CONCURRENT_REQUESTS);
|
||||||
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
const batchPromises = batch.map(async (link) => {
|
||||||
);
|
try {
|
||||||
}
|
const html = await fetchHtml(link, 0, {
|
||||||
},
|
// No per-request delay, batch handles rate limit
|
||||||
});
|
onRateInfo: (remaining, reset) => {
|
||||||
const parsed = await parseDetailedListing(
|
if (remaining && reset) {
|
||||||
html,
|
console.log(
|
||||||
BASE_URL,
|
`\nItem - Rate limit remaining: ${remaining}, reset in: ${reset}s`,
|
||||||
finalListingOptions,
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const parsed = await parseDetailedListing(
|
||||||
|
html,
|
||||||
|
BASE_URL,
|
||||||
|
finalListingOptions,
|
||||||
|
);
|
||||||
|
return parsed;
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof HttpError) {
|
||||||
|
console.error(
|
||||||
|
`\nFailed to fetch ${link}\n - ${err.statusCode} ${err.message}`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
console.error(
|
||||||
|
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
currentProgress++;
|
||||||
|
progressBar?.update(currentProgress);
|
||||||
|
if (!progressBar) {
|
||||||
|
console.log(`Progress: ${currentProgress}/${totalProgress}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const batchResults = await Promise.all(batchPromises);
|
||||||
|
results.push(...batchResults);
|
||||||
|
|
||||||
|
// Wait between batches to respect rate limit
|
||||||
|
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, DELAY_MS * batch.length),
|
||||||
);
|
);
|
||||||
if (parsed) {
|
|
||||||
allListings.push(parsed);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof HttpError) {
|
|
||||||
console.error(
|
|
||||||
`\nFailed to fetch ${link}\n - ${err.status} ${err.message}`,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
console.error(
|
|
||||||
`\nFailed to fetch ${link}\n - ${String((err as Error)?.message || err)}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
currentProgress++;
|
|
||||||
progressBar.update(currentProgress);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
progressBar.stop();
|
allListings.push(
|
||||||
|
...results.filter((r): r is DetailedListing => r !== null),
|
||||||
|
);
|
||||||
|
|
||||||
|
progressBar?.stop();
|
||||||
|
|
||||||
// If we got fewer results than expected (40 per page), we've reached the end
|
// If we got fewer results than expected (40 per page), we've reached the end
|
||||||
if (searchResults.length < 40) {
|
if (searchResults.length < 40) {
|
||||||
@@ -1013,3 +835,6 @@ export default async function fetchKijijiItems(
|
|||||||
console.log(`\nParsed ${allListings.length} detailed listings.`);
|
console.log(`\nParsed ${allListings.length} detailed listings.`);
|
||||||
return allListings;
|
return allListings;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Re-export error classes for convenience
|
||||||
|
export { HttpError, NetworkError, ParseError, RateLimitError, ValidationError };
|
||||||
20
packages/core/src/types/common.ts
Normal file
20
packages/core/src/types/common.ts
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
/** HTML string alias for better type clarity */
|
||||||
|
export type HTMLString = string;
|
||||||
|
|
||||||
|
/** Currency price object with formatting options */
|
||||||
|
export interface Price {
|
||||||
|
amountFormatted: string;
|
||||||
|
cents: number;
|
||||||
|
currency: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Base listing details common across all marketplaces */
|
||||||
|
export interface ListingDetails {
|
||||||
|
url: string;
|
||||||
|
title: string;
|
||||||
|
listingPrice: Price;
|
||||||
|
listingType: string;
|
||||||
|
listingStatus: string;
|
||||||
|
address?: string | null;
|
||||||
|
creationDate?: string;
|
||||||
|
}
|
||||||
8
packages/core/src/utils/delay.ts
Normal file
8
packages/core/src/utils/delay.ts
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
/**
|
||||||
|
* Delay execution for a specified number of milliseconds
|
||||||
|
* @param ms - Milliseconds to delay
|
||||||
|
* @returns A promise that resolves after the specified delay
|
||||||
|
*/
|
||||||
|
export function delay(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
24
packages/core/src/utils/format.ts
Normal file
24
packages/core/src/utils/format.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
/**
|
||||||
|
* Format cents to a human-readable currency string
|
||||||
|
* @param cents - Amount in cents (integer)
|
||||||
|
* @param locale - Locale string for formatting (e.g., 'en-CA', 'en-US')
|
||||||
|
* @returns Formatted currency string
|
||||||
|
*/
|
||||||
|
export function formatCentsToCurrency(
|
||||||
|
cents: number,
|
||||||
|
locale: string = "en-CA",
|
||||||
|
): string {
|
||||||
|
try {
|
||||||
|
const formatter = new Intl.NumberFormat(locale, {
|
||||||
|
style: "currency",
|
||||||
|
currency: "CAD",
|
||||||
|
minimumFractionDigits: 2,
|
||||||
|
maximumFractionDigits: 2,
|
||||||
|
});
|
||||||
|
return formatter.format(cents / 100);
|
||||||
|
} catch {
|
||||||
|
// Fallback if locale is not supported
|
||||||
|
const dollars = (cents / 100).toFixed(2);
|
||||||
|
return `$${dollars}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
200
packages/core/src/utils/http.ts
Normal file
200
packages/core/src/utils/http.ts
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
/** Custom error class for HTTP-related failures */
|
||||||
|
export class HttpError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly statusCode: number,
|
||||||
|
public readonly url?: string,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "HttpError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for network failures (timeouts, connection issues) */
|
||||||
|
export class NetworkError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly url: string,
|
||||||
|
public readonly cause?: Error,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "NetworkError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for parsing failures */
|
||||||
|
export class ParseError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly data?: unknown,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ParseError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for rate limiting */
|
||||||
|
export class RateLimitError extends Error {
|
||||||
|
constructor(
|
||||||
|
message: string,
|
||||||
|
public readonly url: string,
|
||||||
|
public readonly resetTime?: number,
|
||||||
|
) {
|
||||||
|
super(message);
|
||||||
|
this.name = "RateLimitError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Error class for validation failures */
|
||||||
|
export class ValidationError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "ValidationError";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Type guard to check if a value is a record (object) */
|
||||||
|
export function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculate exponential backoff delay with jitter
|
||||||
|
*/
|
||||||
|
function calculateBackoffDelay(attempt: number, baseMs: number): number {
|
||||||
|
const exponentialDelay = baseMs * 2 ** attempt;
|
||||||
|
const jitter = Math.random() * 0.1 * exponentialDelay; // 10% jitter
|
||||||
|
return Math.min(exponentialDelay + jitter, 30000); // Cap at 30 seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Options for fetchHtml */
|
||||||
|
export interface FetchHtmlOptions {
|
||||||
|
maxRetries?: number;
|
||||||
|
retryBaseMs?: number;
|
||||||
|
timeoutMs?: number;
|
||||||
|
onRateInfo?: (remaining: string | null, reset: string | null) => void;
|
||||||
|
headers?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch HTML content from a URL with automatic retries, timeout, and exponential backoff
|
||||||
|
* @param url - The URL to fetch
|
||||||
|
* @param delayMs - Delay in milliseconds between requests (rate limiting)
|
||||||
|
* @param opts - Optional fetch options
|
||||||
|
* @returns The HTML content as a string
|
||||||
|
* @throws HttpError, NetworkError, or RateLimitError on failure
|
||||||
|
*/
|
||||||
|
export async function fetchHtml(
|
||||||
|
url: string,
|
||||||
|
delayMs: number,
|
||||||
|
opts?: FetchHtmlOptions,
|
||||||
|
): Promise<string> {
|
||||||
|
const maxRetries = opts?.maxRetries ?? 3;
|
||||||
|
const retryBaseMs = opts?.retryBaseMs ?? 1000;
|
||||||
|
const timeoutMs = opts?.timeoutMs ?? 30000;
|
||||||
|
|
||||||
|
const defaultHeaders: Record<string, string> = {
|
||||||
|
accept:
|
||||||
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||||
|
"accept-language": "en-GB,en-US;q=0.9,en;q=0.8",
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
"upgrade-insecure-requests": "1",
|
||||||
|
"user-agent":
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
|
||||||
|
};
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
|
try {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method: "GET",
|
||||||
|
headers: { ...defaultHeaders, ...opts?.headers },
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
|
||||||
|
const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
|
||||||
|
const rateLimitReset = res.headers.get("X-RateLimit-Reset");
|
||||||
|
opts?.onRateInfo?.(rateLimitRemaining, rateLimitReset);
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
// Handle rate limiting
|
||||||
|
if (res.status === 429) {
|
||||||
|
const resetSeconds = rateLimitReset
|
||||||
|
? Number(rateLimitReset)
|
||||||
|
: Number.NaN;
|
||||||
|
const waitMs = Number.isFinite(resetSeconds)
|
||||||
|
? Math.max(0, resetSeconds * 1000)
|
||||||
|
: calculateBackoffDelay(attempt, retryBaseMs);
|
||||||
|
|
||||||
|
if (attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new RateLimitError(
|
||||||
|
`Rate limit exceeded for ${url}`,
|
||||||
|
url,
|
||||||
|
resetSeconds,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retry on server errors
|
||||||
|
if (res.status >= 500 && res.status < 600 && attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new HttpError(
|
||||||
|
`Request failed with status ${res.status}`,
|
||||||
|
res.status,
|
||||||
|
url,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = await res.text();
|
||||||
|
|
||||||
|
// Respect per-request delay to maintain rate limiting
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
|
return html;
|
||||||
|
} catch (err) {
|
||||||
|
// Re-throw known errors
|
||||||
|
if (
|
||||||
|
err instanceof RateLimitError ||
|
||||||
|
err instanceof HttpError ||
|
||||||
|
err instanceof NetworkError
|
||||||
|
) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (err instanceof Error && err.name === "AbortError") {
|
||||||
|
if (attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new NetworkError(`Request timeout for ${url}`, url, err);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Network or other errors
|
||||||
|
if (attempt < maxRetries) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, calculateBackoffDelay(attempt, retryBaseMs)),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw new NetworkError(
|
||||||
|
`Network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
url,
|
||||||
|
err instanceof Error ? err : undefined,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new NetworkError(`Exhausted retries without response for ${url}`, url);
|
||||||
|
}
|
||||||
@@ -5,11 +5,10 @@ import {
|
|||||||
fetchFacebookItem,
|
fetchFacebookItem,
|
||||||
formatCentsToCurrency,
|
formatCentsToCurrency,
|
||||||
formatCookiesForHeader,
|
formatCookiesForHeader,
|
||||||
loadFacebookCookies,
|
|
||||||
parseFacebookAds,
|
parseFacebookAds,
|
||||||
parseFacebookCookieString,
|
parseFacebookCookieString,
|
||||||
parseFacebookItem,
|
parseFacebookItem,
|
||||||
} from "../src/facebook";
|
} from "../src/scrapers/facebook";
|
||||||
|
|
||||||
// Mock fetch globally
|
// Mock fetch globally
|
||||||
const originalFetch = global.fetch;
|
const originalFetch = global.fetch;
|
||||||
@@ -183,7 +182,7 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = await fetchFacebookItem("123", mockCookies);
|
const _result = await fetchFacebookItem("123", mockCookies);
|
||||||
expect(attempts).toBe(2);
|
expect(attempts).toBe(2);
|
||||||
// Should eventually succeed after retry
|
// Should eventually succeed after retry
|
||||||
});
|
});
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
import fetchFacebookItems, { fetchFacebookItem } from "../src/facebook";
|
import { fetchFacebookItems } from "../src/scrapers/facebook";
|
||||||
|
|
||||||
// Mock fetch globally
|
// Mock fetch globally
|
||||||
const originalFetch = global.fetch;
|
const originalFetch = global.fetch;
|
||||||
@@ -1,14 +1,13 @@
|
|||||||
import { describe, expect, test } from "bun:test";
|
import { describe, expect, test } from "bun:test";
|
||||||
import {
|
import {
|
||||||
HttpError,
|
buildSearchUrl,
|
||||||
NetworkError,
|
NetworkError,
|
||||||
ParseError,
|
ParseError,
|
||||||
RateLimitError,
|
RateLimitError,
|
||||||
ValidationError,
|
|
||||||
buildSearchUrl,
|
|
||||||
resolveCategoryId,
|
resolveCategoryId,
|
||||||
resolveLocationId,
|
resolveLocationId,
|
||||||
} from "../src/kijiji";
|
ValidationError,
|
||||||
|
} from "../src/scrapers/kijiji";
|
||||||
|
|
||||||
describe("Location and Category Resolution", () => {
|
describe("Location and Category Resolution", () => {
|
||||||
describe("resolveLocationId", () => {
|
describe("resolveLocationId", () => {
|
||||||
@@ -121,14 +120,6 @@ describe("URL Construction", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("Error Classes", () => {
|
describe("Error Classes", () => {
|
||||||
test("HttpError should store status and URL", () => {
|
|
||||||
const error = new HttpError("Not found", 404, "https://example.com");
|
|
||||||
expect(error.message).toBe("Not found");
|
|
||||||
expect(error.status).toBe(404);
|
|
||||||
expect(error.url).toBe("https://example.com");
|
|
||||||
expect(error.name).toBe("HttpError");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("NetworkError should store URL and cause", () => {
|
test("NetworkError should store URL and cause", () => {
|
||||||
const cause = new Error("Connection failed");
|
const cause = new Error("Connection failed");
|
||||||
const error = new NetworkError(
|
const error = new NetworkError(
|
||||||
@@ -3,7 +3,7 @@ import {
|
|||||||
extractApolloState,
|
extractApolloState,
|
||||||
parseDetailedListing,
|
parseDetailedListing,
|
||||||
parseSearch,
|
parseSearch,
|
||||||
} from "../src/kijiji";
|
} from "../src/scrapers/kijiji";
|
||||||
|
|
||||||
// Mock fetch globally
|
// Mock fetch globally
|
||||||
const originalFetch = global.fetch;
|
const originalFetch = global.fetch;
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
import { describe, expect, test } from "bun:test";
|
||||||
import { formatCentsToCurrency, slugify } from "../src/kijiji";
|
import { formatCentsToCurrency, slugify } from "../src/scrapers/kijiji";
|
||||||
|
|
||||||
describe("Utility Functions", () => {
|
describe("Utility Functions", () => {
|
||||||
describe("slugify", () => {
|
describe("slugify", () => {
|
||||||
@@ -1,7 +1,4 @@
|
|||||||
// Test setup for Bun test runner
|
// Test setup for Bun test runner
|
||||||
import { expect } from "bun:test";
|
|
||||||
|
|
||||||
// Global test setup
|
|
||||||
// This file is loaded before any tests run due to bunfig.toml preload
|
// This file is loaded before any tests run due to bunfig.toml preload
|
||||||
|
|
||||||
// Mock fetch globally for tests
|
// Mock fetch globally for tests
|
||||||
13
packages/core/tsconfig.json
Normal file
13
packages/core/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
},
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true
|
||||||
|
}
|
||||||
|
}
|
||||||
21
packages/mcp-server/package.json
Normal file
21
packages/mcp-server/package.json
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/mcp-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"module": "./src/index.ts",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun ./src/index.ts",
|
||||||
|
"dev": "bun --watch ./src/index.ts",
|
||||||
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "latest"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
36
packages/mcp-server/src/index.ts
Normal file
36
packages/mcp-server/src/index.ts
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
import { handleMcpRequest } from "./protocol/handler";
|
||||||
|
import { serverCard } from "./protocol/metadata";
|
||||||
|
|
||||||
|
const PORT = process.env.MCP_PORT || 4006;
|
||||||
|
|
||||||
|
const server = Bun.serve({
|
||||||
|
port: PORT as number | string,
|
||||||
|
idleTimeout: 255, // 255 seconds (max allowed)
|
||||||
|
routes: {
|
||||||
|
// MCP metadata discovery endpoint
|
||||||
|
"/.well-known/mcp/server-card.json": new Response(
|
||||||
|
JSON.stringify(serverCard),
|
||||||
|
{
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
},
|
||||||
|
),
|
||||||
|
|
||||||
|
// MCP JSON-RPC 2.0 protocol endpoint
|
||||||
|
"/mcp": async (req: Request) => {
|
||||||
|
if (req.method === "POST") {
|
||||||
|
return await handleMcpRequest(req);
|
||||||
|
}
|
||||||
|
return Response.json(
|
||||||
|
{ message: "MCP endpoint requires POST request" },
|
||||||
|
{ status: 405 },
|
||||||
|
);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
// Fallback for all other routes
|
||||||
|
fetch(_req: Request) {
|
||||||
|
return new Response("Not Found", { status: 404 });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`MCP Server running on ${server.hostname}:${server.port}`);
|
||||||
291
packages/mcp-server/src/protocol/handler.ts
Normal file
291
packages/mcp-server/src/protocol/handler.ts
Normal file
@@ -0,0 +1,291 @@
|
|||||||
|
import { tools } from "./tools";
|
||||||
|
|
||||||
|
const API_BASE_URL = process.env.API_BASE_URL || "http://localhost:4005/api";
|
||||||
|
const API_TIMEOUT = Number(process.env.API_TIMEOUT) || 180000; // 3 minutes default
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle MCP JSON-RPC 2.0 protocol requests
|
||||||
|
*/
|
||||||
|
export async function handleMcpRequest(req: Request): Promise<Response> {
|
||||||
|
try {
|
||||||
|
const body = await req.json();
|
||||||
|
|
||||||
|
// Validate JSON-RPC 2.0 format
|
||||||
|
if (!body.jsonrpc || body.jsonrpc !== "2.0" || !body.method) {
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
error: { code: -32600, message: "Invalid Request" },
|
||||||
|
id: body.id,
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { method, params, id } = body;
|
||||||
|
|
||||||
|
// Handle initialize method
|
||||||
|
if (method === "initialize") {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
result: {
|
||||||
|
protocolVersion: "2025-06-18",
|
||||||
|
capabilities: {
|
||||||
|
tools: {
|
||||||
|
listChanged: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
serverInfo: {
|
||||||
|
name: "marketplace-scrapers",
|
||||||
|
version: "1.0.0",
|
||||||
|
},
|
||||||
|
instructions:
|
||||||
|
"Use search_kijiji, search_facebook, or search_ebay tools to find listings across Canadian marketplaces",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tools/list method
|
||||||
|
if (method === "tools/list") {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
result: {
|
||||||
|
tools,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle notifications (messages without id field should not get a response)
|
||||||
|
if (!id) {
|
||||||
|
// Notifications don't require a response
|
||||||
|
if (method === "notifications/initialized") {
|
||||||
|
// Client initialized successfully, no response needed
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
}
|
||||||
|
if (method === "notifications/progress") {
|
||||||
|
// Progress notifications, no response needed
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
}
|
||||||
|
// Unknown notification - still no response for notifications
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle tools/call method
|
||||||
|
if (method === "tools/call") {
|
||||||
|
const { name, arguments: args } = params || {};
|
||||||
|
|
||||||
|
if (!name || !args) {
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: {
|
||||||
|
code: -32602,
|
||||||
|
message: "Invalid params: name and arguments required",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Route tool calls to appropriate handlers
|
||||||
|
try {
|
||||||
|
let result: unknown;
|
||||||
|
|
||||||
|
if (name === "search_kijiji") {
|
||||||
|
const query = args.query;
|
||||||
|
if (!query) {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const params = new URLSearchParams({ q: query });
|
||||||
|
if (args.location) params.append("location", args.location);
|
||||||
|
if (args.category) params.append("category", args.category);
|
||||||
|
if (args.keywords) params.append("keywords", args.keywords);
|
||||||
|
if (args.sortBy) params.append("sortBy", args.sortBy);
|
||||||
|
if (args.sortOrder) params.append("sortOrder", args.sortOrder);
|
||||||
|
if (args.maxPages)
|
||||||
|
params.append("maxPages", args.maxPages.toString());
|
||||||
|
if (args.priceMin)
|
||||||
|
params.append("priceMin", args.priceMin.toString());
|
||||||
|
if (args.priceMax)
|
||||||
|
params.append("priceMax", args.priceMax.toString());
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[MCP] Calling Kijiji API: ${API_BASE_URL}/kijiji?${params.toString()}`,
|
||||||
|
);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(`${API_BASE_URL}/kijiji?${params.toString()}`),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() =>
|
||||||
|
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
console.error(
|
||||||
|
`[MCP] Kijiji API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
result = await response.json();
|
||||||
|
console.log(
|
||||||
|
`[MCP] Kijiji returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
|
);
|
||||||
|
} else if (name === "search_facebook") {
|
||||||
|
const query = args.query;
|
||||||
|
if (!query) {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const params = new URLSearchParams({ q: query });
|
||||||
|
if (args.location) params.append("location", args.location);
|
||||||
|
if (args.maxItems)
|
||||||
|
params.append("maxItems", args.maxItems.toString());
|
||||||
|
if (args.cookiesSource) params.append("cookies", args.cookiesSource);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[MCP] Calling Facebook API: ${API_BASE_URL}/facebook?${params.toString()}`,
|
||||||
|
);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(`${API_BASE_URL}/facebook?${params.toString()}`),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() =>
|
||||||
|
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
console.error(
|
||||||
|
`[MCP] Facebook API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
result = await response.json();
|
||||||
|
console.log(
|
||||||
|
`[MCP] Facebook returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
|
);
|
||||||
|
} else if (name === "search_ebay") {
|
||||||
|
const query = args.query;
|
||||||
|
if (!query) {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32602, message: "query parameter is required" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const params = new URLSearchParams({ q: query });
|
||||||
|
if (args.minPrice)
|
||||||
|
params.append("minPrice", args.minPrice.toString());
|
||||||
|
if (args.maxPrice)
|
||||||
|
params.append("maxPrice", args.maxPrice.toString());
|
||||||
|
if (args.strictMode !== undefined)
|
||||||
|
params.append("strictMode", args.strictMode.toString());
|
||||||
|
if (args.exclusions?.length)
|
||||||
|
params.append("exclusions", args.exclusions.join(","));
|
||||||
|
if (args.keywords?.length)
|
||||||
|
params.append("keywords", args.keywords.join(","));
|
||||||
|
if (args.buyItNowOnly !== undefined)
|
||||||
|
params.append("buyItNowOnly", args.buyItNowOnly.toString());
|
||||||
|
if (args.canadaOnly !== undefined)
|
||||||
|
params.append("canadaOnly", args.canadaOnly.toString());
|
||||||
|
if (args.maxItems)
|
||||||
|
params.append("maxItems", args.maxItems.toString());
|
||||||
|
if (args.cookies) params.append("cookies", args.cookies);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[MCP] Calling eBay API: ${API_BASE_URL}/ebay?${params.toString()}`,
|
||||||
|
);
|
||||||
|
const response = await Promise.race([
|
||||||
|
fetch(`${API_BASE_URL}/ebay?${params.toString()}`),
|
||||||
|
new Promise<Response>((_, reject) =>
|
||||||
|
setTimeout(
|
||||||
|
() =>
|
||||||
|
reject(new Error(`Request timed out after ${API_TIMEOUT}ms`)),
|
||||||
|
API_TIMEOUT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text();
|
||||||
|
console.error(
|
||||||
|
`[MCP] eBay API error ${response.status}: ${errorText}`,
|
||||||
|
);
|
||||||
|
throw new Error(`API returned ${response.status}: ${errorText}`);
|
||||||
|
}
|
||||||
|
result = await response.json();
|
||||||
|
console.log(
|
||||||
|
`[MCP] eBay returned ${Array.isArray(result) ? result.length : 0} items`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32601, message: `Unknown tool: ${name}` },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
result: {
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: JSON.stringify(result, null, 2),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error";
|
||||||
|
return Response.json({
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: {
|
||||||
|
code: -32603,
|
||||||
|
message: `Tool execution failed: ${errorMessage}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Method not found
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id,
|
||||||
|
error: { code: -32601, message: `Method not found: ${method}` },
|
||||||
|
},
|
||||||
|
{ status: 404 },
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage =
|
||||||
|
error instanceof Error ? error.message : "Unknown error";
|
||||||
|
return Response.json(
|
||||||
|
{
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
error: { code: -32700, message: `Parse error: ${errorMessage}` },
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
27
packages/mcp-server/src/protocol/metadata.ts
Normal file
27
packages/mcp-server/src/protocol/metadata.ts
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
/**
|
||||||
|
* MCP Server metadata for discovery
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const serverCard = {
|
||||||
|
$schema:
|
||||||
|
"https://static.modelcontextprotocol.io/schemas/mcp-server-card/v1.json",
|
||||||
|
version: "1.0",
|
||||||
|
protocolVersion: "2025-06-18",
|
||||||
|
serverInfo: {
|
||||||
|
name: "marketplace-scrapers",
|
||||||
|
title: "Marketplace Scrapers MCP Server",
|
||||||
|
version: "1.0.0",
|
||||||
|
},
|
||||||
|
transport: {
|
||||||
|
type: "streamable-http",
|
||||||
|
endpoint: "/mcp",
|
||||||
|
},
|
||||||
|
capabilities: {
|
||||||
|
tools: {
|
||||||
|
listChanged: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
description:
|
||||||
|
"Scrapes marketplace listings from Kijiji, Facebook Marketplace, and eBay",
|
||||||
|
tools: "dynamic",
|
||||||
|
};
|
||||||
145
packages/mcp-server/src/protocol/tools.ts
Normal file
145
packages/mcp-server/src/protocol/tools.ts
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
/**
|
||||||
|
* MCP tool definitions for marketplace scrapers
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const tools = [
|
||||||
|
{
|
||||||
|
name: "search_kijiji",
|
||||||
|
description: "Search Kijiji marketplace for listings matching a query",
|
||||||
|
inputSchema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
query: {
|
||||||
|
type: "string",
|
||||||
|
description: "Search query for Kijiji listings",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Location name or ID (e.g., 'toronto', 'gta', 'ontario')",
|
||||||
|
},
|
||||||
|
category: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Category name or ID (e.g., 'computers', 'furniture', 'bikes')",
|
||||||
|
},
|
||||||
|
keywords: {
|
||||||
|
type: "string",
|
||||||
|
description: "Additional keywords to filter results",
|
||||||
|
},
|
||||||
|
sortBy: {
|
||||||
|
type: "string",
|
||||||
|
description: "Sort results by field",
|
||||||
|
enum: ["relevancy", "date", "price", "distance"],
|
||||||
|
default: "relevancy",
|
||||||
|
},
|
||||||
|
sortOrder: {
|
||||||
|
type: "string",
|
||||||
|
description: "Sort order",
|
||||||
|
enum: ["asc", "desc"],
|
||||||
|
default: "desc",
|
||||||
|
},
|
||||||
|
maxPages: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum pages to fetch (~40 items per page)",
|
||||||
|
default: 5,
|
||||||
|
},
|
||||||
|
priceMin: {
|
||||||
|
type: "number",
|
||||||
|
description: "Minimum price in cents",
|
||||||
|
},
|
||||||
|
priceMax: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum price in cents",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "search_facebook",
|
||||||
|
description: "Search Facebook Marketplace for listings matching a query",
|
||||||
|
inputSchema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
query: {
|
||||||
|
type: "string",
|
||||||
|
description: "Search query for Facebook Marketplace listings",
|
||||||
|
},
|
||||||
|
location: {
|
||||||
|
type: "string",
|
||||||
|
description: "Location for search (e.g., 'toronto')",
|
||||||
|
default: "toronto",
|
||||||
|
},
|
||||||
|
maxItems: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum number of items to return",
|
||||||
|
default: 5,
|
||||||
|
},
|
||||||
|
cookiesSource: {
|
||||||
|
type: "string",
|
||||||
|
description: "Optional Facebook session cookies source",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "search_ebay",
|
||||||
|
description:
|
||||||
|
"Search eBay for listings matching a query (default: Buy It Now only, Canada only)",
|
||||||
|
inputSchema: {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
query: {
|
||||||
|
type: "string",
|
||||||
|
description: "Search query for eBay listings",
|
||||||
|
},
|
||||||
|
minPrice: {
|
||||||
|
type: "number",
|
||||||
|
description: "Minimum price filter",
|
||||||
|
},
|
||||||
|
maxPrice: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum price filter",
|
||||||
|
},
|
||||||
|
strictMode: {
|
||||||
|
type: "boolean",
|
||||||
|
description: "Enable strict search mode",
|
||||||
|
default: false,
|
||||||
|
},
|
||||||
|
exclusions: {
|
||||||
|
type: "array",
|
||||||
|
items: { type: "string" },
|
||||||
|
description: "Terms to exclude from results",
|
||||||
|
},
|
||||||
|
keywords: {
|
||||||
|
type: "array",
|
||||||
|
items: { type: "string" },
|
||||||
|
description: "Keywords to include in search",
|
||||||
|
},
|
||||||
|
buyItNowOnly: {
|
||||||
|
type: "boolean",
|
||||||
|
description: "Include only Buy It Now listings (exclude auctions)",
|
||||||
|
default: true,
|
||||||
|
},
|
||||||
|
canadaOnly: {
|
||||||
|
type: "boolean",
|
||||||
|
description: "Include only Canadian sellers/listings",
|
||||||
|
default: true,
|
||||||
|
},
|
||||||
|
maxItems: {
|
||||||
|
type: "number",
|
||||||
|
description: "Maximum number of items to return",
|
||||||
|
default: 5,
|
||||||
|
},
|
||||||
|
cookies: {
|
||||||
|
type: "string",
|
||||||
|
description:
|
||||||
|
"Optional: eBay session cookies to bypass bot detection (format: 'name1=value1; name2=value2')",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
13
packages/mcp-server/tsconfig.json
Normal file
13
packages/mcp-server/tsconfig.json
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
},
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true
|
||||||
|
}
|
||||||
|
}
|
||||||
26
scripts/biome-symlink.sh
Executable file
26
scripts/biome-symlink.sh
Executable file
@@ -0,0 +1,26 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# Get the path to the system biome executable
|
||||||
|
BIOME_PATH=$(which biome)
|
||||||
|
|
||||||
|
if [ -z "$BIOME_PATH" ]; then
|
||||||
|
echo "Error: biome executable not found in PATH"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Find all biome executables in node_modules
|
||||||
|
files=$(fd biome node_modules --type executable --no-ignore --follow)
|
||||||
|
|
||||||
|
if [ -z "$files" ]; then
|
||||||
|
echo "No biome executables found in node_modules"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Replace each with a symlink to the system biome
|
||||||
|
for file in $files; do
|
||||||
|
echo "Replacing $file with symlink to $BIOME_PATH"
|
||||||
|
rm "$file"
|
||||||
|
ln -s "$BIOME_PATH" "$file"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Done."
|
||||||
30
scripts/remove-eslint.sh
Executable file
30
scripts/remove-eslint.sh
Executable file
@@ -0,0 +1,30 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
PATTERN="eslint"
|
||||||
|
FILES="$(fd .)" # Or use 'find .' to search recursively
|
||||||
|
|
||||||
|
for file in $FILES; do
|
||||||
|
if [[ -f "$file" ]]; then
|
||||||
|
# 1. Use rg with line numbers (-n) and only the matched line (-o)
|
||||||
|
# 2. Use awk to print ONLY the line number (field 1)
|
||||||
|
# 3. Use xargs to pass multiple line numbers to a single sed command
|
||||||
|
|
||||||
|
LINE_NUMBERS=$(rg --line-number --no-filename "$PATTERN" "$file" | awk -F':' '{print $1}' | tr '\n' ',')
|
||||||
|
|
||||||
|
# Remove trailing comma if any
|
||||||
|
LINE_NUMBERS=${LINE_NUMBERS%,}
|
||||||
|
|
||||||
|
if [[ -n "$LINE_NUMBERS" ]]; then
|
||||||
|
echo "Deleting lines $LINE_NUMBERS from $file..."
|
||||||
|
|
||||||
|
# Use sed to delete the specified comma-separated line numbers in-place (-i)
|
||||||
|
# NOTE: The syntax for -i might vary slightly between GNU sed (Linux) and BSD sed (macOS).
|
||||||
|
sed -i.bak "${LINE_NUMBERS}d" "$file"
|
||||||
|
|
||||||
|
# Optional: Remove the backup file created by sed -i.bak
|
||||||
|
# rm "${file}.bak"
|
||||||
|
else
|
||||||
|
echo "$file: No lines matching pattern found."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
25
scripts/start.sh
Executable file
25
scripts/start.sh
Executable file
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Trap SIGTERM and SIGINT for graceful shutdown
|
||||||
|
trap 'echo "Received shutdown signal, stopping services..."; kill -TERM $API_PID $MCP_PID 2>/dev/null; wait' TERM INT
|
||||||
|
|
||||||
|
# Start API Server in background
|
||||||
|
echo "Starting API Server on port ${API_PORT:-4005}..."
|
||||||
|
bun dist/api/index.js &
|
||||||
|
API_PID=$!
|
||||||
|
|
||||||
|
# Give API server a moment to initialize
|
||||||
|
sleep 1
|
||||||
|
|
||||||
|
# Start MCP Server in background
|
||||||
|
echo "Starting MCP Server on port ${API_PORT:-4006}..."
|
||||||
|
bun dist/mcp/index.js &
|
||||||
|
MCP_PID=$!
|
||||||
|
|
||||||
|
echo "Both services started successfully"
|
||||||
|
echo "API Server PID: $API_PID"
|
||||||
|
echo "MCP Server PID: $MCP_PID"
|
||||||
|
|
||||||
|
# Wait for both processes
|
||||||
|
wait $API_PID $MCP_PID
|
||||||
215
src/index.ts
215
src/index.ts
@@ -1,215 +0,0 @@
|
|||||||
import fetchEbayItems from "@/ebay";
|
|
||||||
import fetchFacebookItems from "@/facebook";
|
|
||||||
import fetchKijijiItems from "@/kijiji";
|
|
||||||
|
|
||||||
const PORT = process.env.PORT || 4005;
|
|
||||||
|
|
||||||
const server = Bun.serve({
|
|
||||||
port: PORT,
|
|
||||||
idleTimeout: 0,
|
|
||||||
routes: {
|
|
||||||
// Static routes
|
|
||||||
"/api/status": new Response("OK"),
|
|
||||||
|
|
||||||
// Dynamic routes
|
|
||||||
"/api/kijiji": async (req: Request) => {
|
|
||||||
const reqUrl = new URL(req.url);
|
|
||||||
|
|
||||||
const SEARCH_QUERY =
|
|
||||||
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
|
||||||
if (!SEARCH_QUERY)
|
|
||||||
return Response.json(
|
|
||||||
{
|
|
||||||
message:
|
|
||||||
"Request didn't have 'query' header or 'q' search parameter!",
|
|
||||||
},
|
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
|
|
||||||
// Parse optional parameters with enhanced defaults
|
|
||||||
const location = reqUrl.searchParams.get("location");
|
|
||||||
const category = reqUrl.searchParams.get("category");
|
|
||||||
const maxPagesParam = reqUrl.searchParams.get("maxPages");
|
|
||||||
const maxPages = maxPagesParam ? Number.parseInt(maxPagesParam, 10) : 5; // Default: 5 pages
|
|
||||||
const sortBy = reqUrl.searchParams.get("sortBy") as
|
|
||||||
| "relevancy"
|
|
||||||
| "date"
|
|
||||||
| "price"
|
|
||||||
| "distance"
|
|
||||||
| undefined;
|
|
||||||
const sortOrder = reqUrl.searchParams.get("sortOrder") as
|
|
||||||
| "asc"
|
|
||||||
| "desc"
|
|
||||||
| undefined;
|
|
||||||
|
|
||||||
// Build search options
|
|
||||||
const locationValue = location
|
|
||||||
? /^\d+$/.test(location)
|
|
||||||
? Number(location)
|
|
||||||
: location
|
|
||||||
: 1700272;
|
|
||||||
const categoryValue = category
|
|
||||||
? /^\d+$/.test(category)
|
|
||||||
? Number(category)
|
|
||||||
: category
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
const searchOptions: import("@/kijiji").SearchOptions = {
|
|
||||||
location: locationValue,
|
|
||||||
category: categoryValue,
|
|
||||||
keywords: SEARCH_QUERY,
|
|
||||||
sortBy: sortBy || "relevancy",
|
|
||||||
sortOrder: sortOrder || "desc",
|
|
||||||
maxPages,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Build listing fetch options with enhanced defaults
|
|
||||||
const listingOptions: import("@/kijiji").ListingFetchOptions = {
|
|
||||||
includeImages: true, // Always include full image arrays
|
|
||||||
sellerDataDepth: "detailed", // Default: detailed seller info
|
|
||||||
includeClientSideData: false, // GraphQL reviews disabled by default
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
const items = await fetchKijijiItems(
|
|
||||||
SEARCH_QUERY,
|
|
||||||
1,
|
|
||||||
undefined,
|
|
||||||
searchOptions,
|
|
||||||
listingOptions,
|
|
||||||
);
|
|
||||||
if (!items || items.length === 0)
|
|
||||||
return Response.json(
|
|
||||||
{ message: "Search didn't return any results!" },
|
|
||||||
{ status: 404 },
|
|
||||||
);
|
|
||||||
return Response.json(items, { status: 200 });
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Kijiji scraping error:", error);
|
|
||||||
const errorMessage =
|
|
||||||
error instanceof Error ? error.message : "Unknown error occurred";
|
|
||||||
return Response.json(
|
|
||||||
{
|
|
||||||
message: `Scraping failed: ${errorMessage}`,
|
|
||||||
query: SEARCH_QUERY,
|
|
||||||
options: { searchOptions, listingOptions },
|
|
||||||
},
|
|
||||||
{ status: 500 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
"/api/facebook": async (req: Request) => {
|
|
||||||
const reqUrl = new URL(req.url);
|
|
||||||
|
|
||||||
const SEARCH_QUERY =
|
|
||||||
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
|
||||||
if (!SEARCH_QUERY)
|
|
||||||
return Response.json(
|
|
||||||
{
|
|
||||||
message:
|
|
||||||
"Request didn't have 'query' header or 'q' search parameter!",
|
|
||||||
},
|
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
|
|
||||||
const LOCATION = reqUrl.searchParams.get("location") || "toronto";
|
|
||||||
const COOKIES_SOURCE = reqUrl.searchParams.get("cookies") || undefined;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const items = await fetchFacebookItems(
|
|
||||||
SEARCH_QUERY,
|
|
||||||
5,
|
|
||||||
LOCATION,
|
|
||||||
25,
|
|
||||||
COOKIES_SOURCE,
|
|
||||||
"./cookies/facebook.json",
|
|
||||||
);
|
|
||||||
if (!items || items.length === 0)
|
|
||||||
return Response.json(
|
|
||||||
{ message: "Search didn't return any results!" },
|
|
||||||
{ status: 404 },
|
|
||||||
);
|
|
||||||
return Response.json(items, { status: 200 });
|
|
||||||
} catch (error) {
|
|
||||||
console.error("Facebook scraping error:", error);
|
|
||||||
const errorMessage =
|
|
||||||
error instanceof Error ? error.message : "Unknown error occurred";
|
|
||||||
return Response.json({ message: errorMessage }, { status: 400 });
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
"/api/ebay": async (req: Request) => {
|
|
||||||
const reqUrl = new URL(req.url);
|
|
||||||
|
|
||||||
const SEARCH_QUERY =
|
|
||||||
req.headers.get("query") || reqUrl.searchParams.get("q") || null;
|
|
||||||
if (!SEARCH_QUERY)
|
|
||||||
return Response.json(
|
|
||||||
{
|
|
||||||
message:
|
|
||||||
"Request didn't have 'query' header or 'q' search parameter!",
|
|
||||||
},
|
|
||||||
{ status: 400 },
|
|
||||||
);
|
|
||||||
|
|
||||||
// Parse optional parameters with defaults
|
|
||||||
const minPriceParam = reqUrl.searchParams.get("minPrice");
|
|
||||||
const minPrice = minPriceParam
|
|
||||||
? Number.parseInt(minPriceParam, 10)
|
|
||||||
: undefined;
|
|
||||||
const maxPriceParam = reqUrl.searchParams.get("maxPrice");
|
|
||||||
const maxPrice = maxPriceParam
|
|
||||||
? Number.parseInt(maxPriceParam, 10)
|
|
||||||
: undefined;
|
|
||||||
const strictMode = reqUrl.searchParams.get("strictMode") === "true";
|
|
||||||
const exclusionsParam = reqUrl.searchParams.get("exclusions");
|
|
||||||
const exclusions = exclusionsParam
|
|
||||||
? exclusionsParam.split(",").map((s) => s.trim())
|
|
||||||
: [];
|
|
||||||
const keywordsParam = reqUrl.searchParams.get("keywords");
|
|
||||||
const keywords = keywordsParam
|
|
||||||
? keywordsParam.split(",").map((s) => s.trim())
|
|
||||||
: [SEARCH_QUERY];
|
|
||||||
|
|
||||||
try {
|
|
||||||
const items = await fetchEbayItems(SEARCH_QUERY, 5, {
|
|
||||||
minPrice,
|
|
||||||
maxPrice,
|
|
||||||
strictMode,
|
|
||||||
exclusions,
|
|
||||||
keywords,
|
|
||||||
});
|
|
||||||
if (!items || items.length === 0)
|
|
||||||
return Response.json(
|
|
||||||
{ message: "Search didn't return any results!" },
|
|
||||||
{ status: 404 },
|
|
||||||
);
|
|
||||||
return Response.json(items, { status: 200 });
|
|
||||||
} catch (error) {
|
|
||||||
console.error("eBay scraping error:", error);
|
|
||||||
const errorMessage =
|
|
||||||
error instanceof Error ? error.message : "Unknown error occurred";
|
|
||||||
return Response.json({ message: errorMessage }, { status: 400 });
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
// Wildcard route for all routes that start with "/api/" and aren't otherwise matched
|
|
||||||
"/api/*": Response.json({ message: "Not found" }, { status: 404 }),
|
|
||||||
|
|
||||||
// // Serve a file by buffering it in memory
|
|
||||||
// "/favicon.ico": new Response(await Bun.file("./favicon.ico").bytes(), {
|
|
||||||
// headers: {
|
|
||||||
// "Content-Type": "image/x-icon",
|
|
||||||
// },
|
|
||||||
// }),
|
|
||||||
},
|
|
||||||
|
|
||||||
// (optional) fallback for unmatched routes:
|
|
||||||
// Required if Bun's version < 1.2.3
|
|
||||||
fetch(req: Request) {
|
|
||||||
return new Response("Not Found", { status: 404 });
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Serving on ${server.hostname}:${server.port}`);
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
// Environment setup & latest features
|
|
||||||
"lib": ["dom"],
|
|
||||||
"target": "ESNext",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleDetection": "force",
|
|
||||||
"jsx": "react-jsx",
|
|
||||||
"allowJs": true,
|
|
||||||
// Bundler mode
|
|
||||||
"moduleResolution": "bundler",
|
|
||||||
"allowImportingTsExtensions": true,
|
|
||||||
"verbatimModuleSyntax": true,
|
|
||||||
"noEmit": true,
|
|
||||||
// Best practices
|
|
||||||
"strict": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"noFallthroughCasesInSwitch": true,
|
|
||||||
"noUncheckedIndexedAccess": true,
|
|
||||||
"noImplicitAny": true,
|
|
||||||
// Some stricter flags (disabled by default)
|
|
||||||
"noUnusedLocals": false,
|
|
||||||
"noUnusedParameters": false,
|
|
||||||
"noPropertyAccessFromIndexSignature": false,
|
|
||||||
"paths": {
|
|
||||||
"@/*": ["./src/*"]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
|
||||||
"exclude": ["node_modules"]
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user