Compare commits
12 Commits
49e90d45f8
...
d77a006ded
| Author | SHA1 | Date | |
|---|---|---|---|
| d77a006ded | |||
| 56b2198df1 | |||
| 63716272c5 | |||
| 1d21c66945 | |||
| f2f78225f3 | |||
| 43d15fce5f | |||
| fef2f1968a | |||
| 01081f6b2e | |||
| d10d5305a3 | |||
| bf393eacae | |||
| 79bb249603 | |||
| 957e0f137b |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -33,6 +33,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
.eslintcache
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
.turbo
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
58
bun.lock
58
bun.lock
@@ -6,6 +6,8 @@
|
||||
"name": "marketplace-scrapers-monorepo",
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.3.11",
|
||||
"@tsconfig/bun": "catalog:",
|
||||
"turbo": "2.5.4",
|
||||
},
|
||||
},
|
||||
"packages/api-server": {
|
||||
@@ -13,9 +15,10 @@
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@marketplace-scrapers/core": "workspace:*",
|
||||
"@typescript/native-preview": "catalog:",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/bun": "catalog:",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5",
|
||||
@@ -25,14 +28,15 @@
|
||||
"name": "@marketplace-scrapers/core",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@typescript/native-preview": "catalog:",
|
||||
"cli-progress": "^3.12.0",
|
||||
"linkedom": "^0.18.12",
|
||||
"unidecode": "^1.1.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/cli-progress": "^3.11.6",
|
||||
"@types/unidecode": "^1.1.0",
|
||||
"@types/bun": "catalog:",
|
||||
"@types/cli-progress": "catalog:",
|
||||
"@types/unidecode": "catalog:",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5",
|
||||
@@ -43,15 +47,23 @@
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@marketplace-scrapers/core": "workspace:*",
|
||||
"@typescript/native-preview": "catalog:",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/bun": "catalog:",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5",
|
||||
},
|
||||
},
|
||||
},
|
||||
"catalog": {
|
||||
"@tsconfig/bun": "1.0.9",
|
||||
"@types/bun": "1.3.13",
|
||||
"@types/cli-progress": "3.11.6",
|
||||
"@types/unidecode": "1.1.0",
|
||||
"@typescript/native-preview": "7.0.0-dev.20260428.1",
|
||||
},
|
||||
"packages": {
|
||||
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="],
|
||||
|
||||
@@ -77,7 +89,9 @@
|
||||
|
||||
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="],
|
||||
"@tsconfig/bun": ["@tsconfig/bun@1.0.9", "", {}, "sha512-4M0/Ivfwcpz325z6CwSifOBZYji3DFOEpY6zEUt0+Xi2qRhzwvmqQN9XAHJh3OVvRJuAqVTLU2abdCplvp6mwQ=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="],
|
||||
|
||||
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
|
||||
|
||||
@@ -85,11 +99,27 @@
|
||||
|
||||
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
||||
|
||||
"@typescript/native-preview": ["@typescript/native-preview@7.0.0-dev.20260428.1", "", { "optionalDependencies": { "@typescript/native-preview-darwin-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-darwin-x64": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-arm": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-x64": "7.0.0-dev.20260428.1", "@typescript/native-preview-win32-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-win32-x64": "7.0.0-dev.20260428.1" }, "bin": { "tsgo": "bin/tsgo.js" } }, "sha512-JiM4PYWDGs57TT0mV2KArmaW7BnTkk3XRid79NdG17tfvDbRyg4hBCpKI7vARiQPtxjKrHlxyzxOGDpv5W5T7Q=="],
|
||||
|
||||
"@typescript/native-preview-darwin-arm64": ["@typescript/native-preview-darwin-arm64@7.0.0-dev.20260428.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Lll6WmXfgTEj1G3QBIoHlabQwUtJiyhlRgSLksa06QFL5BoA7V+Lu1waa9PtPNZbGsXLDMHodtk/bRQABKuPiw=="],
|
||||
|
||||
"@typescript/native-preview-darwin-x64": ["@typescript/native-preview-darwin-x64@7.0.0-dev.20260428.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-WbsBNSHlo+4sGrTxDWdmI7r8x48tCtSCuKdmK62FvVOq58UWAs6sL13Z4Rev4ohLcGHdXC5E/8AIdpLPqDYQpw=="],
|
||||
|
||||
"@typescript/native-preview-linux-arm": ["@typescript/native-preview-linux-arm@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "arm" }, "sha512-/d/NnZFvEJU67L5mHh+cO3gsfwNCvJ9HGtxGq1KGz1VwTabOIcwLdpTpfsAR39WXzzfh9GJHL28n6GSGZInPow=="],
|
||||
|
||||
"@typescript/native-preview-linux-arm64": ["@typescript/native-preview-linux-arm64@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-cgcBX/ZBMdepkamLT8g8jQdHe7DZS/s6zTZRof6mvcrnJHlMeUnKoC9UO8/c22IrUMV3n0XPh7R8FYjUP0ll+Q=="],
|
||||
|
||||
"@typescript/native-preview-linux-x64": ["@typescript/native-preview-linux-x64@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "x64" }, "sha512-4gJCE7wzenx1BH2Vtx2uKWUo8rFxnhGkxNEH1zxbYy/6ASwo+PnOPYmKHAzNE1C3yB5lzw71/vR5p5zyO57Y4A=="],
|
||||
|
||||
"@typescript/native-preview-win32-arm64": ["@typescript/native-preview-win32-arm64@7.0.0-dev.20260428.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-yn6Rzbn62L4QTWrp0QgG8al6l/VG7PCPRdbE0vuGDSlKhInlC+Flo4QSc1qA8KHTbpHgl+nEsq9DymiitI4G4g=="],
|
||||
|
||||
"@typescript/native-preview-win32-x64": ["@typescript/native-preview-win32-x64@7.0.0-dev.20260428.1", "", { "os": "win32", "cpu": "x64" }, "sha512-T9z13mcMowXmwGjprA2FIR2EEdYZxgqH8+qk7dFZVBlo5vfk41AN/qJfAdN7IsAhEb640MJ8cMN/aiczweZKmA=="],
|
||||
|
||||
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-5ua817+BZPZOlNaRgGBpZJOSAQ9RQ17pkwPD0yR7CfJg+r8DgIILByFifDTa+IPDDxzf5VNhtNlcKqFzDgJvlQ=="],
|
||||
"bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],
|
||||
|
||||
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
||||
|
||||
@@ -125,6 +155,20 @@
|
||||
|
||||
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
||||
|
||||
"turbo": ["turbo@2.5.4", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.4", "turbo-darwin-arm64": "2.5.4", "turbo-linux-64": "2.5.4", "turbo-linux-arm64": "2.5.4", "turbo-windows-64": "2.5.4", "turbo-windows-arm64": "2.5.4" }, "bin": { "turbo": "bin/turbo" } }, "sha512-kc8ZibdRcuWUG1pbYSBFWqmIjynlD8Lp7IB6U3vIzvOv9VG+6Sp8bzyeBWE3Oi8XV5KsQrznyRTBPvrf99E4mA=="],
|
||||
|
||||
"turbo-darwin-64": ["turbo-darwin-64@2.5.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-ah6YnH2dErojhFooxEzmvsoZQTMImaruZhFPfMKPBq8sb+hALRdvBNLqfc8NWlZq576FkfRZ/MSi4SHvVFT9PQ=="],
|
||||
|
||||
"turbo-darwin-arm64": ["turbo-darwin-arm64@2.5.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2+Nx6LAyuXw2MdXb7pxqle3MYignLvS7OwtsP9SgtSBaMlnNlxl9BovzqdYAgkUW3AsYiQMJ/wBRb7d+xemM5A=="],
|
||||
|
||||
"turbo-linux-64": ["turbo-linux-64@2.5.4", "", { "os": "linux", "cpu": "x64" }, "sha512-5May2kjWbc8w4XxswGAl74GZ5eM4Gr6IiroqdLhXeXyfvWEdm2mFYCSWOzz0/z5cAgqyGidF1jt1qzUR8hTmOA=="],
|
||||
|
||||
"turbo-linux-arm64": ["turbo-linux-arm64@2.5.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-/2yqFaS3TbfxV3P5yG2JUI79P7OUQKOUvAnx4MV9Bdz6jqHsHwc9WZPpO4QseQm+NvmgY6ICORnoVPODxGUiJg=="],
|
||||
|
||||
"turbo-windows-64": ["turbo-windows-64@2.5.4", "", { "os": "win32", "cpu": "x64" }, "sha512-EQUO4SmaCDhO6zYohxIjJpOKRN3wlfU7jMAj3CgcyTPvQR/UFLEKAYHqJOnJtymbQmiiM/ihX6c6W6Uq0yC7mA=="],
|
||||
|
||||
"turbo-windows-arm64": ["turbo-windows-arm64@2.5.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-oQ8RrK1VS8lrxkLriotFq+PiF7iiGgkZtfLKF4DDKsmdbPo0O9R2mQxm7jHLuXraRCuIQDWMIw6dpcr7Iykf4A=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||
|
||||
5
bunfig.toml
Normal file
5
bunfig.toml
Normal file
@@ -0,0 +1,5 @@
|
||||
[install]
|
||||
exact = true
|
||||
|
||||
[test]
|
||||
root = "./do-not-run-tests-from-root"
|
||||
477
docs/superpowers/plans/2025-07-14-opencode-monorepo-config.md
Normal file
477
docs/superpowers/plans/2025-07-14-opencode-monorepo-config.md
Normal file
@@ -0,0 +1,477 @@
|
||||
# opencode Monorepo Config Adoption Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Adopt opencode-style monorepo config: Turbo task orchestration, workspace dep catalog, shared root tsconfig, bunfig.toml, and `exports` field in all packages.
|
||||
|
||||
**Architecture:** Pure config changes across 10 files — no source code touched. Root config files are added/updated first, then per-package files updated to reference them. Changes are independent within each task and safe to commit atomically.
|
||||
|
||||
**Tech Stack:** Bun workspaces, Turbo 2.x, @tsconfig/bun, TypeScript (tsgo / @typescript/native-preview)
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
| File | Action | Responsible for |
|
||||
|---|---|---|
|
||||
| `package.json` | Modify | Workspace catalog, turbo devDep, @tsconfig/bun devDep, updated scripts |
|
||||
| `turbo.json` | Create | Task graph: typecheck, build, test |
|
||||
| `tsconfig.json` | Create | Shared TS compiler options for all packages |
|
||||
| `bunfig.toml` | Create | Exact installs, root test guard |
|
||||
| `packages/core/package.json` | Modify | exports field, catalog refs, script rename |
|
||||
| `packages/api-server/package.json` | Modify | exports field, catalog refs, script rename |
|
||||
| `packages/mcp-server/package.json` | Modify | exports field, catalog refs, script rename |
|
||||
| `packages/core/tsconfig.json` | Modify | Slim — extends root, paths only |
|
||||
| `packages/api-server/tsconfig.json` | Modify | Slim — extends root, paths only |
|
||||
| `packages/mcp-server/tsconfig.json` | Modify | Slim — extends root, paths only |
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Add `bunfig.toml` and `turbo.json`
|
||||
|
||||
Two new root config files with no dependencies on other tasks.
|
||||
|
||||
**Files:**
|
||||
- Create: `bunfig.toml`
|
||||
- Create: `turbo.json`
|
||||
|
||||
- [ ] **Step 1: Create `bunfig.toml`**
|
||||
|
||||
Write this file at repo root (`/path/to/ca-marketplace-scraper/bunfig.toml`):
|
||||
|
||||
```toml
|
||||
[install]
|
||||
exact = true
|
||||
|
||||
[test]
|
||||
root = "./do-not-run-tests-from-root"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Create `turbo.json`**
|
||||
|
||||
Write this file at repo root:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://turbo.build/schema.json",
|
||||
"tasks": {
|
||||
"typecheck": {},
|
||||
"build": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": ["dist/**"]
|
||||
},
|
||||
"test": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify files exist**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
ls bunfig.toml turbo.json
|
||||
```
|
||||
Expected: both files listed, no errors.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add bunfig.toml turbo.json
|
||||
git commit -m "chore: add bunfig.toml and turbo.json"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Create root `tsconfig.json`
|
||||
|
||||
Shared base tsconfig all packages will extend. Extracts the common options currently duplicated in all 3 per-package tsconfigs.
|
||||
|
||||
**Files:**
|
||||
- Create: `tsconfig.json`
|
||||
|
||||
- [ ] **Step 1: Create root `tsconfig.json`**
|
||||
|
||||
Write this file at repo root:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "@tsconfig/bun/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["dom", "ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "preserve",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Commit**
|
||||
|
||||
```bash
|
||||
git add tsconfig.json
|
||||
git commit -m "chore: add shared root tsconfig.json"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Update root `package.json`
|
||||
|
||||
Add workspace catalog, `turbo` + `@tsconfig/bun` devDependencies, and update scripts to use `turbo run`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `package.json`
|
||||
|
||||
- [ ] **Step 1: Replace root `package.json`**
|
||||
|
||||
Write this complete file:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "marketplace-scrapers-monorepo",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"packageManager": "bun@1.3.13",
|
||||
"scripts": {
|
||||
"typecheck": "turbo run typecheck",
|
||||
"build": "bun run clean && turbo run build",
|
||||
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
||||
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
||||
"build:all": "bun run build:api && bun run build:mcp",
|
||||
"ci": "biome ci",
|
||||
"clean": "rm -rf dist",
|
||||
"start": "./scripts/start.sh"
|
||||
},
|
||||
"workspaces": {
|
||||
"packages": [
|
||||
"packages/*"
|
||||
],
|
||||
"catalog": {
|
||||
"@tsconfig/bun": "1.0.9",
|
||||
"@typescript/native-preview": "7.0.0-dev.20260428.1",
|
||||
"@types/bun": "1.2.18",
|
||||
"@types/cli-progress": "3.11.6",
|
||||
"@types/unidecode": "1.1.0"
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.3.11",
|
||||
"@tsconfig/bun": "catalog:",
|
||||
"turbo": "2.5.4"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> **Note on catalog versions:** The catalog pins exact versions. The values above are taken from the current package installs. If `@types/bun` was `latest`, check `node_modules/@types/bun/package.json` for the actual installed version and use that. Same for `@typescript/native-preview`.
|
||||
|
||||
- [ ] **Step 2: Check actual installed versions**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cat node_modules/@types/bun/package.json | grep '"version"'
|
||||
cat node_modules/@typescript/native-preview/package.json | grep '"version"'
|
||||
cat node_modules/@types/cli-progress/package.json | grep '"version"'
|
||||
cat node_modules/@types/unidecode/package.json | grep '"version"'
|
||||
```
|
||||
|
||||
Update the catalog values in `package.json` to match the exact installed versions.
|
||||
|
||||
- [ ] **Step 3: Install turbo and @tsconfig/bun**
|
||||
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
|
||||
Expected: lock file updated, `turbo` and `@tsconfig/bun` appear in `node_modules`.
|
||||
|
||||
- [ ] **Step 4: Verify turbo works**
|
||||
|
||||
```bash
|
||||
bunx turbo run typecheck --dry
|
||||
```
|
||||
|
||||
Expected: output lists the `typecheck` task for each package (even if no `typecheck` script exists yet — turbo will note them as skipped/missing).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add package.json bun.lock
|
||||
git commit -m "chore: add workspace catalog and turbo to root package.json"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Update per-package `package.json` files
|
||||
|
||||
Rename `type:check` → `typecheck`, replace `main`/`module` with `exports`, swap pinned dep versions for `catalog:` references.
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/core/package.json`
|
||||
- Modify: `packages/api-server/package.json`
|
||||
- Modify: `packages/mcp-server/package.json`
|
||||
|
||||
- [ ] **Step 1: Replace `packages/core/package.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@marketplace-scrapers/core",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"typecheck": "bun tsgo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@typescript/native-preview": "catalog:",
|
||||
"cli-progress": "^3.12.0",
|
||||
"linkedom": "^0.18.12",
|
||||
"unidecode": "^1.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "catalog:",
|
||||
"@types/cli-progress": "catalog:",
|
||||
"@types/unidecode": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Replace `packages/api-server/package.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@marketplace-scrapers/api-server",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "bun ./src/index.ts",
|
||||
"dev": "bun --watch ./src/index.ts",
|
||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api",
|
||||
"typecheck": "bun tsgo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@marketplace-scrapers/core": "workspace:*",
|
||||
"@typescript/native-preview": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Replace `packages/mcp-server/package.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@marketplace-scrapers/mcp-server",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "bun ./src/index.ts",
|
||||
"dev": "bun --watch ./src/index.ts",
|
||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp",
|
||||
"typecheck": "bun tsgo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@marketplace-scrapers/core": "workspace:*",
|
||||
"@typescript/native-preview": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run `bun install` to sync lockfile**
|
||||
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
|
||||
Expected: no errors. Catalog refs resolved. `bun.lock` updated.
|
||||
|
||||
- [ ] **Step 5: Verify typecheck still works per-package**
|
||||
|
||||
```bash
|
||||
cd packages/core && bun run typecheck
|
||||
cd ../api-server && bun run typecheck
|
||||
cd ../mcp-server && bun run typecheck
|
||||
cd ../..
|
||||
```
|
||||
|
||||
Expected: each exits 0 (or same errors as before — no new errors introduced).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/core/package.json packages/api-server/package.json packages/mcp-server/package.json bun.lock
|
||||
git commit -m "chore: use exports field and catalog refs in all packages"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Slim per-package `tsconfig.json` files
|
||||
|
||||
Replace the duplicated full tsconfig in each package with a slim `extends`-based one pointing to root.
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/core/tsconfig.json`
|
||||
- Modify: `packages/api-server/tsconfig.json`
|
||||
- Modify: `packages/mcp-server/tsconfig.json`
|
||||
|
||||
- [ ] **Step 1: Replace `packages/core/tsconfig.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Replace `packages/api-server/tsconfig.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Replace `packages/mcp-server/tsconfig.json`**
|
||||
|
||||
```json
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify `@tsconfig/bun` is resolvable**
|
||||
|
||||
The root tsconfig extends `@tsconfig/bun/tsconfig.json`. Confirm the package is installed:
|
||||
|
||||
```bash
|
||||
ls node_modules/@tsconfig/bun/tsconfig.json
|
||||
```
|
||||
|
||||
Expected: file exists.
|
||||
|
||||
- [ ] **Step 5: Run typecheck via Turbo**
|
||||
|
||||
```bash
|
||||
bun run typecheck
|
||||
```
|
||||
|
||||
Expected: Turbo runs `typecheck` for all 3 packages in parallel, all pass (or same pre-existing errors — no new ones).
|
||||
|
||||
- [ ] **Step 6: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/core/tsconfig.json packages/api-server/tsconfig.json packages/mcp-server/tsconfig.json
|
||||
git commit -m "chore: slim per-package tsconfigs to extend root"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Smoke test full build pipeline
|
||||
|
||||
Verify everything works end-to-end.
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Run turbo typecheck**
|
||||
|
||||
```bash
|
||||
bun run typecheck
|
||||
```
|
||||
|
||||
Expected: Turbo runs `typecheck` across all packages. Exit 0.
|
||||
|
||||
- [ ] **Step 2: Run full build**
|
||||
|
||||
```bash
|
||||
bun run build
|
||||
```
|
||||
|
||||
Expected: `dist/` cleaned, Turbo runs `build` (core first, then api-server and mcp-server in parallel), build artifacts appear in `dist/api/` and `dist/mcp/`.
|
||||
|
||||
- [ ] **Step 3: Verify dist artifacts**
|
||||
|
||||
```bash
|
||||
ls dist/api/ dist/mcp/
|
||||
```
|
||||
|
||||
Expected: compiled output files in both directories.
|
||||
|
||||
- [ ] **Step 4: Verify `bun install` is exact**
|
||||
|
||||
```bash
|
||||
grep -c '\^' bun.lock | head -5
|
||||
```
|
||||
|
||||
With `exact = true` in bunfig.toml, new installs won't add `^` ranges. Existing `^` ranges in `bun.lock` from before are fine — they'll be resolved to exact on next fresh install.
|
||||
|
||||
- [ ] **Step 5: Final commit if any loose files**
|
||||
|
||||
```bash
|
||||
git status
|
||||
```
|
||||
|
||||
If clean: done. If any files modified by `bun install` (e.g. `bun.lock`):
|
||||
|
||||
```bash
|
||||
git add bun.lock
|
||||
git commit -m "chore: sync lockfile after monorepo config adoption"
|
||||
```
|
||||
@@ -0,0 +1,136 @@
|
||||
# Design: Adopt opencode Monorepo Config
|
||||
|
||||
**Date:** 2025-07-14
|
||||
**Status:** Approved
|
||||
**Approach:** Full adoption (A)
|
||||
|
||||
## Context
|
||||
|
||||
Current repo (`marketplace-scrapers-monorepo`) has basic bun workspaces with 3 packages (`core`, `api-server`, `mcp-server`). Reference: `anomalyco/opencode` monorepo patterns.
|
||||
|
||||
**Gaps vs opencode:**
|
||||
- No Turbo (task orchestration, caching, dep graph)
|
||||
- No workspace catalog (shared dep versions duplicated across packages)
|
||||
- No root tsconfig (identical tsconfigs duplicated in all 3 packages)
|
||||
- No `bunfig.toml` (no exact installs, no root test guard)
|
||||
- `main`/`module` fields instead of `exports` field
|
||||
|
||||
## Changes
|
||||
|
||||
### 1. Root `package.json`
|
||||
|
||||
- Add `workspaces.catalog` block with shared deps:
|
||||
- `@typescript/native-preview`, `@types/bun`, `@types/unidecode`, `@types/cli-progress`
|
||||
- Add `turbo` to `devDependencies`
|
||||
- Add `@tsconfig/bun` to `devDependencies` + catalog
|
||||
- Update root scripts: `typecheck` and `build` delegate to `turbo run`
|
||||
- Keep `build:api`, `build:mcp`, `build:all`, `start` as-is (deployment-specific)
|
||||
- Rename `type:check` → `typecheck` in all packages (Turbo convention)
|
||||
|
||||
### 2. `turbo.json` (new file)
|
||||
|
||||
Tasks:
|
||||
```json
|
||||
{
|
||||
"tasks": {
|
||||
"typecheck": {},
|
||||
"build": { "dependsOn": ["^build"], "outputs": ["dist/**"] },
|
||||
"test": { "dependsOn": ["^build"], "outputs": [] }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`core` builds before `api-server`/`mcp-server` due to `^build` dep.
|
||||
|
||||
### 3. Root `tsconfig.json` (new file)
|
||||
|
||||
```json
|
||||
{
|
||||
"extends": "@tsconfig/bun/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["dom", "ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "preserve",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Per-package `tsconfig.json` (slim)
|
||||
|
||||
All 3 packages slim to:
|
||||
```json
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"paths": { "@/*": ["./src/*"] }
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
```
|
||||
|
||||
### 5. `bunfig.toml` (new file)
|
||||
|
||||
```toml
|
||||
[install]
|
||||
exact = true
|
||||
|
||||
[test]
|
||||
root = "./do-not-run-tests-from-root"
|
||||
```
|
||||
|
||||
Exact installs = reproducible. Root test guard prevents accidental root-level test runs.
|
||||
|
||||
### 6. Package `exports` field
|
||||
|
||||
Replace `main`/`module` with `exports` in all 3 packages:
|
||||
```json
|
||||
"exports": { ".": "./src/index.ts" }
|
||||
```
|
||||
|
||||
Remove `main` and `module` fields. Bun resolves `.ts` directly.
|
||||
|
||||
### 7. Catalog references in per-package `package.json`
|
||||
|
||||
Replace pinned versions with `"catalog:"` for shared deps:
|
||||
- `@typescript/native-preview: "catalog:"`
|
||||
- `@types/bun: "catalog:"`
|
||||
- `@types/unidecode: "catalog:"` (core only)
|
||||
- `@types/cli-progress: "catalog:"` (core only)
|
||||
|
||||
## Files Changed
|
||||
|
||||
| File | Action |
|
||||
|---|---|
|
||||
| `package.json` | Update (catalog, turbo dep, scripts) |
|
||||
| `turbo.json` | Create |
|
||||
| `tsconfig.json` | Create |
|
||||
| `bunfig.toml` | Create |
|
||||
| `packages/core/package.json` | Update (exports, catalog refs, script rename) |
|
||||
| `packages/api-server/package.json` | Update (exports, catalog refs, script rename) |
|
||||
| `packages/mcp-server/package.json` | Update (exports, catalog refs, script rename) |
|
||||
| `packages/core/tsconfig.json` | Update (slim, extends root) |
|
||||
| `packages/api-server/tsconfig.json` | Update (slim, extends root) |
|
||||
| `packages/mcp-server/tsconfig.json` | Update (slim, extends root) |
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- No Husky/git hooks (not needed yet)
|
||||
- No SST/cloud infra (not applicable)
|
||||
- No prettier (keep biome as formatter)
|
||||
- No patches mechanism
|
||||
- No `postinstall` scripts
|
||||
28
package.json
28
package.json
@@ -1,21 +1,35 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/package.json",
|
||||
"name": "marketplace-scrapers-monorepo",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"packageManager": "bun@1.3.13",
|
||||
"scripts": {
|
||||
"ci": "biome ci",
|
||||
"clean": "rm -rf dist",
|
||||
"typecheck": "turbo run typecheck",
|
||||
"build": "bun run clean && turbo run build",
|
||||
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
||||
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
||||
"build:all": "bun run build:api && bun run build:mcp",
|
||||
"build": "bun run clean && bun run build:all",
|
||||
"ci": "biome ci",
|
||||
"clean": "rm -rf dist",
|
||||
"start": "./scripts/start.sh"
|
||||
},
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
"workspaces": {
|
||||
"packages": [
|
||||
"packages/*"
|
||||
],
|
||||
"catalog": {
|
||||
"@tsconfig/bun": "1.0.9",
|
||||
"@typescript/native-preview": "7.0.0-dev.20260428.1",
|
||||
"@types/bun": "1.3.13",
|
||||
"@types/cli-progress": "3.11.6",
|
||||
"@types/unidecode": "1.1.0"
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "2.3.11"
|
||||
"@biomejs/biome": "2.3.11",
|
||||
"@tsconfig/bun": "catalog:",
|
||||
"turbo": "2.5.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,18 +2,22 @@
|
||||
"name": "@marketplace-scrapers/api-server",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"module": "./src/index.ts",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "bun ./src/index.ts",
|
||||
"dev": "bun --watch ./src/index.ts",
|
||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api"
|
||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api",
|
||||
"typecheck": "bun tsgo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@marketplace-scrapers/core": "workspace:*"
|
||||
"@marketplace-scrapers/core": "workspace:*",
|
||||
"@typescript/native-preview": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
"@types/bun": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
|
||||
@@ -48,16 +48,16 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
||||
location: reqUrl.searchParams.get("location") || undefined,
|
||||
category: reqUrl.searchParams.get("category") || undefined,
|
||||
keywords: reqUrl.searchParams.get("keywords") || undefined,
|
||||
sortBy: (reqUrl.searchParams.get("sortBy") as
|
||||
sortBy:
|
||||
(reqUrl.searchParams.get("sortBy") as
|
||||
| "relevancy"
|
||||
| "date"
|
||||
| "price"
|
||||
| "distance"
|
||||
| undefined) || undefined,
|
||||
sortOrder: (reqUrl.searchParams.get("sortOrder") as
|
||||
| "desc"
|
||||
| "asc"
|
||||
| undefined) || undefined,
|
||||
sortOrder:
|
||||
(reqUrl.searchParams.get("sortOrder") as "desc" | "asc" | undefined) ||
|
||||
undefined,
|
||||
maxPages,
|
||||
priceMin,
|
||||
priceMax,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
|
||||
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
|
||||
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
|
||||
@@ -123,7 +123,10 @@ describe("API routes", () => {
|
||||
),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||
"laptop",
|
||||
1,
|
||||
{
|
||||
minPrice: undefined,
|
||||
maxPrice: undefined,
|
||||
strictMode: false,
|
||||
@@ -131,9 +134,11 @@ describe("API routes", () => {
|
||||
keywords: ["laptop"],
|
||||
buyItNowOnly: true,
|
||||
canadaOnly: true,
|
||||
}, {
|
||||
},
|
||||
{
|
||||
hideUnstableResults: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("kijijiRoute forwards unstableFilter=true to core", async () => {
|
||||
@@ -202,9 +207,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&buyItNowOnly=true",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&buyItNowOnly=true"),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
||||
@@ -242,9 +245,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&maxPages=5",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&maxPages=5"),
|
||||
);
|
||||
|
||||
expect(fetchKijijiItems).toHaveBeenCalledWith(
|
||||
@@ -385,17 +386,17 @@ describe("API routes", () => {
|
||||
test("ebayRoute forwards maxItems to core in default mode", async () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
fetchEbayItems.mockImplementation(() =>
|
||||
Promise.resolve([{ title: "a" }]),
|
||||
);
|
||||
fetchEbayItems.mockImplementation(() => Promise.resolve([{ title: "a" }]));
|
||||
|
||||
await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&maxItems=2",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&maxItems=2"),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }));
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||
"laptop",
|
||||
1,
|
||||
expect.objectContaining({ maxItems: 2 }),
|
||||
);
|
||||
});
|
||||
|
||||
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
|
||||
@@ -419,9 +420,14 @@ describe("API routes", () => {
|
||||
expect(body.unstableResults).toHaveLength(2);
|
||||
expect(body.results[0].title).toBe("a");
|
||||
expect(body.unstableResults[0].title).toBe("d");
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 4 }), {
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||
"laptop",
|
||||
1,
|
||||
expect.objectContaining({ maxItems: 4 }),
|
||||
{
|
||||
hideUnstableResults: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
|
||||
@@ -440,9 +446,14 @@ describe("API routes", () => {
|
||||
),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }), {
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||
"laptop",
|
||||
1,
|
||||
expect.objectContaining({ maxItems: 2 }),
|
||||
{
|
||||
hideUnstableResults: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("ebayRoute returns 404 when unstable results are empty", async () => {
|
||||
@@ -456,9 +467,7 @@ describe("API routes", () => {
|
||||
);
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&unstableFilter=true",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&unstableFilter=true"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(404);
|
||||
@@ -470,9 +479,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&maxItems=abc",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&maxItems=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -484,9 +491,7 @@ describe("API routes", () => {
|
||||
const { facebookRoute } = await import("../src/routes/facebook");
|
||||
|
||||
const response = await facebookRoute(
|
||||
new Request(
|
||||
"http://localhost/api/facebook?q=laptop&maxItems=abc",
|
||||
),
|
||||
new Request("http://localhost/api/facebook?q=laptop&maxItems=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -498,9 +503,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&minPrice=abc",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&minPrice=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -512,9 +515,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&maxPrice=abc",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&maxPrice=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -526,9 +527,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
const response = await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&maxPages=abc",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&maxPages=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -540,9 +539,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
const response = await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&priceMin=abc",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&priceMin=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -554,9 +551,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
const response = await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&priceMax=abc",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&priceMax=abc"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -568,9 +563,7 @@ describe("API routes", () => {
|
||||
const { facebookRoute } = await import("../src/routes/facebook");
|
||||
|
||||
const response = await facebookRoute(
|
||||
new Request(
|
||||
"http://localhost/api/facebook?q=laptop&maxItems=-1",
|
||||
),
|
||||
new Request("http://localhost/api/facebook?q=laptop&maxItems=-1"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -582,9 +575,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&maxItems=-1",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&maxItems=-1"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -596,9 +587,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&minPrice=-5",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&minPrice=-5"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -610,9 +599,7 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
const response = await ebayRoute(
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&maxPrice=-10",
|
||||
),
|
||||
new Request("http://localhost/api/ebay?q=laptop&maxPrice=-10"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -624,9 +611,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
const response = await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&maxPages=-2",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&maxPages=-2"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -638,9 +623,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
const response = await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&priceMin=-5",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&priceMin=-5"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -652,9 +635,7 @@ describe("API routes", () => {
|
||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||
|
||||
const response = await kijijiRoute(
|
||||
new Request(
|
||||
"http://localhost/api/kijiji?q=laptop&priceMax=-10",
|
||||
),
|
||||
new Request("http://localhost/api/kijiji?q=laptop&priceMax=-10"),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["dom"],
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
},
|
||||
"strict": true,
|
||||
"noEmit": true
|
||||
}
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
|
||||
@@ -2,18 +2,23 @@
|
||||
"name": "@marketplace-scrapers/core",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"main": "./src/index.ts",
|
||||
"module": "./src/index.ts",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"typecheck": "bun tsgo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@typescript/native-preview": "catalog:",
|
||||
"cli-progress": "^3.12.0",
|
||||
"linkedom": "^0.18.12",
|
||||
"unidecode": "^1.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
"@types/unidecode": "^1.1.0",
|
||||
"@types/cli-progress": "^3.11.6"
|
||||
"@types/bun": "catalog:",
|
||||
"@types/cli-progress": "catalog:",
|
||||
"@types/unidecode": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
|
||||
@@ -4,13 +4,13 @@ import type {
|
||||
UnstableListingBuckets,
|
||||
UnstableListingModeOptions,
|
||||
} from "../types/common";
|
||||
import { classifyUnstableListings } from "../utils/unstable";
|
||||
import {
|
||||
type CookieConfig,
|
||||
ensureCookies,
|
||||
formatCookiesForHeader,
|
||||
} from "../utils/cookies";
|
||||
import { delay } from "../utils/delay";
|
||||
import { classifyUnstableListings } from "../utils/unstable";
|
||||
|
||||
// eBay cookie configuration
|
||||
const EBAY_COOKIE_CONFIG: CookieConfig = {
|
||||
@@ -44,7 +44,9 @@ function canonicalizeEbayItemUrl(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url, "https://www.ebay.ca");
|
||||
const match = parsed.pathname.match(/\/itm\/(?:[^/?#]+\/)?\d+/);
|
||||
return match ? `${parsed.origin}${match[0]}` : `${parsed.origin}${parsed.pathname}`;
|
||||
return match
|
||||
? `${parsed.origin}${match[0]}`
|
||||
: `${parsed.origin}${parsed.pathname}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
@@ -275,11 +277,7 @@ function parseEbayListings(
|
||||
const actualPrices: HTMLElement[] = [];
|
||||
for (const el of allPriceElements) {
|
||||
const text = el.textContent?.trim();
|
||||
if (
|
||||
text &&
|
||||
EBAY_PRICE_TEXT_RE.test(text) &&
|
||||
text.length < 50
|
||||
) {
|
||||
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
|
||||
actualPrices.push(el);
|
||||
}
|
||||
}
|
||||
@@ -386,7 +384,8 @@ async function loadEbayCookies(): Promise<string | undefined> {
|
||||
export default async function fetchEbayItems(
|
||||
SEARCH_QUERY: string,
|
||||
REQUESTS_PER_SECOND: number | undefined,
|
||||
opts: {
|
||||
opts:
|
||||
| {
|
||||
minPrice?: number;
|
||||
maxPrice?: number;
|
||||
strictMode?: boolean;
|
||||
@@ -395,7 +394,8 @@ export default async function fetchEbayItems(
|
||||
buyItNowOnly?: boolean;
|
||||
canadaOnly?: boolean;
|
||||
maxItems?: number;
|
||||
} | undefined,
|
||||
}
|
||||
| undefined,
|
||||
unstableMode: { hideUnstableResults: true },
|
||||
): Promise<UnstableListingBuckets<EbayListingDetails>>;
|
||||
export default async function fetchEbayItems(
|
||||
@@ -529,7 +529,9 @@ export default async function fetchEbayItems(
|
||||
// Filter by price range (additional safety check)
|
||||
const filteredListings = listings.filter((listing) => {
|
||||
const cents = listing.listingPrice?.cents;
|
||||
return typeof cents === "number" && cents >= minPrice && cents <= maxPrice;
|
||||
return (
|
||||
typeof cents === "number" && cents >= minPrice && cents <= maxPrice
|
||||
);
|
||||
});
|
||||
|
||||
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||
|
||||
@@ -5,7 +5,6 @@ import type {
|
||||
UnstableListingBuckets,
|
||||
UnstableListingModeOptions,
|
||||
} from "../types/common";
|
||||
import { classifyUnstableListings } from "../utils/unstable";
|
||||
import {
|
||||
type Cookie,
|
||||
type CookieConfig,
|
||||
@@ -16,6 +15,7 @@ import {
|
||||
import { delay } from "../utils/delay";
|
||||
import { formatCentsToCurrency } from "../utils/format";
|
||||
import { isRecord } from "../utils/http";
|
||||
import { classifyUnstableListings } from "../utils/unstable";
|
||||
|
||||
/**
|
||||
* Facebook Marketplace Scraper
|
||||
@@ -408,7 +408,11 @@ export function classifyFacebookResponse(
|
||||
htmlString.includes("This listing is no longer available") ||
|
||||
htmlString.includes("listing has been removed");
|
||||
if (unavailable) {
|
||||
return { kind: "unavailable" as const, authGated: false, unavailable: true };
|
||||
return {
|
||||
kind: "unavailable" as const,
|
||||
authGated: false,
|
||||
unavailable: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (responseUrl.includes("/marketplace/item/")) {
|
||||
@@ -455,7 +459,8 @@ function isFacebookSearchEdgeArray(value: unknown): value is FacebookEdge[] {
|
||||
Array.isArray(value) &&
|
||||
value.length > 0 &&
|
||||
value.every(
|
||||
(edge) => isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
|
||||
(edge) =>
|
||||
isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
|
||||
)
|
||||
);
|
||||
}
|
||||
@@ -552,8 +557,7 @@ function scoreMarketplaceItemPath(path: string[]): number {
|
||||
|
||||
if (
|
||||
path.some(
|
||||
(segment) =>
|
||||
segment.includes("recommend") || segment.includes("related"),
|
||||
(segment) => segment.includes("recommend") || segment.includes("related"),
|
||||
)
|
||||
) {
|
||||
score -= 10;
|
||||
@@ -567,7 +571,9 @@ function collectMarketplaceItemCandidates(
|
||||
path: string[] = [],
|
||||
): FacebookMarketplaceItemMatch[] {
|
||||
if (Array.isArray(candidate)) {
|
||||
return candidate.flatMap((item) => collectMarketplaceItemCandidates(item, path));
|
||||
return candidate.flatMap((item) =>
|
||||
collectMarketplaceItemCandidates(item, path),
|
||||
);
|
||||
}
|
||||
|
||||
if (!isRecord(candidate)) {
|
||||
@@ -628,7 +634,9 @@ function extractRenderedText(node: ParentNode, selector: string): string[] {
|
||||
.filter((text): text is string => Boolean(text));
|
||||
}
|
||||
|
||||
function extractMarketplaceItemIdFromElement(element: Element | null): string | null {
|
||||
function extractMarketplaceItemIdFromElement(
|
||||
element: Element | null,
|
||||
): string | null {
|
||||
const href = element?.getAttribute("href") || "";
|
||||
return href.match(FACEBOOK_ITEM_HREF_RE)?.[1] ?? null;
|
||||
}
|
||||
@@ -666,7 +674,9 @@ function extractFacebookPermalinkItemId(document: Document): string | null {
|
||||
return extractMarketplaceItemIdFromElement(itemLinks.at(-1) ?? null);
|
||||
}
|
||||
|
||||
function extractFacebookDescriptionText(document: Document): string | undefined {
|
||||
function extractFacebookDescriptionText(
|
||||
document: Document,
|
||||
): string | undefined {
|
||||
const labels = Array.from(document.querySelectorAll("div, span, h2, h3, p"));
|
||||
|
||||
for (const label of labels) {
|
||||
@@ -759,7 +769,10 @@ function extractFacebookItemHtmlFallback(
|
||||
const priceText = texts.find((text) => FACEBOOK_PRICE_TEXT_RE.test(text));
|
||||
const parsedPrice = priceText ? parseFacebookRenderedPrice(priceText) : null;
|
||||
const location = texts.find(
|
||||
(text) => text !== title && text !== priceText && FACEBOOK_LOCATION_TEXT_RE.test(text),
|
||||
(text) =>
|
||||
text !== title &&
|
||||
text !== priceText &&
|
||||
FACEBOOK_LOCATION_TEXT_RE.test(text),
|
||||
);
|
||||
const description = extractFacebookDescriptionText(document);
|
||||
|
||||
@@ -841,7 +854,8 @@ export function extractFacebookItemData(
|
||||
if (
|
||||
!bestMatch ||
|
||||
match.score > bestMatch.score ||
|
||||
(match.score === bestMatch.score && match.path.length < bestMatch.path.length)
|
||||
(match.score === bestMatch.score &&
|
||||
match.path.length < bestMatch.path.length)
|
||||
) {
|
||||
bestMatch = match;
|
||||
}
|
||||
@@ -1101,7 +1115,9 @@ export default async function fetchFacebookItems(
|
||||
|
||||
const finalizeResults = (
|
||||
listings: FacebookListingDetails[],
|
||||
): FacebookListingDetails[] | UnstableListingBuckets<FacebookListingDetails> => {
|
||||
):
|
||||
| FacebookListingDetails[]
|
||||
| UnstableListingBuckets<FacebookListingDetails> => {
|
||||
if (!unstableMode.hideUnstableResults) {
|
||||
return listings.slice(0, MAX_ITEMS);
|
||||
}
|
||||
@@ -1166,9 +1182,14 @@ export default async function fetchFacebookItems(
|
||||
throw err;
|
||||
}
|
||||
|
||||
const classification = classifyFacebookResponse(searchHtml, searchResponseUrl);
|
||||
const classification = classifyFacebookResponse(
|
||||
searchHtml,
|
||||
searchResponseUrl,
|
||||
);
|
||||
if (classification.authGated) {
|
||||
console.warn("Facebook marketplace search redirected to login. Cookies may be expired.");
|
||||
console.warn(
|
||||
"Facebook marketplace search redirected to login. Cookies may be expired.",
|
||||
);
|
||||
return finalizeResults([]);
|
||||
}
|
||||
|
||||
@@ -1204,7 +1225,8 @@ export default async function fetchFacebookItems(
|
||||
// Filter to only priced items (already done in parseFacebookAds)
|
||||
const pricedItems = items.filter(
|
||||
(item) =>
|
||||
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
|
||||
typeof item.listingPrice?.cents === "number" &&
|
||||
item.listingPrice.cents >= 0,
|
||||
);
|
||||
|
||||
progressBar?.update(totalProgress);
|
||||
@@ -1293,7 +1315,9 @@ export async function fetchFacebookItem(
|
||||
|
||||
if (classification.authGated) {
|
||||
logExtractionMetrics(false, itemId);
|
||||
console.warn(`Authentication failed for item ${itemId}. Cookies may be expired.`);
|
||||
console.warn(
|
||||
`Authentication failed for item ${itemId}. Cookies may be expired.`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1301,7 +1325,9 @@ export async function fetchFacebookItem(
|
||||
|
||||
if (classification.unavailable && !itemData) {
|
||||
logExtractionMetrics(false, itemId);
|
||||
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
|
||||
console.warn(
|
||||
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1317,7 +1343,9 @@ export async function fetchFacebookItem(
|
||||
logExtractionMetrics(false, itemId);
|
||||
|
||||
if (itemHtml.includes("This item has been sold")) {
|
||||
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
|
||||
console.warn(
|
||||
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import type {
|
||||
UnstableListingBuckets,
|
||||
UnstableListingModeOptions,
|
||||
} from "../types/common";
|
||||
import { classifyUnstableListings } from "../utils/unstable";
|
||||
import {
|
||||
type CookieConfig,
|
||||
formatCookiesForHeader,
|
||||
@@ -22,6 +21,7 @@ import {
|
||||
RateLimitError,
|
||||
ValidationError,
|
||||
} from "../utils/http";
|
||||
import { classifyUnstableListings } from "../utils/unstable";
|
||||
|
||||
// Kijiji cookie configuration
|
||||
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
|
||||
@@ -203,11 +203,17 @@ const SORT_MAPPINGS: Record<string, string> = {
|
||||
};
|
||||
|
||||
const LOCATION_SLUGS = Object.fromEntries(
|
||||
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
|
||||
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [
|
||||
id,
|
||||
slug.replace(/\s+/g, "-"),
|
||||
]),
|
||||
) as Record<number, string>;
|
||||
|
||||
const CATEGORY_SLUGS = Object.fromEntries(
|
||||
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
|
||||
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [
|
||||
id,
|
||||
slug.replace(/\s+/g, "-"),
|
||||
]),
|
||||
) as Record<number, string>;
|
||||
|
||||
// ----------------------------- Utilities -----------------------------
|
||||
@@ -816,7 +822,10 @@ export default async function fetchKijijiItems(
|
||||
: undefined;
|
||||
|
||||
// Set defaults for configuration
|
||||
const finalSearchOptions: Omit<Required<SearchOptions>, "priceMin" | "priceMax"> & {
|
||||
const finalSearchOptions: Omit<
|
||||
Required<SearchOptions>,
|
||||
"priceMin" | "priceMax"
|
||||
> & {
|
||||
priceMin?: number;
|
||||
priceMax?: number;
|
||||
} = {
|
||||
@@ -903,7 +912,9 @@ export default async function fetchKijijiItems(
|
||||
const batchPromises = batch.map(async (link, batchIndex) => {
|
||||
try {
|
||||
if (batchIndex > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, DELAY_MS * batchIndex));
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, DELAY_MS * batchIndex),
|
||||
);
|
||||
}
|
||||
|
||||
const html = await fetchHtml(link, 0, {
|
||||
@@ -949,7 +960,6 @@ export default async function fetchKijijiItems(
|
||||
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
||||
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
allListings.push(
|
||||
@@ -968,9 +978,7 @@ export default async function fetchKijijiItems(
|
||||
matchesPriceFilters(listing, finalSearchOptions),
|
||||
);
|
||||
|
||||
console.log(
|
||||
`\nParsed ${filteredListings.length} detailed listings.`,
|
||||
);
|
||||
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
|
||||
return finalizeResults(filteredListings);
|
||||
}
|
||||
|
||||
|
||||
@@ -136,7 +136,9 @@ describe("eBay Scraper Cookie Handling", () => {
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toEqual(
|
||||
expect.objectContaining({ url: "https://www.ebay.ca/itm/123?_trkparms=foo" }),
|
||||
expect.objectContaining({
|
||||
url: "https://www.ebay.ca/itm/123?_trkparms=foo",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -229,7 +231,10 @@ describe("eBay Scraper Cookie Handling", () => {
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({
|
||||
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
|
||||
listingPrice: expect.objectContaining({
|
||||
currency: "USD",
|
||||
cents: 12345,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
@@ -255,7 +260,10 @@ describe("eBay Scraper Cookie Handling", () => {
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({
|
||||
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
|
||||
listingPrice: expect.objectContaining({
|
||||
currency: "USD",
|
||||
cents: 12345,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
@@ -281,7 +289,10 @@ describe("eBay Scraper Cookie Handling", () => {
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({
|
||||
listingPrice: expect.objectContaining({ currency: "GBP", cents: 12345 }),
|
||||
listingPrice: expect.objectContaining({
|
||||
currency: "GBP",
|
||||
cents: 12345,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
@@ -314,10 +325,16 @@ describe("eBay Scraper Cookie Handling", () => {
|
||||
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({
|
||||
listingPrice: expect.objectContaining({ currency: "EUR", cents: 12345 }),
|
||||
listingPrice: expect.objectContaining({
|
||||
currency: "EUR",
|
||||
cents: 12345,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
listingPrice: expect.objectContaining({ currency: "JPY", cents: 12300 }),
|
||||
listingPrice: expect.objectContaining({
|
||||
currency: "JPY",
|
||||
cents: 12300,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -2,13 +2,13 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import cliProgress from "cli-progress";
|
||||
import {
|
||||
classifyFacebookResponse,
|
||||
type FacebookListingDetails,
|
||||
ensureFacebookCookies,
|
||||
extractFacebookBootstrapCandidates,
|
||||
extractFacebookItemData,
|
||||
extractFacebookMarketplaceData,
|
||||
default as fetchFacebookItems,
|
||||
type FacebookListingDetails,
|
||||
fetchFacebookItem,
|
||||
default as fetchFacebookItems,
|
||||
parseFacebookAds,
|
||||
parseFacebookCookieString,
|
||||
parseFacebookItem,
|
||||
@@ -30,9 +30,13 @@ type IsExact<T, U> =
|
||||
const getDefaultFacebookItems = async () => fetchFacebookItems("chair");
|
||||
const getUnstableFacebookItems = async (): Promise<
|
||||
UnstableListingBuckets<FacebookListingDetails>
|
||||
> => fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
|
||||
> =>
|
||||
fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
|
||||
type _FacebookDefaultReturn = Assert<
|
||||
IsExact<Awaited<ReturnType<typeof getDefaultFacebookItems>>, FacebookListingDetails[]>
|
||||
IsExact<
|
||||
Awaited<ReturnType<typeof getDefaultFacebookItems>>,
|
||||
FacebookListingDetails[]
|
||||
>
|
||||
>;
|
||||
type _FacebookUnstableReturn = Assert<
|
||||
IsExact<
|
||||
@@ -533,7 +537,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
});
|
||||
|
||||
test("returns an array by default", async () => {
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||
{
|
||||
payload: {
|
||||
resultGroups: [
|
||||
{
|
||||
@@ -556,7 +561,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
})}</script></body></html>`;
|
||||
},
|
||||
)}</script></body></html>`;
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
@@ -576,7 +582,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
});
|
||||
|
||||
test("preserves free listings through the public fetch entrypoint", async () => {
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||
{
|
||||
payload: {
|
||||
resultGroups: [
|
||||
{
|
||||
@@ -599,7 +606,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
})}</script></body></html>`;
|
||||
},
|
||||
)}</script></body></html>`;
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
@@ -626,7 +634,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
});
|
||||
|
||||
test("does not start a progress bar when stdout is not a TTY", async () => {
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||
{
|
||||
payload: {
|
||||
resultGroups: [
|
||||
{
|
||||
@@ -649,7 +658,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
})}</script></body></html>`;
|
||||
},
|
||||
)}</script></body></html>`;
|
||||
|
||||
process.stdout.isTTY = false;
|
||||
const startSpy = mock(() => {});
|
||||
@@ -688,7 +698,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
});
|
||||
|
||||
test("returns results and unstableResults when unstable mode is enabled", async () => {
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||
{
|
||||
payload: {
|
||||
resultGroups: [
|
||||
{
|
||||
@@ -739,7 +750,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
})}</script></body></html>`;
|
||||
},
|
||||
)}</script></body></html>`;
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
@@ -768,7 +780,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
});
|
||||
|
||||
test("unstable mode classifies before the final MAX_ITEMS limit", async () => {
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||
{
|
||||
payload: {
|
||||
resultGroups: [
|
||||
{
|
||||
@@ -791,7 +804,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "Second Boundary Stable Chair",
|
||||
marketplace_listing_title:
|
||||
"Second Boundary Stable Chair",
|
||||
listing_price: {
|
||||
amount: "110.00",
|
||||
formatted_amount: "CA$110",
|
||||
@@ -819,7 +833,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
})}</script></body></html>`;
|
||||
},
|
||||
)}</script></body></html>`;
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
@@ -869,7 +884,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
},
|
||||
redacted_description: { text: "Solid wood chair" },
|
||||
location_text: { text: "Toronto, ON" },
|
||||
marketplace_listing_seller: { id: "seller-1", name: "Alex" },
|
||||
marketplace_listing_seller: {
|
||||
id: "seller-1",
|
||||
name: "Alex",
|
||||
},
|
||||
condition: "USED",
|
||||
is_live: true,
|
||||
},
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import fetchFacebookItems, { fetchFacebookItem } from "../src/scrapers/facebook";
|
||||
import fetchFacebookItems, {
|
||||
fetchFacebookItem,
|
||||
} from "../src/scrapers/facebook";
|
||||
|
||||
// Mock fetch globally
|
||||
const originalFetch = global.fetch;
|
||||
@@ -27,7 +29,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
|
||||
describe("Main Search Function", () => {
|
||||
test("should successfully fetch search results", async () => {
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||
{
|
||||
payload: {
|
||||
resultGroups: [
|
||||
{
|
||||
@@ -55,7 +58,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
})}</script></body></html>`;
|
||||
},
|
||||
)}</script></body></html>`;
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import {
|
||||
buildSearchUrl,
|
||||
default as fetchKijijiItems,
|
||||
type DetailedListing,
|
||||
default as fetchKijijiItems,
|
||||
NetworkError,
|
||||
parseSearch,
|
||||
parseDetailedListing,
|
||||
ParseError,
|
||||
parseDetailedListing,
|
||||
parseSearch,
|
||||
RateLimitError,
|
||||
resolveCategoryId,
|
||||
resolveLocationId,
|
||||
@@ -282,7 +282,8 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-low/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -291,7 +292,8 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-mid/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -300,7 +302,8 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-high/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -534,9 +537,18 @@ describe("fetchKijijiItems", () => {
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
|
||||
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
|
||||
"Listing:3": { url: "/v-unstable/k0l0", title: "Unstable Listing" },
|
||||
"Listing:1": {
|
||||
url: "/v-stable-one/k0l0",
|
||||
title: "Stable Listing One",
|
||||
},
|
||||
"Listing:2": {
|
||||
url: "/v-stable-two/k0l0",
|
||||
title: "Stable Listing Two",
|
||||
},
|
||||
"Listing:3": {
|
||||
url: "/v-unstable/k0l0",
|
||||
title: "Unstable Listing",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -582,7 +594,10 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-stable-one/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -591,7 +606,10 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-stable-two/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -600,7 +618,10 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-unstable/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Unstable Listing", 7000, "v-unstable/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml("Unstable Listing", 7000, "v-unstable/k0l0"),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -635,10 +656,22 @@ describe("fetchKijijiItems", () => {
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
|
||||
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
|
||||
"Listing:3": { url: "/v-out-of-range-high/k0l0", title: "Out Of Range High" },
|
||||
"Listing:4": { url: "/v-out-of-range-low/k0l0", title: "Out Of Range Low" },
|
||||
"Listing:1": {
|
||||
url: "/v-stable-one/k0l0",
|
||||
title: "Stable Listing One",
|
||||
},
|
||||
"Listing:2": {
|
||||
url: "/v-stable-two/k0l0",
|
||||
title: "Stable Listing Two",
|
||||
},
|
||||
"Listing:3": {
|
||||
url: "/v-out-of-range-high/k0l0",
|
||||
title: "Out Of Range High",
|
||||
},
|
||||
"Listing:4": {
|
||||
url: "/v-out-of-range-low/k0l0",
|
||||
title: "Out Of Range Low",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -672,7 +705,11 @@ describe("fetchKijijiItems", () => {
|
||||
global.fetch = mock((input: string | URL | Request) => {
|
||||
const url = typeof input === "string" ? input : input.toString();
|
||||
|
||||
if (url.includes("/k0c0l1700272") && url.includes("priceMin=80") && url.includes("priceMax=150")) {
|
||||
if (
|
||||
url.includes("/k0c0l1700272") &&
|
||||
url.includes("priceMin=80") &&
|
||||
url.includes("priceMax=150")
|
||||
) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(searchHtml),
|
||||
@@ -684,7 +721,10 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-stable-one/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -693,7 +733,10 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-stable-two/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -702,7 +745,14 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-out-of-range-high/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Out Of Range High", 20000, "v-out-of-range-high/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml(
|
||||
"Out Of Range High",
|
||||
20000,
|
||||
"v-out-of-range-high/k0l0",
|
||||
),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
@@ -711,7 +761,10 @@ describe("fetchKijijiItems", () => {
|
||||
if (url.endsWith("/v-out-of-range-low/k0l0")) {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0")),
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0"),
|
||||
),
|
||||
headers: { get: () => null },
|
||||
url,
|
||||
});
|
||||
|
||||
@@ -31,8 +31,13 @@ describe("classifyUnstableListings", () => {
|
||||
|
||||
const buckets = classifyUnstableListings(listings);
|
||||
|
||||
expect(buckets.results.map((listing) => listing.id)).toEqual(["stable-1", "stable-2"]);
|
||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
|
||||
expect(buckets.results.map((listing) => listing.id)).toEqual([
|
||||
"stable-1",
|
||||
"stable-2",
|
||||
]);
|
||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
|
||||
"unstable",
|
||||
]);
|
||||
});
|
||||
|
||||
test("uses the midpoint median for even-sized priced inputs", () => {
|
||||
@@ -45,8 +50,14 @@ describe("classifyUnstableListings", () => {
|
||||
|
||||
const buckets = classifyUnstableListings(listings);
|
||||
|
||||
expect(buckets.results.map((listing) => listing.id)).toEqual(["mid-low", "mid-high", "high"]);
|
||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["low"]);
|
||||
expect(buckets.results.map((listing) => listing.id)).toEqual([
|
||||
"mid-low",
|
||||
"mid-high",
|
||||
"high",
|
||||
]);
|
||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
|
||||
"low",
|
||||
]);
|
||||
});
|
||||
|
||||
test("keeps non-positive prices in results and excludes them from the median input", () => {
|
||||
@@ -66,7 +77,9 @@ describe("classifyUnstableListings", () => {
|
||||
"stable-1",
|
||||
"stable-2",
|
||||
]);
|
||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
|
||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
|
||||
"unstable",
|
||||
]);
|
||||
});
|
||||
|
||||
test("returns all listings in results when fewer than two valid prices are present", () => {
|
||||
@@ -78,7 +91,11 @@ describe("classifyUnstableListings", () => {
|
||||
|
||||
const buckets = classifyUnstableListings(listings);
|
||||
|
||||
expect(buckets.results.map((listing) => listing.id)).toEqual(["zero", "negative", "only-valid"]);
|
||||
expect(buckets.results.map((listing) => listing.id)).toEqual([
|
||||
"zero",
|
||||
"negative",
|
||||
"only-valid",
|
||||
]);
|
||||
expect(buckets.unstableResults).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["dom"],
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
},
|
||||
"strict": true,
|
||||
"noEmit": true
|
||||
}
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
|
||||
@@ -2,18 +2,22 @@
|
||||
"name": "@marketplace-scrapers/mcp-server",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"module": "./src/index.ts",
|
||||
"exports": {
|
||||
".": "./src/index.ts"
|
||||
},
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "bun ./src/index.ts",
|
||||
"dev": "bun --watch ./src/index.ts",
|
||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp"
|
||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp",
|
||||
"typecheck": "bun tsgo"
|
||||
},
|
||||
"dependencies": {
|
||||
"@marketplace-scrapers/core": "workspace:*"
|
||||
"@marketplace-scrapers/core": "workspace:*",
|
||||
"@typescript/native-preview": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
"@types/bun": "catalog:"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
|
||||
@@ -72,7 +72,10 @@ describe("MCP protocol unstableFilter", () => {
|
||||
const tool = tools.find((t) => t.name === toolName);
|
||||
expect(tool).toBeDefined();
|
||||
expect(tool?.inputSchema.properties).toHaveProperty("unstableFilter");
|
||||
const prop = tool?.inputSchema.properties.unstableFilter as any;
|
||||
const prop = tool?.inputSchema.properties.unstableFilter as {
|
||||
type: string;
|
||||
description: string;
|
||||
};
|
||||
expect(prop.type).toBe("boolean");
|
||||
expect(prop.description).toContain("optional");
|
||||
expect(prop.description).toContain("20%");
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["dom"],
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"paths": {
|
||||
"@/*": ["./src/*"]
|
||||
},
|
||||
"strict": true,
|
||||
"noEmit": true
|
||||
}
|
||||
},
|
||||
"include": ["./src", "./test"]
|
||||
}
|
||||
|
||||
24
tsconfig.json
Normal file
24
tsconfig.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "@tsconfig/bun/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["dom", "ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "preserve",
|
||||
"moduleResolution": "bundler",
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
14
turbo.json
Normal file
14
turbo.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"$schema": "https://turbo.build/schema.json",
|
||||
"tasks": {
|
||||
"typecheck": {},
|
||||
"build": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": ["../../dist/**"]
|
||||
},
|
||||
"test": {
|
||||
"dependsOn": ["^build"],
|
||||
"outputs": []
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user