Compare commits
12 Commits
49e90d45f8
...
d77a006ded
| Author | SHA1 | Date | |
|---|---|---|---|
| d77a006ded | |||
| 56b2198df1 | |||
| 63716272c5 | |||
| 1d21c66945 | |||
| f2f78225f3 | |||
| 43d15fce5f | |||
| fef2f1968a | |||
| 01081f6b2e | |||
| d10d5305a3 | |||
| bf393eacae | |||
| 79bb249603 | |||
| 957e0f137b |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -33,6 +33,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
|||||||
.eslintcache
|
.eslintcache
|
||||||
.cache
|
.cache
|
||||||
*.tsbuildinfo
|
*.tsbuildinfo
|
||||||
|
.turbo
|
||||||
|
|
||||||
# IntelliJ based IDEs
|
# IntelliJ based IDEs
|
||||||
.idea
|
.idea
|
||||||
|
|||||||
58
bun.lock
58
bun.lock
@@ -6,6 +6,8 @@
|
|||||||
"name": "marketplace-scrapers-monorepo",
|
"name": "marketplace-scrapers-monorepo",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "2.3.11",
|
"@biomejs/biome": "2.3.11",
|
||||||
|
"@tsconfig/bun": "catalog:",
|
||||||
|
"turbo": "2.5.4",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"packages/api-server": {
|
"packages/api-server": {
|
||||||
@@ -13,9 +15,10 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@marketplace-scrapers/core": "workspace:*",
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest",
|
"@types/bun": "catalog:",
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5",
|
"typescript": "^5",
|
||||||
@@ -25,14 +28,15 @@
|
|||||||
"name": "@marketplace-scrapers/core",
|
"name": "@marketplace-scrapers/core",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
"cli-progress": "^3.12.0",
|
"cli-progress": "^3.12.0",
|
||||||
"linkedom": "^0.18.12",
|
"linkedom": "^0.18.12",
|
||||||
"unidecode": "^1.1.0",
|
"unidecode": "^1.1.0",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest",
|
"@types/bun": "catalog:",
|
||||||
"@types/cli-progress": "^3.11.6",
|
"@types/cli-progress": "catalog:",
|
||||||
"@types/unidecode": "^1.1.0",
|
"@types/unidecode": "catalog:",
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5",
|
"typescript": "^5",
|
||||||
@@ -43,15 +47,23 @@
|
|||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@marketplace-scrapers/core": "workspace:*",
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest",
|
"@types/bun": "catalog:",
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5",
|
"typescript": "^5",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
"catalog": {
|
||||||
|
"@tsconfig/bun": "1.0.9",
|
||||||
|
"@types/bun": "1.3.13",
|
||||||
|
"@types/cli-progress": "3.11.6",
|
||||||
|
"@types/unidecode": "1.1.0",
|
||||||
|
"@typescript/native-preview": "7.0.0-dev.20260428.1",
|
||||||
|
},
|
||||||
"packages": {
|
"packages": {
|
||||||
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="],
|
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="],
|
||||||
|
|
||||||
@@ -77,7 +89,9 @@
|
|||||||
|
|
||||||
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"],
|
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"],
|
||||||
|
|
||||||
"@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="],
|
"@tsconfig/bun": ["@tsconfig/bun@1.0.9", "", {}, "sha512-4M0/Ivfwcpz325z6CwSifOBZYji3DFOEpY6zEUt0+Xi2qRhzwvmqQN9XAHJh3OVvRJuAqVTLU2abdCplvp6mwQ=="],
|
||||||
|
|
||||||
|
"@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="],
|
||||||
|
|
||||||
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
|
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
|
||||||
|
|
||||||
@@ -85,11 +99,27 @@
|
|||||||
|
|
||||||
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview": ["@typescript/native-preview@7.0.0-dev.20260428.1", "", { "optionalDependencies": { "@typescript/native-preview-darwin-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-darwin-x64": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-arm": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-x64": "7.0.0-dev.20260428.1", "@typescript/native-preview-win32-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-win32-x64": "7.0.0-dev.20260428.1" }, "bin": { "tsgo": "bin/tsgo.js" } }, "sha512-JiM4PYWDGs57TT0mV2KArmaW7BnTkk3XRid79NdG17tfvDbRyg4hBCpKI7vARiQPtxjKrHlxyzxOGDpv5W5T7Q=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-darwin-arm64": ["@typescript/native-preview-darwin-arm64@7.0.0-dev.20260428.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Lll6WmXfgTEj1G3QBIoHlabQwUtJiyhlRgSLksa06QFL5BoA7V+Lu1waa9PtPNZbGsXLDMHodtk/bRQABKuPiw=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-darwin-x64": ["@typescript/native-preview-darwin-x64@7.0.0-dev.20260428.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-WbsBNSHlo+4sGrTxDWdmI7r8x48tCtSCuKdmK62FvVOq58UWAs6sL13Z4Rev4ohLcGHdXC5E/8AIdpLPqDYQpw=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-linux-arm": ["@typescript/native-preview-linux-arm@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "arm" }, "sha512-/d/NnZFvEJU67L5mHh+cO3gsfwNCvJ9HGtxGq1KGz1VwTabOIcwLdpTpfsAR39WXzzfh9GJHL28n6GSGZInPow=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-linux-arm64": ["@typescript/native-preview-linux-arm64@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-cgcBX/ZBMdepkamLT8g8jQdHe7DZS/s6zTZRof6mvcrnJHlMeUnKoC9UO8/c22IrUMV3n0XPh7R8FYjUP0ll+Q=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-linux-x64": ["@typescript/native-preview-linux-x64@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "x64" }, "sha512-4gJCE7wzenx1BH2Vtx2uKWUo8rFxnhGkxNEH1zxbYy/6ASwo+PnOPYmKHAzNE1C3yB5lzw71/vR5p5zyO57Y4A=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-win32-arm64": ["@typescript/native-preview-win32-arm64@7.0.0-dev.20260428.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-yn6Rzbn62L4QTWrp0QgG8al6l/VG7PCPRdbE0vuGDSlKhInlC+Flo4QSc1qA8KHTbpHgl+nEsq9DymiitI4G4g=="],
|
||||||
|
|
||||||
|
"@typescript/native-preview-win32-x64": ["@typescript/native-preview-win32-x64@7.0.0-dev.20260428.1", "", { "os": "win32", "cpu": "x64" }, "sha512-T9z13mcMowXmwGjprA2FIR2EEdYZxgqH8+qk7dFZVBlo5vfk41AN/qJfAdN7IsAhEb640MJ8cMN/aiczweZKmA=="],
|
||||||
|
|
||||||
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
|
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
|
||||||
|
|
||||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||||
|
|
||||||
"bun-types": ["bun-types@1.3.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-5ua817+BZPZOlNaRgGBpZJOSAQ9RQ17pkwPD0yR7CfJg+r8DgIILByFifDTa+IPDDxzf5VNhtNlcKqFzDgJvlQ=="],
|
"bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],
|
||||||
|
|
||||||
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
||||||
|
|
||||||
@@ -125,6 +155,20 @@
|
|||||||
|
|
||||||
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
|
||||||
|
|
||||||
|
"turbo": ["turbo@2.5.4", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.4", "turbo-darwin-arm64": "2.5.4", "turbo-linux-64": "2.5.4", "turbo-linux-arm64": "2.5.4", "turbo-windows-64": "2.5.4", "turbo-windows-arm64": "2.5.4" }, "bin": { "turbo": "bin/turbo" } }, "sha512-kc8ZibdRcuWUG1pbYSBFWqmIjynlD8Lp7IB6U3vIzvOv9VG+6Sp8bzyeBWE3Oi8XV5KsQrznyRTBPvrf99E4mA=="],
|
||||||
|
|
||||||
|
"turbo-darwin-64": ["turbo-darwin-64@2.5.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-ah6YnH2dErojhFooxEzmvsoZQTMImaruZhFPfMKPBq8sb+hALRdvBNLqfc8NWlZq576FkfRZ/MSi4SHvVFT9PQ=="],
|
||||||
|
|
||||||
|
"turbo-darwin-arm64": ["turbo-darwin-arm64@2.5.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2+Nx6LAyuXw2MdXb7pxqle3MYignLvS7OwtsP9SgtSBaMlnNlxl9BovzqdYAgkUW3AsYiQMJ/wBRb7d+xemM5A=="],
|
||||||
|
|
||||||
|
"turbo-linux-64": ["turbo-linux-64@2.5.4", "", { "os": "linux", "cpu": "x64" }, "sha512-5May2kjWbc8w4XxswGAl74GZ5eM4Gr6IiroqdLhXeXyfvWEdm2mFYCSWOzz0/z5cAgqyGidF1jt1qzUR8hTmOA=="],
|
||||||
|
|
||||||
|
"turbo-linux-arm64": ["turbo-linux-arm64@2.5.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-/2yqFaS3TbfxV3P5yG2JUI79P7OUQKOUvAnx4MV9Bdz6jqHsHwc9WZPpO4QseQm+NvmgY6ICORnoVPODxGUiJg=="],
|
||||||
|
|
||||||
|
"turbo-windows-64": ["turbo-windows-64@2.5.4", "", { "os": "win32", "cpu": "x64" }, "sha512-EQUO4SmaCDhO6zYohxIjJpOKRN3wlfU7jMAj3CgcyTPvQR/UFLEKAYHqJOnJtymbQmiiM/ihX6c6W6Uq0yC7mA=="],
|
||||||
|
|
||||||
|
"turbo-windows-arm64": ["turbo-windows-arm64@2.5.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-oQ8RrK1VS8lrxkLriotFq+PiF7iiGgkZtfLKF4DDKsmdbPo0O9R2mQxm7jHLuXraRCuIQDWMIw6dpcr7Iykf4A=="],
|
||||||
|
|
||||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||||
|
|
||||||
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],
|
||||||
|
|||||||
5
bunfig.toml
Normal file
5
bunfig.toml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
[install]
|
||||||
|
exact = true
|
||||||
|
|
||||||
|
[test]
|
||||||
|
root = "./do-not-run-tests-from-root"
|
||||||
477
docs/superpowers/plans/2025-07-14-opencode-monorepo-config.md
Normal file
477
docs/superpowers/plans/2025-07-14-opencode-monorepo-config.md
Normal file
@@ -0,0 +1,477 @@
|
|||||||
|
# opencode Monorepo Config Adoption Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Adopt opencode-style monorepo config: Turbo task orchestration, workspace dep catalog, shared root tsconfig, bunfig.toml, and `exports` field in all packages.
|
||||||
|
|
||||||
|
**Architecture:** Pure config changes across 10 files — no source code touched. Root config files are added/updated first, then per-package files updated to reference them. Changes are independent within each task and safe to commit atomically.
|
||||||
|
|
||||||
|
**Tech Stack:** Bun workspaces, Turbo 2.x, @tsconfig/bun, TypeScript (tsgo / @typescript/native-preview)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File Map
|
||||||
|
|
||||||
|
| File | Action | Responsible for |
|
||||||
|
|---|---|---|
|
||||||
|
| `package.json` | Modify | Workspace catalog, turbo devDep, @tsconfig/bun devDep, updated scripts |
|
||||||
|
| `turbo.json` | Create | Task graph: typecheck, build, test |
|
||||||
|
| `tsconfig.json` | Create | Shared TS compiler options for all packages |
|
||||||
|
| `bunfig.toml` | Create | Exact installs, root test guard |
|
||||||
|
| `packages/core/package.json` | Modify | exports field, catalog refs, script rename |
|
||||||
|
| `packages/api-server/package.json` | Modify | exports field, catalog refs, script rename |
|
||||||
|
| `packages/mcp-server/package.json` | Modify | exports field, catalog refs, script rename |
|
||||||
|
| `packages/core/tsconfig.json` | Modify | Slim — extends root, paths only |
|
||||||
|
| `packages/api-server/tsconfig.json` | Modify | Slim — extends root, paths only |
|
||||||
|
| `packages/mcp-server/tsconfig.json` | Modify | Slim — extends root, paths only |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Add `bunfig.toml` and `turbo.json`
|
||||||
|
|
||||||
|
Two new root config files with no dependencies on other tasks.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `bunfig.toml`
|
||||||
|
- Create: `turbo.json`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Create `bunfig.toml`**
|
||||||
|
|
||||||
|
Write this file at repo root (`/path/to/ca-marketplace-scraper/bunfig.toml`):
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[install]
|
||||||
|
exact = true
|
||||||
|
|
||||||
|
[test]
|
||||||
|
root = "./do-not-run-tests-from-root"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Create `turbo.json`**
|
||||||
|
|
||||||
|
Write this file at repo root:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://turbo.build/schema.json",
|
||||||
|
"tasks": {
|
||||||
|
"typecheck": {},
|
||||||
|
"build": {
|
||||||
|
"dependsOn": ["^build"],
|
||||||
|
"outputs": ["dist/**"]
|
||||||
|
},
|
||||||
|
"test": {
|
||||||
|
"dependsOn": ["^build"],
|
||||||
|
"outputs": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify files exist**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
```bash
|
||||||
|
ls bunfig.toml turbo.json
|
||||||
|
```
|
||||||
|
Expected: both files listed, no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add bunfig.toml turbo.json
|
||||||
|
git commit -m "chore: add bunfig.toml and turbo.json"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Create root `tsconfig.json`
|
||||||
|
|
||||||
|
Shared base tsconfig all packages will extend. Extracts the common options currently duplicated in all 3 per-package tsconfigs.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `tsconfig.json`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Create root `tsconfig.json`**
|
||||||
|
|
||||||
|
Write this file at repo root:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"$schema": "https://json.schemastore.org/tsconfig",
|
||||||
|
"extends": "@tsconfig/bun/tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom", "ESNext"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "preserve",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"moduleDetection": "force",
|
||||||
|
"jsx": "react-jsx",
|
||||||
|
"allowJs": true,
|
||||||
|
"allowImportingTsExtensions": true,
|
||||||
|
"verbatimModuleSyntax": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
"noUncheckedIndexedAccess": true,
|
||||||
|
"noImplicitOverride": true,
|
||||||
|
"noUnusedLocals": false,
|
||||||
|
"noUnusedParameters": false,
|
||||||
|
"noPropertyAccessFromIndexSignature": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add tsconfig.json
|
||||||
|
git commit -m "chore: add shared root tsconfig.json"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Update root `package.json`
|
||||||
|
|
||||||
|
Add workspace catalog, `turbo` + `@tsconfig/bun` devDependencies, and update scripts to use `turbo run`.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `package.json`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Replace root `package.json`**
|
||||||
|
|
||||||
|
Write this complete file:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "marketplace-scrapers-monorepo",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"packageManager": "bun@1.3.13",
|
||||||
|
"scripts": {
|
||||||
|
"typecheck": "turbo run typecheck",
|
||||||
|
"build": "bun run clean && turbo run build",
|
||||||
|
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
||||||
|
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
||||||
|
"build:all": "bun run build:api && bun run build:mcp",
|
||||||
|
"ci": "biome ci",
|
||||||
|
"clean": "rm -rf dist",
|
||||||
|
"start": "./scripts/start.sh"
|
||||||
|
},
|
||||||
|
"workspaces": {
|
||||||
|
"packages": [
|
||||||
|
"packages/*"
|
||||||
|
],
|
||||||
|
"catalog": {
|
||||||
|
"@tsconfig/bun": "1.0.9",
|
||||||
|
"@typescript/native-preview": "7.0.0-dev.20260428.1",
|
||||||
|
"@types/bun": "1.2.18",
|
||||||
|
"@types/cli-progress": "3.11.6",
|
||||||
|
"@types/unidecode": "1.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@biomejs/biome": "2.3.11",
|
||||||
|
"@tsconfig/bun": "catalog:",
|
||||||
|
"turbo": "2.5.4"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Note on catalog versions:** The catalog pins exact versions. The values above are taken from the current package installs. If `@types/bun` was `latest`, check `node_modules/@types/bun/package.json` for the actual installed version and use that. Same for `@typescript/native-preview`.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Check actual installed versions**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
```bash
|
||||||
|
cat node_modules/@types/bun/package.json | grep '"version"'
|
||||||
|
cat node_modules/@typescript/native-preview/package.json | grep '"version"'
|
||||||
|
cat node_modules/@types/cli-progress/package.json | grep '"version"'
|
||||||
|
cat node_modules/@types/unidecode/package.json | grep '"version"'
|
||||||
|
```
|
||||||
|
|
||||||
|
Update the catalog values in `package.json` to match the exact installed versions.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Install turbo and @tsconfig/bun**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun install
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: lock file updated, `turbo` and `@tsconfig/bun` appear in `node_modules`.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify turbo works**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bunx turbo run typecheck --dry
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: output lists the `typecheck` task for each package (even if no `typecheck` script exists yet — turbo will note them as skipped/missing).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add package.json bun.lock
|
||||||
|
git commit -m "chore: add workspace catalog and turbo to root package.json"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: Update per-package `package.json` files
|
||||||
|
|
||||||
|
Rename `type:check` → `typecheck`, replace `main`/`module` with `exports`, swap pinned dep versions for `catalog:` references.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `packages/core/package.json`
|
||||||
|
- Modify: `packages/api-server/package.json`
|
||||||
|
- Modify: `packages/mcp-server/package.json`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Replace `packages/core/package.json`**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/core",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"exports": {
|
||||||
|
".": "./src/index.ts"
|
||||||
|
},
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"typecheck": "bun tsgo"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
|
"cli-progress": "^3.12.0",
|
||||||
|
"linkedom": "^0.18.12",
|
||||||
|
"unidecode": "^1.1.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "catalog:",
|
||||||
|
"@types/cli-progress": "catalog:",
|
||||||
|
"@types/unidecode": "catalog:"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Replace `packages/api-server/package.json`**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/api-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"exports": {
|
||||||
|
".": "./src/index.ts"
|
||||||
|
},
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun ./src/index.ts",
|
||||||
|
"dev": "bun --watch ./src/index.ts",
|
||||||
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api",
|
||||||
|
"typecheck": "bun tsgo"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
"@typescript/native-preview": "catalog:"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "catalog:"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Replace `packages/mcp-server/package.json`**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "@marketplace-scrapers/mcp-server",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"exports": {
|
||||||
|
".": "./src/index.ts"
|
||||||
|
},
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"start": "bun ./src/index.ts",
|
||||||
|
"dev": "bun --watch ./src/index.ts",
|
||||||
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp",
|
||||||
|
"typecheck": "bun tsgo"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
"@typescript/native-preview": "catalog:"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/bun": "catalog:"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"typescript": "^5"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run `bun install` to sync lockfile**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun install
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no errors. Catalog refs resolved. `bun.lock` updated.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Verify typecheck still works per-package**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd packages/core && bun run typecheck
|
||||||
|
cd ../api-server && bun run typecheck
|
||||||
|
cd ../mcp-server && bun run typecheck
|
||||||
|
cd ../..
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: each exits 0 (or same errors as before — no new errors introduced).
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add packages/core/package.json packages/api-server/package.json packages/mcp-server/package.json bun.lock
|
||||||
|
git commit -m "chore: use exports field and catalog refs in all packages"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: Slim per-package `tsconfig.json` files
|
||||||
|
|
||||||
|
Replace the duplicated full tsconfig in each package with a slim `extends`-based one pointing to root.
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `packages/core/tsconfig.json`
|
||||||
|
- Modify: `packages/api-server/tsconfig.json`
|
||||||
|
- Modify: `packages/mcp-server/tsconfig.json`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Replace `packages/core/tsconfig.json`**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": ["./src", "./test"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Replace `packages/api-server/tsconfig.json`**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": ["./src", "./test"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Replace `packages/mcp-server/tsconfig.json`**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"paths": {
|
||||||
|
"@/*": ["./src/*"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"include": ["./src", "./test"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify `@tsconfig/bun` is resolvable**
|
||||||
|
|
||||||
|
The root tsconfig extends `@tsconfig/bun/tsconfig.json`. Confirm the package is installed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ls node_modules/@tsconfig/bun/tsconfig.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: file exists.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Run typecheck via Turbo**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run typecheck
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: Turbo runs `typecheck` for all 3 packages in parallel, all pass (or same pre-existing errors — no new ones).
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add packages/core/tsconfig.json packages/api-server/tsconfig.json packages/mcp-server/tsconfig.json
|
||||||
|
git commit -m "chore: slim per-package tsconfigs to extend root"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 6: Smoke test full build pipeline
|
||||||
|
|
||||||
|
Verify everything works end-to-end.
|
||||||
|
|
||||||
|
**Files:** none (verification only)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Run turbo typecheck**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run typecheck
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: Turbo runs `typecheck` across all packages. Exit 0.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run full build**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bun run build
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: `dist/` cleaned, Turbo runs `build` (core first, then api-server and mcp-server in parallel), build artifacts appear in `dist/api/` and `dist/mcp/`.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Verify dist artifacts**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ls dist/api/ dist/mcp/
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: compiled output files in both directories.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Verify `bun install` is exact**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
grep -c '\^' bun.lock | head -5
|
||||||
|
```
|
||||||
|
|
||||||
|
With `exact = true` in bunfig.toml, new installs won't add `^` ranges. Existing `^` ranges in `bun.lock` from before are fine — they'll be resolved to exact on next fresh install.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Final commit if any loose files**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git status
|
||||||
|
```
|
||||||
|
|
||||||
|
If clean: done. If any files modified by `bun install` (e.g. `bun.lock`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add bun.lock
|
||||||
|
git commit -m "chore: sync lockfile after monorepo config adoption"
|
||||||
|
```
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
# Design: Adopt opencode Monorepo Config
|
||||||
|
|
||||||
|
**Date:** 2025-07-14
|
||||||
|
**Status:** Approved
|
||||||
|
**Approach:** Full adoption (A)
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Current repo (`marketplace-scrapers-monorepo`) has basic bun workspaces with 3 packages (`core`, `api-server`, `mcp-server`). Reference: `anomalyco/opencode` monorepo patterns.
|
||||||
|
|
||||||
|
**Gaps vs opencode:**
|
||||||
|
- No Turbo (task orchestration, caching, dep graph)
|
||||||
|
- No workspace catalog (shared dep versions duplicated across packages)
|
||||||
|
- No root tsconfig (identical tsconfigs duplicated in all 3 packages)
|
||||||
|
- No `bunfig.toml` (no exact installs, no root test guard)
|
||||||
|
- `main`/`module` fields instead of `exports` field
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
### 1. Root `package.json`
|
||||||
|
|
||||||
|
- Add `workspaces.catalog` block with shared deps:
|
||||||
|
- `@typescript/native-preview`, `@types/bun`, `@types/unidecode`, `@types/cli-progress`
|
||||||
|
- Add `turbo` to `devDependencies`
|
||||||
|
- Add `@tsconfig/bun` to `devDependencies` + catalog
|
||||||
|
- Update root scripts: `typecheck` and `build` delegate to `turbo run`
|
||||||
|
- Keep `build:api`, `build:mcp`, `build:all`, `start` as-is (deployment-specific)
|
||||||
|
- Rename `type:check` → `typecheck` in all packages (Turbo convention)
|
||||||
|
|
||||||
|
### 2. `turbo.json` (new file)
|
||||||
|
|
||||||
|
Tasks:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"tasks": {
|
||||||
|
"typecheck": {},
|
||||||
|
"build": { "dependsOn": ["^build"], "outputs": ["dist/**"] },
|
||||||
|
"test": { "dependsOn": ["^build"], "outputs": [] }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`core` builds before `api-server`/`mcp-server` due to `^build` dep.
|
||||||
|
|
||||||
|
### 3. Root `tsconfig.json` (new file)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"extends": "@tsconfig/bun/tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom", "ESNext"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "preserve",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"moduleDetection": "force",
|
||||||
|
"jsx": "react-jsx",
|
||||||
|
"allowJs": true,
|
||||||
|
"allowImportingTsExtensions": true,
|
||||||
|
"verbatimModuleSyntax": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
"noUncheckedIndexedAccess": true,
|
||||||
|
"noImplicitOverride": true,
|
||||||
|
"noUnusedLocals": false,
|
||||||
|
"noUnusedParameters": false,
|
||||||
|
"noPropertyAccessFromIndexSignature": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Per-package `tsconfig.json` (slim)
|
||||||
|
|
||||||
|
All 3 packages slim to:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"paths": { "@/*": ["./src/*"] }
|
||||||
|
},
|
||||||
|
"include": ["./src", "./test"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. `bunfig.toml` (new file)
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[install]
|
||||||
|
exact = true
|
||||||
|
|
||||||
|
[test]
|
||||||
|
root = "./do-not-run-tests-from-root"
|
||||||
|
```
|
||||||
|
|
||||||
|
Exact installs = reproducible. Root test guard prevents accidental root-level test runs.
|
||||||
|
|
||||||
|
### 6. Package `exports` field
|
||||||
|
|
||||||
|
Replace `main`/`module` with `exports` in all 3 packages:
|
||||||
|
```json
|
||||||
|
"exports": { ".": "./src/index.ts" }
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove `main` and `module` fields. Bun resolves `.ts` directly.
|
||||||
|
|
||||||
|
### 7. Catalog references in per-package `package.json`
|
||||||
|
|
||||||
|
Replace pinned versions with `"catalog:"` for shared deps:
|
||||||
|
- `@typescript/native-preview: "catalog:"`
|
||||||
|
- `@types/bun: "catalog:"`
|
||||||
|
- `@types/unidecode: "catalog:"` (core only)
|
||||||
|
- `@types/cli-progress: "catalog:"` (core only)
|
||||||
|
|
||||||
|
## Files Changed
|
||||||
|
|
||||||
|
| File | Action |
|
||||||
|
|---|---|
|
||||||
|
| `package.json` | Update (catalog, turbo dep, scripts) |
|
||||||
|
| `turbo.json` | Create |
|
||||||
|
| `tsconfig.json` | Create |
|
||||||
|
| `bunfig.toml` | Create |
|
||||||
|
| `packages/core/package.json` | Update (exports, catalog refs, script rename) |
|
||||||
|
| `packages/api-server/package.json` | Update (exports, catalog refs, script rename) |
|
||||||
|
| `packages/mcp-server/package.json` | Update (exports, catalog refs, script rename) |
|
||||||
|
| `packages/core/tsconfig.json` | Update (slim, extends root) |
|
||||||
|
| `packages/api-server/tsconfig.json` | Update (slim, extends root) |
|
||||||
|
| `packages/mcp-server/tsconfig.json` | Update (slim, extends root) |
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
- No Husky/git hooks (not needed yet)
|
||||||
|
- No SST/cloud infra (not applicable)
|
||||||
|
- No prettier (keep biome as formatter)
|
||||||
|
- No patches mechanism
|
||||||
|
- No `postinstall` scripts
|
||||||
32
package.json
32
package.json
@@ -1,21 +1,35 @@
|
|||||||
{
|
{
|
||||||
|
"$schema": "https://json.schemastore.org/package.json",
|
||||||
"name": "marketplace-scrapers-monorepo",
|
"name": "marketplace-scrapers-monorepo",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"packageManager": "bun@1.3.13",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"ci": "biome ci",
|
"typecheck": "turbo run typecheck",
|
||||||
"clean": "rm -rf dist",
|
"build": "bun run clean && turbo run build",
|
||||||
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
|
||||||
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
|
||||||
"build:all": "bun run build:api && bun run build:mcp",
|
"build:all": "bun run build:api && bun run build:mcp",
|
||||||
"build": "bun run clean && bun run build:all",
|
"ci": "biome ci",
|
||||||
|
"clean": "rm -rf dist",
|
||||||
"start": "./scripts/start.sh"
|
"start": "./scripts/start.sh"
|
||||||
},
|
},
|
||||||
"private": true,
|
"workspaces": {
|
||||||
"type": "module",
|
"packages": [
|
||||||
"workspaces": [
|
"packages/*"
|
||||||
"packages/*"
|
],
|
||||||
],
|
"catalog": {
|
||||||
|
"@tsconfig/bun": "1.0.9",
|
||||||
|
"@typescript/native-preview": "7.0.0-dev.20260428.1",
|
||||||
|
"@types/bun": "1.3.13",
|
||||||
|
"@types/cli-progress": "3.11.6",
|
||||||
|
"@types/unidecode": "1.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "2.3.11"
|
"@biomejs/biome": "2.3.11",
|
||||||
|
"@tsconfig/bun": "catalog:",
|
||||||
|
"turbo": "2.5.4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,18 +2,22 @@
|
|||||||
"name": "@marketplace-scrapers/api-server",
|
"name": "@marketplace-scrapers/api-server",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"module": "./src/index.ts",
|
"exports": {
|
||||||
|
".": "./src/index.ts"
|
||||||
|
},
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "bun ./src/index.ts",
|
"start": "bun ./src/index.ts",
|
||||||
"dev": "bun --watch ./src/index.ts",
|
"dev": "bun --watch ./src/index.ts",
|
||||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api"
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api",
|
||||||
|
"typecheck": "bun tsgo"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@marketplace-scrapers/core": "workspace:*"
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
"@typescript/native-preview": "catalog:"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest"
|
"@types/bun": "catalog:"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5"
|
"typescript": "^5"
|
||||||
|
|||||||
@@ -48,16 +48,16 @@ export async function kijijiRoute(req: Request): Promise<Response> {
|
|||||||
location: reqUrl.searchParams.get("location") || undefined,
|
location: reqUrl.searchParams.get("location") || undefined,
|
||||||
category: reqUrl.searchParams.get("category") || undefined,
|
category: reqUrl.searchParams.get("category") || undefined,
|
||||||
keywords: reqUrl.searchParams.get("keywords") || undefined,
|
keywords: reqUrl.searchParams.get("keywords") || undefined,
|
||||||
sortBy: (reqUrl.searchParams.get("sortBy") as
|
sortBy:
|
||||||
| "relevancy"
|
(reqUrl.searchParams.get("sortBy") as
|
||||||
| "date"
|
| "relevancy"
|
||||||
| "price"
|
| "date"
|
||||||
| "distance"
|
| "price"
|
||||||
| undefined) || undefined,
|
| "distance"
|
||||||
sortOrder: (reqUrl.searchParams.get("sortOrder") as
|
| undefined) || undefined,
|
||||||
| "desc"
|
sortOrder:
|
||||||
| "asc"
|
(reqUrl.searchParams.get("sortOrder") as "desc" | "asc" | undefined) ||
|
||||||
| undefined) || undefined,
|
undefined,
|
||||||
maxPages,
|
maxPages,
|
||||||
priceMin,
|
priceMin,
|
||||||
priceMax,
|
priceMax,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
|
|
||||||
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
|
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
|
||||||
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
|
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
|
||||||
@@ -123,17 +123,22 @@ describe("API routes", () => {
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||||
minPrice: undefined,
|
"laptop",
|
||||||
maxPrice: undefined,
|
1,
|
||||||
strictMode: false,
|
{
|
||||||
exclusions: [],
|
minPrice: undefined,
|
||||||
keywords: ["laptop"],
|
maxPrice: undefined,
|
||||||
buyItNowOnly: true,
|
strictMode: false,
|
||||||
canadaOnly: true,
|
exclusions: [],
|
||||||
}, {
|
keywords: ["laptop"],
|
||||||
hideUnstableResults: true,
|
buyItNowOnly: true,
|
||||||
});
|
canadaOnly: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
hideUnstableResults: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("kijijiRoute forwards unstableFilter=true to core", async () => {
|
test("kijijiRoute forwards unstableFilter=true to core", async () => {
|
||||||
@@ -202,9 +207,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
await ebayRoute(
|
await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&buyItNowOnly=true"),
|
||||||
"http://localhost/api/ebay?q=laptop&buyItNowOnly=true",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
||||||
@@ -242,9 +245,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
await kijijiRoute(
|
await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages=5"),
|
||||||
"http://localhost/api/kijiji?q=laptop&maxPages=5",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(fetchKijijiItems).toHaveBeenCalledWith(
|
expect(fetchKijijiItems).toHaveBeenCalledWith(
|
||||||
@@ -385,17 +386,17 @@ describe("API routes", () => {
|
|||||||
test("ebayRoute forwards maxItems to core in default mode", async () => {
|
test("ebayRoute forwards maxItems to core in default mode", async () => {
|
||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
fetchEbayItems.mockImplementation(() =>
|
fetchEbayItems.mockImplementation(() => Promise.resolve([{ title: "a" }]));
|
||||||
Promise.resolve([{ title: "a" }]),
|
|
||||||
);
|
|
||||||
|
|
||||||
await ebayRoute(
|
await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=2"),
|
||||||
"http://localhost/api/ebay?q=laptop&maxItems=2",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }));
|
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||||
|
"laptop",
|
||||||
|
1,
|
||||||
|
expect.objectContaining({ maxItems: 2 }),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
|
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
|
||||||
@@ -419,9 +420,14 @@ describe("API routes", () => {
|
|||||||
expect(body.unstableResults).toHaveLength(2);
|
expect(body.unstableResults).toHaveLength(2);
|
||||||
expect(body.results[0].title).toBe("a");
|
expect(body.results[0].title).toBe("a");
|
||||||
expect(body.unstableResults[0].title).toBe("d");
|
expect(body.unstableResults[0].title).toBe("d");
|
||||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 4 }), {
|
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||||
hideUnstableResults: true,
|
"laptop",
|
||||||
});
|
1,
|
||||||
|
expect.objectContaining({ maxItems: 4 }),
|
||||||
|
{
|
||||||
|
hideUnstableResults: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
|
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
|
||||||
@@ -440,9 +446,14 @@ describe("API routes", () => {
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }), {
|
expect(fetchEbayItems).toHaveBeenCalledWith(
|
||||||
hideUnstableResults: true,
|
"laptop",
|
||||||
});
|
1,
|
||||||
|
expect.objectContaining({ maxItems: 2 }),
|
||||||
|
{
|
||||||
|
hideUnstableResults: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("ebayRoute returns 404 when unstable results are empty", async () => {
|
test("ebayRoute returns 404 when unstable results are empty", async () => {
|
||||||
@@ -456,9 +467,7 @@ describe("API routes", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&unstableFilter=true"),
|
||||||
"http://localhost/api/ebay?q=laptop&unstableFilter=true",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(404);
|
expect(response.status).toBe(404);
|
||||||
@@ -470,9 +479,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=abc"),
|
||||||
"http://localhost/api/ebay?q=laptop&maxItems=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -484,9 +491,7 @@ describe("API routes", () => {
|
|||||||
const { facebookRoute } = await import("../src/routes/facebook");
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
const response = await facebookRoute(
|
const response = await facebookRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/facebook?q=laptop&maxItems=abc"),
|
||||||
"http://localhost/api/facebook?q=laptop&maxItems=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -498,9 +503,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&minPrice=abc"),
|
||||||
"http://localhost/api/ebay?q=laptop&minPrice=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -512,9 +515,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&maxPrice=abc"),
|
||||||
"http://localhost/api/ebay?q=laptop&maxPrice=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -526,9 +527,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages=abc"),
|
||||||
"http://localhost/api/kijiji?q=laptop&maxPages=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -540,9 +539,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin=abc"),
|
||||||
"http://localhost/api/kijiji?q=laptop&priceMin=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -554,9 +551,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax=abc"),
|
||||||
"http://localhost/api/kijiji?q=laptop&priceMax=abc",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -568,9 +563,7 @@ describe("API routes", () => {
|
|||||||
const { facebookRoute } = await import("../src/routes/facebook");
|
const { facebookRoute } = await import("../src/routes/facebook");
|
||||||
|
|
||||||
const response = await facebookRoute(
|
const response = await facebookRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/facebook?q=laptop&maxItems=-1"),
|
||||||
"http://localhost/api/facebook?q=laptop&maxItems=-1",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -582,9 +575,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&maxItems=-1"),
|
||||||
"http://localhost/api/ebay?q=laptop&maxItems=-1",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -596,9 +587,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&minPrice=-5"),
|
||||||
"http://localhost/api/ebay?q=laptop&minPrice=-5",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -610,9 +599,7 @@ describe("API routes", () => {
|
|||||||
const { ebayRoute } = await import("../src/routes/ebay");
|
const { ebayRoute } = await import("../src/routes/ebay");
|
||||||
|
|
||||||
const response = await ebayRoute(
|
const response = await ebayRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/ebay?q=laptop&maxPrice=-10"),
|
||||||
"http://localhost/api/ebay?q=laptop&maxPrice=-10",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -624,9 +611,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&maxPages=-2"),
|
||||||
"http://localhost/api/kijiji?q=laptop&maxPages=-2",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -638,9 +623,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&priceMin=-5"),
|
||||||
"http://localhost/api/kijiji?q=laptop&priceMin=-5",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
@@ -652,9 +635,7 @@ describe("API routes", () => {
|
|||||||
const { kijijiRoute } = await import("../src/routes/kijiji");
|
const { kijijiRoute } = await import("../src/routes/kijiji");
|
||||||
|
|
||||||
const response = await kijijiRoute(
|
const response = await kijijiRoute(
|
||||||
new Request(
|
new Request("http://localhost/api/kijiji?q=laptop&priceMax=-10"),
|
||||||
"http://localhost/api/kijiji?q=laptop&priceMax=-10",
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
|
|||||||
@@ -1,13 +1,9 @@
|
|||||||
{
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"lib": ["dom"],
|
|
||||||
"target": "ESNext",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleResolution": "bundler",
|
|
||||||
"paths": {
|
"paths": {
|
||||||
"@/*": ["./src/*"]
|
"@/*": ["./src/*"]
|
||||||
},
|
}
|
||||||
"strict": true,
|
},
|
||||||
"noEmit": true
|
"include": ["./src", "./test"]
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,18 +2,23 @@
|
|||||||
"name": "@marketplace-scrapers/core",
|
"name": "@marketplace-scrapers/core",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "./src/index.ts",
|
"exports": {
|
||||||
"module": "./src/index.ts",
|
".": "./src/index.ts"
|
||||||
|
},
|
||||||
"private": true,
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"typecheck": "bun tsgo"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@typescript/native-preview": "catalog:",
|
||||||
"cli-progress": "^3.12.0",
|
"cli-progress": "^3.12.0",
|
||||||
"linkedom": "^0.18.12",
|
"linkedom": "^0.18.12",
|
||||||
"unidecode": "^1.1.0"
|
"unidecode": "^1.1.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest",
|
"@types/bun": "catalog:",
|
||||||
"@types/unidecode": "^1.1.0",
|
"@types/cli-progress": "catalog:",
|
||||||
"@types/cli-progress": "^3.11.6"
|
"@types/unidecode": "catalog:"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5"
|
"typescript": "^5"
|
||||||
|
|||||||
@@ -4,13 +4,13 @@ import type {
|
|||||||
UnstableListingBuckets,
|
UnstableListingBuckets,
|
||||||
UnstableListingModeOptions,
|
UnstableListingModeOptions,
|
||||||
} from "../types/common";
|
} from "../types/common";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
|
||||||
import {
|
import {
|
||||||
type CookieConfig,
|
type CookieConfig,
|
||||||
ensureCookies,
|
ensureCookies,
|
||||||
formatCookiesForHeader,
|
formatCookiesForHeader,
|
||||||
} from "../utils/cookies";
|
} from "../utils/cookies";
|
||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
// eBay cookie configuration
|
// eBay cookie configuration
|
||||||
const EBAY_COOKIE_CONFIG: CookieConfig = {
|
const EBAY_COOKIE_CONFIG: CookieConfig = {
|
||||||
@@ -44,7 +44,9 @@ function canonicalizeEbayItemUrl(url: string): string {
|
|||||||
try {
|
try {
|
||||||
const parsed = new URL(url, "https://www.ebay.ca");
|
const parsed = new URL(url, "https://www.ebay.ca");
|
||||||
const match = parsed.pathname.match(/\/itm\/(?:[^/?#]+\/)?\d+/);
|
const match = parsed.pathname.match(/\/itm\/(?:[^/?#]+\/)?\d+/);
|
||||||
return match ? `${parsed.origin}${match[0]}` : `${parsed.origin}${parsed.pathname}`;
|
return match
|
||||||
|
? `${parsed.origin}${match[0]}`
|
||||||
|
: `${parsed.origin}${parsed.pathname}`;
|
||||||
} catch {
|
} catch {
|
||||||
return url;
|
return url;
|
||||||
}
|
}
|
||||||
@@ -275,11 +277,7 @@ function parseEbayListings(
|
|||||||
const actualPrices: HTMLElement[] = [];
|
const actualPrices: HTMLElement[] = [];
|
||||||
for (const el of allPriceElements) {
|
for (const el of allPriceElements) {
|
||||||
const text = el.textContent?.trim();
|
const text = el.textContent?.trim();
|
||||||
if (
|
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
|
||||||
text &&
|
|
||||||
EBAY_PRICE_TEXT_RE.test(text) &&
|
|
||||||
text.length < 50
|
|
||||||
) {
|
|
||||||
actualPrices.push(el);
|
actualPrices.push(el);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -386,16 +384,18 @@ async function loadEbayCookies(): Promise<string | undefined> {
|
|||||||
export default async function fetchEbayItems(
|
export default async function fetchEbayItems(
|
||||||
SEARCH_QUERY: string,
|
SEARCH_QUERY: string,
|
||||||
REQUESTS_PER_SECOND: number | undefined,
|
REQUESTS_PER_SECOND: number | undefined,
|
||||||
opts: {
|
opts:
|
||||||
minPrice?: number;
|
| {
|
||||||
maxPrice?: number;
|
minPrice?: number;
|
||||||
strictMode?: boolean;
|
maxPrice?: number;
|
||||||
exclusions?: string[];
|
strictMode?: boolean;
|
||||||
keywords?: string[];
|
exclusions?: string[];
|
||||||
buyItNowOnly?: boolean;
|
keywords?: string[];
|
||||||
canadaOnly?: boolean;
|
buyItNowOnly?: boolean;
|
||||||
maxItems?: number;
|
canadaOnly?: boolean;
|
||||||
} | undefined,
|
maxItems?: number;
|
||||||
|
}
|
||||||
|
| undefined,
|
||||||
unstableMode: { hideUnstableResults: true },
|
unstableMode: { hideUnstableResults: true },
|
||||||
): Promise<UnstableListingBuckets<EbayListingDetails>>;
|
): Promise<UnstableListingBuckets<EbayListingDetails>>;
|
||||||
export default async function fetchEbayItems(
|
export default async function fetchEbayItems(
|
||||||
@@ -529,7 +529,9 @@ export default async function fetchEbayItems(
|
|||||||
// Filter by price range (additional safety check)
|
// Filter by price range (additional safety check)
|
||||||
const filteredListings = listings.filter((listing) => {
|
const filteredListings = listings.filter((listing) => {
|
||||||
const cents = listing.listingPrice?.cents;
|
const cents = listing.listingPrice?.cents;
|
||||||
return typeof cents === "number" && cents >= minPrice && cents <= maxPrice;
|
return (
|
||||||
|
typeof cents === "number" && cents >= minPrice && cents <= maxPrice
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
console.log(`Parsed ${filteredListings.length} eBay listings.`);
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import type {
|
|||||||
UnstableListingBuckets,
|
UnstableListingBuckets,
|
||||||
UnstableListingModeOptions,
|
UnstableListingModeOptions,
|
||||||
} from "../types/common";
|
} from "../types/common";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
|
||||||
import {
|
import {
|
||||||
type Cookie,
|
type Cookie,
|
||||||
type CookieConfig,
|
type CookieConfig,
|
||||||
@@ -16,6 +15,7 @@ import {
|
|||||||
import { delay } from "../utils/delay";
|
import { delay } from "../utils/delay";
|
||||||
import { formatCentsToCurrency } from "../utils/format";
|
import { formatCentsToCurrency } from "../utils/format";
|
||||||
import { isRecord } from "../utils/http";
|
import { isRecord } from "../utils/http";
|
||||||
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Facebook Marketplace Scraper
|
* Facebook Marketplace Scraper
|
||||||
@@ -408,7 +408,11 @@ export function classifyFacebookResponse(
|
|||||||
htmlString.includes("This listing is no longer available") ||
|
htmlString.includes("This listing is no longer available") ||
|
||||||
htmlString.includes("listing has been removed");
|
htmlString.includes("listing has been removed");
|
||||||
if (unavailable) {
|
if (unavailable) {
|
||||||
return { kind: "unavailable" as const, authGated: false, unavailable: true };
|
return {
|
||||||
|
kind: "unavailable" as const,
|
||||||
|
authGated: false,
|
||||||
|
unavailable: true,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (responseUrl.includes("/marketplace/item/")) {
|
if (responseUrl.includes("/marketplace/item/")) {
|
||||||
@@ -455,7 +459,8 @@ function isFacebookSearchEdgeArray(value: unknown): value is FacebookEdge[] {
|
|||||||
Array.isArray(value) &&
|
Array.isArray(value) &&
|
||||||
value.length > 0 &&
|
value.length > 0 &&
|
||||||
value.every(
|
value.every(
|
||||||
(edge) => isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
|
(edge) =>
|
||||||
|
isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -552,8 +557,7 @@ function scoreMarketplaceItemPath(path: string[]): number {
|
|||||||
|
|
||||||
if (
|
if (
|
||||||
path.some(
|
path.some(
|
||||||
(segment) =>
|
(segment) => segment.includes("recommend") || segment.includes("related"),
|
||||||
segment.includes("recommend") || segment.includes("related"),
|
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
score -= 10;
|
score -= 10;
|
||||||
@@ -567,7 +571,9 @@ function collectMarketplaceItemCandidates(
|
|||||||
path: string[] = [],
|
path: string[] = [],
|
||||||
): FacebookMarketplaceItemMatch[] {
|
): FacebookMarketplaceItemMatch[] {
|
||||||
if (Array.isArray(candidate)) {
|
if (Array.isArray(candidate)) {
|
||||||
return candidate.flatMap((item) => collectMarketplaceItemCandidates(item, path));
|
return candidate.flatMap((item) =>
|
||||||
|
collectMarketplaceItemCandidates(item, path),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isRecord(candidate)) {
|
if (!isRecord(candidate)) {
|
||||||
@@ -628,7 +634,9 @@ function extractRenderedText(node: ParentNode, selector: string): string[] {
|
|||||||
.filter((text): text is string => Boolean(text));
|
.filter((text): text is string => Boolean(text));
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractMarketplaceItemIdFromElement(element: Element | null): string | null {
|
function extractMarketplaceItemIdFromElement(
|
||||||
|
element: Element | null,
|
||||||
|
): string | null {
|
||||||
const href = element?.getAttribute("href") || "";
|
const href = element?.getAttribute("href") || "";
|
||||||
return href.match(FACEBOOK_ITEM_HREF_RE)?.[1] ?? null;
|
return href.match(FACEBOOK_ITEM_HREF_RE)?.[1] ?? null;
|
||||||
}
|
}
|
||||||
@@ -666,7 +674,9 @@ function extractFacebookPermalinkItemId(document: Document): string | null {
|
|||||||
return extractMarketplaceItemIdFromElement(itemLinks.at(-1) ?? null);
|
return extractMarketplaceItemIdFromElement(itemLinks.at(-1) ?? null);
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractFacebookDescriptionText(document: Document): string | undefined {
|
function extractFacebookDescriptionText(
|
||||||
|
document: Document,
|
||||||
|
): string | undefined {
|
||||||
const labels = Array.from(document.querySelectorAll("div, span, h2, h3, p"));
|
const labels = Array.from(document.querySelectorAll("div, span, h2, h3, p"));
|
||||||
|
|
||||||
for (const label of labels) {
|
for (const label of labels) {
|
||||||
@@ -759,7 +769,10 @@ function extractFacebookItemHtmlFallback(
|
|||||||
const priceText = texts.find((text) => FACEBOOK_PRICE_TEXT_RE.test(text));
|
const priceText = texts.find((text) => FACEBOOK_PRICE_TEXT_RE.test(text));
|
||||||
const parsedPrice = priceText ? parseFacebookRenderedPrice(priceText) : null;
|
const parsedPrice = priceText ? parseFacebookRenderedPrice(priceText) : null;
|
||||||
const location = texts.find(
|
const location = texts.find(
|
||||||
(text) => text !== title && text !== priceText && FACEBOOK_LOCATION_TEXT_RE.test(text),
|
(text) =>
|
||||||
|
text !== title &&
|
||||||
|
text !== priceText &&
|
||||||
|
FACEBOOK_LOCATION_TEXT_RE.test(text),
|
||||||
);
|
);
|
||||||
const description = extractFacebookDescriptionText(document);
|
const description = extractFacebookDescriptionText(document);
|
||||||
|
|
||||||
@@ -841,7 +854,8 @@ export function extractFacebookItemData(
|
|||||||
if (
|
if (
|
||||||
!bestMatch ||
|
!bestMatch ||
|
||||||
match.score > bestMatch.score ||
|
match.score > bestMatch.score ||
|
||||||
(match.score === bestMatch.score && match.path.length < bestMatch.path.length)
|
(match.score === bestMatch.score &&
|
||||||
|
match.path.length < bestMatch.path.length)
|
||||||
) {
|
) {
|
||||||
bestMatch = match;
|
bestMatch = match;
|
||||||
}
|
}
|
||||||
@@ -1101,7 +1115,9 @@ export default async function fetchFacebookItems(
|
|||||||
|
|
||||||
const finalizeResults = (
|
const finalizeResults = (
|
||||||
listings: FacebookListingDetails[],
|
listings: FacebookListingDetails[],
|
||||||
): FacebookListingDetails[] | UnstableListingBuckets<FacebookListingDetails> => {
|
):
|
||||||
|
| FacebookListingDetails[]
|
||||||
|
| UnstableListingBuckets<FacebookListingDetails> => {
|
||||||
if (!unstableMode.hideUnstableResults) {
|
if (!unstableMode.hideUnstableResults) {
|
||||||
return listings.slice(0, MAX_ITEMS);
|
return listings.slice(0, MAX_ITEMS);
|
||||||
}
|
}
|
||||||
@@ -1166,9 +1182,14 @@ export default async function fetchFacebookItems(
|
|||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
const classification = classifyFacebookResponse(searchHtml, searchResponseUrl);
|
const classification = classifyFacebookResponse(
|
||||||
|
searchHtml,
|
||||||
|
searchResponseUrl,
|
||||||
|
);
|
||||||
if (classification.authGated) {
|
if (classification.authGated) {
|
||||||
console.warn("Facebook marketplace search redirected to login. Cookies may be expired.");
|
console.warn(
|
||||||
|
"Facebook marketplace search redirected to login. Cookies may be expired.",
|
||||||
|
);
|
||||||
return finalizeResults([]);
|
return finalizeResults([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1204,7 +1225,8 @@ export default async function fetchFacebookItems(
|
|||||||
// Filter to only priced items (already done in parseFacebookAds)
|
// Filter to only priced items (already done in parseFacebookAds)
|
||||||
const pricedItems = items.filter(
|
const pricedItems = items.filter(
|
||||||
(item) =>
|
(item) =>
|
||||||
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
|
typeof item.listingPrice?.cents === "number" &&
|
||||||
|
item.listingPrice.cents >= 0,
|
||||||
);
|
);
|
||||||
|
|
||||||
progressBar?.update(totalProgress);
|
progressBar?.update(totalProgress);
|
||||||
@@ -1293,7 +1315,9 @@ export async function fetchFacebookItem(
|
|||||||
|
|
||||||
if (classification.authGated) {
|
if (classification.authGated) {
|
||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
console.warn(`Authentication failed for item ${itemId}. Cookies may be expired.`);
|
console.warn(
|
||||||
|
`Authentication failed for item ${itemId}. Cookies may be expired.`,
|
||||||
|
);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1301,7 +1325,9 @@ export async function fetchFacebookItem(
|
|||||||
|
|
||||||
if (classification.unavailable && !itemData) {
|
if (classification.unavailable && !itemData) {
|
||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
|
console.warn(
|
||||||
|
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||||
|
);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1317,7 +1343,9 @@ export async function fetchFacebookItem(
|
|||||||
logExtractionMetrics(false, itemId);
|
logExtractionMetrics(false, itemId);
|
||||||
|
|
||||||
if (itemHtml.includes("This item has been sold")) {
|
if (itemHtml.includes("This item has been sold")) {
|
||||||
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
|
console.warn(
|
||||||
|
`Item ${itemId} appears to be sold or removed from marketplace.`,
|
||||||
|
);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import type {
|
|||||||
UnstableListingBuckets,
|
UnstableListingBuckets,
|
||||||
UnstableListingModeOptions,
|
UnstableListingModeOptions,
|
||||||
} from "../types/common";
|
} from "../types/common";
|
||||||
import { classifyUnstableListings } from "../utils/unstable";
|
|
||||||
import {
|
import {
|
||||||
type CookieConfig,
|
type CookieConfig,
|
||||||
formatCookiesForHeader,
|
formatCookiesForHeader,
|
||||||
@@ -22,6 +21,7 @@ import {
|
|||||||
RateLimitError,
|
RateLimitError,
|
||||||
ValidationError,
|
ValidationError,
|
||||||
} from "../utils/http";
|
} from "../utils/http";
|
||||||
|
import { classifyUnstableListings } from "../utils/unstable";
|
||||||
|
|
||||||
// Kijiji cookie configuration
|
// Kijiji cookie configuration
|
||||||
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
|
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
|
||||||
@@ -203,11 +203,17 @@ const SORT_MAPPINGS: Record<string, string> = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const LOCATION_SLUGS = Object.fromEntries(
|
const LOCATION_SLUGS = Object.fromEntries(
|
||||||
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
|
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [
|
||||||
|
id,
|
||||||
|
slug.replace(/\s+/g, "-"),
|
||||||
|
]),
|
||||||
) as Record<number, string>;
|
) as Record<number, string>;
|
||||||
|
|
||||||
const CATEGORY_SLUGS = Object.fromEntries(
|
const CATEGORY_SLUGS = Object.fromEntries(
|
||||||
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
|
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [
|
||||||
|
id,
|
||||||
|
slug.replace(/\s+/g, "-"),
|
||||||
|
]),
|
||||||
) as Record<number, string>;
|
) as Record<number, string>;
|
||||||
|
|
||||||
// ----------------------------- Utilities -----------------------------
|
// ----------------------------- Utilities -----------------------------
|
||||||
@@ -816,7 +822,10 @@ export default async function fetchKijijiItems(
|
|||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
// Set defaults for configuration
|
// Set defaults for configuration
|
||||||
const finalSearchOptions: Omit<Required<SearchOptions>, "priceMin" | "priceMax"> & {
|
const finalSearchOptions: Omit<
|
||||||
|
Required<SearchOptions>,
|
||||||
|
"priceMin" | "priceMax"
|
||||||
|
> & {
|
||||||
priceMin?: number;
|
priceMin?: number;
|
||||||
priceMax?: number;
|
priceMax?: number;
|
||||||
} = {
|
} = {
|
||||||
@@ -903,7 +912,9 @@ export default async function fetchKijijiItems(
|
|||||||
const batchPromises = batch.map(async (link, batchIndex) => {
|
const batchPromises = batch.map(async (link, batchIndex) => {
|
||||||
try {
|
try {
|
||||||
if (batchIndex > 0) {
|
if (batchIndex > 0) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, DELAY_MS * batchIndex));
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, DELAY_MS * batchIndex),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const html = await fetchHtml(link, 0, {
|
const html = await fetchHtml(link, 0, {
|
||||||
@@ -949,7 +960,6 @@ export default async function fetchKijijiItems(
|
|||||||
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
|
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
allListings.push(
|
allListings.push(
|
||||||
@@ -968,9 +978,7 @@ export default async function fetchKijijiItems(
|
|||||||
matchesPriceFilters(listing, finalSearchOptions),
|
matchesPriceFilters(listing, finalSearchOptions),
|
||||||
);
|
);
|
||||||
|
|
||||||
console.log(
|
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
|
||||||
`\nParsed ${filteredListings.length} detailed listings.`,
|
|
||||||
);
|
|
||||||
return finalizeResults(filteredListings);
|
return finalizeResults(filteredListings);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -136,7 +136,9 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
|
|
||||||
expect(results).toHaveLength(1);
|
expect(results).toHaveLength(1);
|
||||||
expect(results[0]).toEqual(
|
expect(results[0]).toEqual(
|
||||||
expect.objectContaining({ url: "https://www.ebay.ca/itm/123?_trkparms=foo" }),
|
expect.objectContaining({
|
||||||
|
url: "https://www.ebay.ca/itm/123?_trkparms=foo",
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -229,7 +231,10 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
|
|
||||||
expect(results).toEqual([
|
expect(results).toEqual([
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
|
listingPrice: expect.objectContaining({
|
||||||
|
currency: "USD",
|
||||||
|
cents: 12345,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -255,7 +260,10 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
|
|
||||||
expect(results).toEqual([
|
expect(results).toEqual([
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
|
listingPrice: expect.objectContaining({
|
||||||
|
currency: "USD",
|
||||||
|
cents: 12345,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -281,7 +289,10 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
|
|
||||||
expect(results).toEqual([
|
expect(results).toEqual([
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
listingPrice: expect.objectContaining({ currency: "GBP", cents: 12345 }),
|
listingPrice: expect.objectContaining({
|
||||||
|
currency: "GBP",
|
||||||
|
cents: 12345,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -314,10 +325,16 @@ describe("eBay Scraper Cookie Handling", () => {
|
|||||||
|
|
||||||
expect(results).toEqual([
|
expect(results).toEqual([
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
listingPrice: expect.objectContaining({ currency: "EUR", cents: 12345 }),
|
listingPrice: expect.objectContaining({
|
||||||
|
currency: "EUR",
|
||||||
|
cents: 12345,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
listingPrice: expect.objectContaining({ currency: "JPY", cents: 12300 }),
|
listingPrice: expect.objectContaining({
|
||||||
|
currency: "JPY",
|
||||||
|
cents: 12300,
|
||||||
|
}),
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,13 +2,13 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
|||||||
import cliProgress from "cli-progress";
|
import cliProgress from "cli-progress";
|
||||||
import {
|
import {
|
||||||
classifyFacebookResponse,
|
classifyFacebookResponse,
|
||||||
type FacebookListingDetails,
|
|
||||||
ensureFacebookCookies,
|
ensureFacebookCookies,
|
||||||
extractFacebookBootstrapCandidates,
|
extractFacebookBootstrapCandidates,
|
||||||
extractFacebookItemData,
|
extractFacebookItemData,
|
||||||
extractFacebookMarketplaceData,
|
extractFacebookMarketplaceData,
|
||||||
default as fetchFacebookItems,
|
type FacebookListingDetails,
|
||||||
fetchFacebookItem,
|
fetchFacebookItem,
|
||||||
|
default as fetchFacebookItems,
|
||||||
parseFacebookAds,
|
parseFacebookAds,
|
||||||
parseFacebookCookieString,
|
parseFacebookCookieString,
|
||||||
parseFacebookItem,
|
parseFacebookItem,
|
||||||
@@ -30,9 +30,13 @@ type IsExact<T, U> =
|
|||||||
const getDefaultFacebookItems = async () => fetchFacebookItems("chair");
|
const getDefaultFacebookItems = async () => fetchFacebookItems("chair");
|
||||||
const getUnstableFacebookItems = async (): Promise<
|
const getUnstableFacebookItems = async (): Promise<
|
||||||
UnstableListingBuckets<FacebookListingDetails>
|
UnstableListingBuckets<FacebookListingDetails>
|
||||||
> => fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
|
> =>
|
||||||
|
fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
|
||||||
type _FacebookDefaultReturn = Assert<
|
type _FacebookDefaultReturn = Assert<
|
||||||
IsExact<Awaited<ReturnType<typeof getDefaultFacebookItems>>, FacebookListingDetails[]>
|
IsExact<
|
||||||
|
Awaited<ReturnType<typeof getDefaultFacebookItems>>,
|
||||||
|
FacebookListingDetails[]
|
||||||
|
>
|
||||||
>;
|
>;
|
||||||
type _FacebookUnstableReturn = Assert<
|
type _FacebookUnstableReturn = Assert<
|
||||||
IsExact<
|
IsExact<
|
||||||
@@ -533,30 +537,32 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("returns an array by default", async () => {
|
test("returns an array by default", async () => {
|
||||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||||
payload: {
|
{
|
||||||
resultGroups: [
|
payload: {
|
||||||
{
|
resultGroups: [
|
||||||
edges: [
|
{
|
||||||
{
|
edges: [
|
||||||
node: {
|
{
|
||||||
listing: {
|
node: {
|
||||||
id: "1",
|
listing: {
|
||||||
marketplace_listing_title: "Stable Chair Listing",
|
id: "1",
|
||||||
listing_price: {
|
marketplace_listing_title: "Stable Chair Listing",
|
||||||
amount: "120.00",
|
listing_price: {
|
||||||
formatted_amount: "CA$120",
|
amount: "120.00",
|
||||||
currency: "CAD",
|
formatted_amount: "CA$120",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
},
|
||||||
})}</script></body></html>`;
|
)}</script></body></html>`;
|
||||||
|
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
@@ -576,30 +582,32 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("preserves free listings through the public fetch entrypoint", async () => {
|
test("preserves free listings through the public fetch entrypoint", async () => {
|
||||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||||
payload: {
|
{
|
||||||
resultGroups: [
|
payload: {
|
||||||
{
|
resultGroups: [
|
||||||
edges: [
|
{
|
||||||
{
|
edges: [
|
||||||
node: {
|
{
|
||||||
listing: {
|
node: {
|
||||||
id: "free-1",
|
listing: {
|
||||||
marketplace_listing_title: "Free Chair",
|
id: "free-1",
|
||||||
listing_price: {
|
marketplace_listing_title: "Free Chair",
|
||||||
amount: "0.00",
|
listing_price: {
|
||||||
formatted_amount: "FREE",
|
amount: "0.00",
|
||||||
currency: "CAD",
|
formatted_amount: "FREE",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
},
|
||||||
})}</script></body></html>`;
|
)}</script></body></html>`;
|
||||||
|
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
@@ -626,30 +634,32 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("does not start a progress bar when stdout is not a TTY", async () => {
|
test("does not start a progress bar when stdout is not a TTY", async () => {
|
||||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||||
payload: {
|
{
|
||||||
resultGroups: [
|
payload: {
|
||||||
{
|
resultGroups: [
|
||||||
edges: [
|
{
|
||||||
{
|
edges: [
|
||||||
node: {
|
{
|
||||||
listing: {
|
node: {
|
||||||
id: "1",
|
listing: {
|
||||||
marketplace_listing_title: "Chair Listing",
|
id: "1",
|
||||||
listing_price: {
|
marketplace_listing_title: "Chair Listing",
|
||||||
amount: "120.00",
|
listing_price: {
|
||||||
formatted_amount: "CA$120",
|
amount: "120.00",
|
||||||
currency: "CAD",
|
formatted_amount: "CA$120",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
},
|
||||||
})}</script></body></html>`;
|
)}</script></body></html>`;
|
||||||
|
|
||||||
process.stdout.isTTY = false;
|
process.stdout.isTTY = false;
|
||||||
const startSpy = mock(() => {});
|
const startSpy = mock(() => {});
|
||||||
@@ -688,58 +698,60 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("returns results and unstableResults when unstable mode is enabled", async () => {
|
test("returns results and unstableResults when unstable mode is enabled", async () => {
|
||||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||||
payload: {
|
{
|
||||||
resultGroups: [
|
payload: {
|
||||||
{
|
resultGroups: [
|
||||||
edges: [
|
{
|
||||||
{
|
edges: [
|
||||||
node: {
|
{
|
||||||
listing: {
|
node: {
|
||||||
id: "1",
|
listing: {
|
||||||
marketplace_listing_title: "Stable Chair Listing",
|
id: "1",
|
||||||
listing_price: {
|
marketplace_listing_title: "Stable Chair Listing",
|
||||||
amount: "100.00",
|
listing_price: {
|
||||||
formatted_amount: "CA$100",
|
amount: "100.00",
|
||||||
currency: "CAD",
|
formatted_amount: "CA$100",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
{
|
||||||
{
|
node: {
|
||||||
node: {
|
listing: {
|
||||||
listing: {
|
id: "2",
|
||||||
id: "2",
|
marketplace_listing_title: "Another Stable Chair",
|
||||||
marketplace_listing_title: "Another Stable Chair",
|
listing_price: {
|
||||||
listing_price: {
|
amount: "110.00",
|
||||||
amount: "110.00",
|
formatted_amount: "CA$110",
|
||||||
formatted_amount: "CA$110",
|
currency: "CAD",
|
||||||
currency: "CAD",
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
{
|
||||||
{
|
node: {
|
||||||
node: {
|
listing: {
|
||||||
listing: {
|
id: "3",
|
||||||
id: "3",
|
marketplace_listing_title: "Suspiciously Cheap Chair",
|
||||||
marketplace_listing_title: "Suspiciously Cheap Chair",
|
listing_price: {
|
||||||
listing_price: {
|
amount: "70.00",
|
||||||
amount: "70.00",
|
formatted_amount: "CA$70",
|
||||||
formatted_amount: "CA$70",
|
currency: "CAD",
|
||||||
currency: "CAD",
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
},
|
||||||
})}</script></body></html>`;
|
)}</script></body></html>`;
|
||||||
|
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
@@ -768,58 +780,61 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("unstable mode classifies before the final MAX_ITEMS limit", async () => {
|
test("unstable mode classifies before the final MAX_ITEMS limit", async () => {
|
||||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||||
payload: {
|
{
|
||||||
resultGroups: [
|
payload: {
|
||||||
{
|
resultGroups: [
|
||||||
edges: [
|
{
|
||||||
{
|
edges: [
|
||||||
node: {
|
{
|
||||||
listing: {
|
node: {
|
||||||
id: "1",
|
listing: {
|
||||||
marketplace_listing_title: "Boundary Stable Chair",
|
id: "1",
|
||||||
listing_price: {
|
marketplace_listing_title: "Boundary Stable Chair",
|
||||||
amount: "100.00",
|
listing_price: {
|
||||||
formatted_amount: "CA$100",
|
amount: "100.00",
|
||||||
currency: "CAD",
|
formatted_amount: "CA$100",
|
||||||
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
{
|
||||||
{
|
node: {
|
||||||
node: {
|
listing: {
|
||||||
listing: {
|
id: "2",
|
||||||
id: "2",
|
marketplace_listing_title:
|
||||||
marketplace_listing_title: "Second Boundary Stable Chair",
|
"Second Boundary Stable Chair",
|
||||||
listing_price: {
|
listing_price: {
|
||||||
amount: "110.00",
|
amount: "110.00",
|
||||||
formatted_amount: "CA$110",
|
formatted_amount: "CA$110",
|
||||||
currency: "CAD",
|
currency: "CAD",
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
{
|
||||||
{
|
node: {
|
||||||
node: {
|
listing: {
|
||||||
listing: {
|
id: "3",
|
||||||
id: "3",
|
marketplace_listing_title: "Past Boundary Cheap Chair",
|
||||||
marketplace_listing_title: "Past Boundary Cheap Chair",
|
listing_price: {
|
||||||
listing_price: {
|
amount: "70.00",
|
||||||
amount: "70.00",
|
formatted_amount: "CA$70",
|
||||||
formatted_amount: "CA$70",
|
currency: "CAD",
|
||||||
currency: "CAD",
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
},
|
||||||
})}</script></body></html>`;
|
)}</script></body></html>`;
|
||||||
|
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
@@ -869,7 +884,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
|||||||
},
|
},
|
||||||
redacted_description: { text: "Solid wood chair" },
|
redacted_description: { text: "Solid wood chair" },
|
||||||
location_text: { text: "Toronto, ON" },
|
location_text: { text: "Toronto, ON" },
|
||||||
marketplace_listing_seller: { id: "seller-1", name: "Alex" },
|
marketplace_listing_seller: {
|
||||||
|
id: "seller-1",
|
||||||
|
name: "Alex",
|
||||||
|
},
|
||||||
condition: "USED",
|
condition: "USED",
|
||||||
is_live: true,
|
is_live: true,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
import fetchFacebookItems, { fetchFacebookItem } from "../src/scrapers/facebook";
|
import fetchFacebookItems, {
|
||||||
|
fetchFacebookItem,
|
||||||
|
} from "../src/scrapers/facebook";
|
||||||
|
|
||||||
// Mock fetch globally
|
// Mock fetch globally
|
||||||
const originalFetch = global.fetch;
|
const originalFetch = global.fetch;
|
||||||
@@ -27,35 +29,37 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
|||||||
|
|
||||||
describe("Main Search Function", () => {
|
describe("Main Search Function", () => {
|
||||||
test("should successfully fetch search results", async () => {
|
test("should successfully fetch search results", async () => {
|
||||||
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
|
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
|
||||||
payload: {
|
{
|
||||||
resultGroups: [
|
payload: {
|
||||||
{
|
resultGroups: [
|
||||||
edges: [
|
{
|
||||||
{
|
edges: [
|
||||||
node: {
|
{
|
||||||
listing: {
|
node: {
|
||||||
id: "1",
|
listing: {
|
||||||
marketplace_listing_title: "iPhone 13",
|
id: "1",
|
||||||
listing_price: {
|
marketplace_listing_title: "iPhone 13",
|
||||||
amount: "500.00",
|
listing_price: {
|
||||||
formatted_amount: "CA$500",
|
amount: "500.00",
|
||||||
currency: "CAD",
|
formatted_amount: "CA$500",
|
||||||
},
|
currency: "CAD",
|
||||||
location: {
|
|
||||||
reverse_geocode: {
|
|
||||||
city_page: { display_name: "Toronto" },
|
|
||||||
},
|
},
|
||||||
|
location: {
|
||||||
|
reverse_geocode: {
|
||||||
|
city_page: { display_name: "Toronto" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
is_live: true,
|
||||||
},
|
},
|
||||||
is_live: true,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
},
|
},
|
||||||
})}</script></body></html>`;
|
)}</script></body></html>`;
|
||||||
|
|
||||||
global.fetch = mock(() =>
|
global.fetch = mock(() =>
|
||||||
Promise.resolve({
|
Promise.resolve({
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||||
import {
|
import {
|
||||||
buildSearchUrl,
|
buildSearchUrl,
|
||||||
default as fetchKijijiItems,
|
|
||||||
type DetailedListing,
|
type DetailedListing,
|
||||||
|
default as fetchKijijiItems,
|
||||||
NetworkError,
|
NetworkError,
|
||||||
parseSearch,
|
|
||||||
parseDetailedListing,
|
|
||||||
ParseError,
|
ParseError,
|
||||||
|
parseDetailedListing,
|
||||||
|
parseSearch,
|
||||||
RateLimitError,
|
RateLimitError,
|
||||||
resolveCategoryId,
|
resolveCategoryId,
|
||||||
resolveLocationId,
|
resolveLocationId,
|
||||||
@@ -282,7 +282,8 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-low/k0l0")) {
|
if (url.endsWith("/v-low/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -291,7 +292,8 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-mid/k0l0")) {
|
if (url.endsWith("/v-mid/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -300,7 +302,8 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-high/k0l0")) {
|
if (url.endsWith("/v-high/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -534,9 +537,18 @@ describe("fetchKijijiItems", () => {
|
|||||||
props: {
|
props: {
|
||||||
pageProps: {
|
pageProps: {
|
||||||
__APOLLO_STATE__: {
|
__APOLLO_STATE__: {
|
||||||
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
|
"Listing:1": {
|
||||||
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
|
url: "/v-stable-one/k0l0",
|
||||||
"Listing:3": { url: "/v-unstable/k0l0", title: "Unstable Listing" },
|
title: "Stable Listing One",
|
||||||
|
},
|
||||||
|
"Listing:2": {
|
||||||
|
url: "/v-stable-two/k0l0",
|
||||||
|
title: "Stable Listing Two",
|
||||||
|
},
|
||||||
|
"Listing:3": {
|
||||||
|
url: "/v-unstable/k0l0",
|
||||||
|
title: "Unstable Listing",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -582,7 +594,10 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-stable-one/k0l0")) {
|
if (url.endsWith("/v-stable-one/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -591,7 +606,10 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-stable-two/k0l0")) {
|
if (url.endsWith("/v-stable-two/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -600,7 +618,10 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-unstable/k0l0")) {
|
if (url.endsWith("/v-unstable/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Unstable Listing", 7000, "v-unstable/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml("Unstable Listing", 7000, "v-unstable/k0l0"),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -635,10 +656,22 @@ describe("fetchKijijiItems", () => {
|
|||||||
props: {
|
props: {
|
||||||
pageProps: {
|
pageProps: {
|
||||||
__APOLLO_STATE__: {
|
__APOLLO_STATE__: {
|
||||||
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
|
"Listing:1": {
|
||||||
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
|
url: "/v-stable-one/k0l0",
|
||||||
"Listing:3": { url: "/v-out-of-range-high/k0l0", title: "Out Of Range High" },
|
title: "Stable Listing One",
|
||||||
"Listing:4": { url: "/v-out-of-range-low/k0l0", title: "Out Of Range Low" },
|
},
|
||||||
|
"Listing:2": {
|
||||||
|
url: "/v-stable-two/k0l0",
|
||||||
|
title: "Stable Listing Two",
|
||||||
|
},
|
||||||
|
"Listing:3": {
|
||||||
|
url: "/v-out-of-range-high/k0l0",
|
||||||
|
title: "Out Of Range High",
|
||||||
|
},
|
||||||
|
"Listing:4": {
|
||||||
|
url: "/v-out-of-range-low/k0l0",
|
||||||
|
title: "Out Of Range Low",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -672,7 +705,11 @@ describe("fetchKijijiItems", () => {
|
|||||||
global.fetch = mock((input: string | URL | Request) => {
|
global.fetch = mock((input: string | URL | Request) => {
|
||||||
const url = typeof input === "string" ? input : input.toString();
|
const url = typeof input === "string" ? input : input.toString();
|
||||||
|
|
||||||
if (url.includes("/k0c0l1700272") && url.includes("priceMin=80") && url.includes("priceMax=150")) {
|
if (
|
||||||
|
url.includes("/k0c0l1700272") &&
|
||||||
|
url.includes("priceMin=80") &&
|
||||||
|
url.includes("priceMax=150")
|
||||||
|
) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(searchHtml),
|
text: () => Promise.resolve(searchHtml),
|
||||||
@@ -684,7 +721,10 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-stable-one/k0l0")) {
|
if (url.endsWith("/v-stable-one/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -693,7 +733,10 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-stable-two/k0l0")) {
|
if (url.endsWith("/v-stable-two/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -702,7 +745,14 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-out-of-range-high/k0l0")) {
|
if (url.endsWith("/v-out-of-range-high/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Out Of Range High", 20000, "v-out-of-range-high/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml(
|
||||||
|
"Out Of Range High",
|
||||||
|
20000,
|
||||||
|
"v-out-of-range-high/k0l0",
|
||||||
|
),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
@@ -711,7 +761,10 @@ describe("fetchKijijiItems", () => {
|
|||||||
if (url.endsWith("/v-out-of-range-low/k0l0")) {
|
if (url.endsWith("/v-out-of-range-low/k0l0")) {
|
||||||
return Promise.resolve({
|
return Promise.resolve({
|
||||||
ok: true,
|
ok: true,
|
||||||
text: () => Promise.resolve(listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0")),
|
text: () =>
|
||||||
|
Promise.resolve(
|
||||||
|
listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0"),
|
||||||
|
),
|
||||||
headers: { get: () => null },
|
headers: { get: () => null },
|
||||||
url,
|
url,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -31,8 +31,13 @@ describe("classifyUnstableListings", () => {
|
|||||||
|
|
||||||
const buckets = classifyUnstableListings(listings);
|
const buckets = classifyUnstableListings(listings);
|
||||||
|
|
||||||
expect(buckets.results.map((listing) => listing.id)).toEqual(["stable-1", "stable-2"]);
|
expect(buckets.results.map((listing) => listing.id)).toEqual([
|
||||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
|
"stable-1",
|
||||||
|
"stable-2",
|
||||||
|
]);
|
||||||
|
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
|
||||||
|
"unstable",
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("uses the midpoint median for even-sized priced inputs", () => {
|
test("uses the midpoint median for even-sized priced inputs", () => {
|
||||||
@@ -45,8 +50,14 @@ describe("classifyUnstableListings", () => {
|
|||||||
|
|
||||||
const buckets = classifyUnstableListings(listings);
|
const buckets = classifyUnstableListings(listings);
|
||||||
|
|
||||||
expect(buckets.results.map((listing) => listing.id)).toEqual(["mid-low", "mid-high", "high"]);
|
expect(buckets.results.map((listing) => listing.id)).toEqual([
|
||||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["low"]);
|
"mid-low",
|
||||||
|
"mid-high",
|
||||||
|
"high",
|
||||||
|
]);
|
||||||
|
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
|
||||||
|
"low",
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps non-positive prices in results and excludes them from the median input", () => {
|
test("keeps non-positive prices in results and excludes them from the median input", () => {
|
||||||
@@ -66,7 +77,9 @@ describe("classifyUnstableListings", () => {
|
|||||||
"stable-1",
|
"stable-1",
|
||||||
"stable-2",
|
"stable-2",
|
||||||
]);
|
]);
|
||||||
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
|
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
|
||||||
|
"unstable",
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("returns all listings in results when fewer than two valid prices are present", () => {
|
test("returns all listings in results when fewer than two valid prices are present", () => {
|
||||||
@@ -78,7 +91,11 @@ describe("classifyUnstableListings", () => {
|
|||||||
|
|
||||||
const buckets = classifyUnstableListings(listings);
|
const buckets = classifyUnstableListings(listings);
|
||||||
|
|
||||||
expect(buckets.results.map((listing) => listing.id)).toEqual(["zero", "negative", "only-valid"]);
|
expect(buckets.results.map((listing) => listing.id)).toEqual([
|
||||||
|
"zero",
|
||||||
|
"negative",
|
||||||
|
"only-valid",
|
||||||
|
]);
|
||||||
expect(buckets.unstableResults).toEqual([]);
|
expect(buckets.unstableResults).toEqual([]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,13 +1,9 @@
|
|||||||
{
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"lib": ["dom"],
|
|
||||||
"target": "ESNext",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleResolution": "bundler",
|
|
||||||
"paths": {
|
"paths": {
|
||||||
"@/*": ["./src/*"]
|
"@/*": ["./src/*"]
|
||||||
},
|
}
|
||||||
"strict": true,
|
},
|
||||||
"noEmit": true
|
"include": ["./src", "./test"]
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,18 +2,22 @@
|
|||||||
"name": "@marketplace-scrapers/mcp-server",
|
"name": "@marketplace-scrapers/mcp-server",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"module": "./src/index.ts",
|
"exports": {
|
||||||
|
".": "./src/index.ts"
|
||||||
|
},
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "bun ./src/index.ts",
|
"start": "bun ./src/index.ts",
|
||||||
"dev": "bun --watch ./src/index.ts",
|
"dev": "bun --watch ./src/index.ts",
|
||||||
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp"
|
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp",
|
||||||
|
"typecheck": "bun tsgo"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@marketplace-scrapers/core": "workspace:*"
|
"@marketplace-scrapers/core": "workspace:*",
|
||||||
|
"@typescript/native-preview": "catalog:"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/bun": "latest"
|
"@types/bun": "catalog:"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"typescript": "^5"
|
"typescript": "^5"
|
||||||
|
|||||||
@@ -72,7 +72,10 @@ describe("MCP protocol unstableFilter", () => {
|
|||||||
const tool = tools.find((t) => t.name === toolName);
|
const tool = tools.find((t) => t.name === toolName);
|
||||||
expect(tool).toBeDefined();
|
expect(tool).toBeDefined();
|
||||||
expect(tool?.inputSchema.properties).toHaveProperty("unstableFilter");
|
expect(tool?.inputSchema.properties).toHaveProperty("unstableFilter");
|
||||||
const prop = tool?.inputSchema.properties.unstableFilter as any;
|
const prop = tool?.inputSchema.properties.unstableFilter as {
|
||||||
|
type: string;
|
||||||
|
description: string;
|
||||||
|
};
|
||||||
expect(prop.type).toBe("boolean");
|
expect(prop.type).toBe("boolean");
|
||||||
expect(prop.description).toContain("optional");
|
expect(prop.description).toContain("optional");
|
||||||
expect(prop.description).toContain("20%");
|
expect(prop.description).toContain("20%");
|
||||||
|
|||||||
@@ -1,13 +1,9 @@
|
|||||||
{
|
{
|
||||||
|
"extends": "../../tsconfig.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"lib": ["dom"],
|
|
||||||
"target": "ESNext",
|
|
||||||
"module": "ESNext",
|
|
||||||
"moduleResolution": "bundler",
|
|
||||||
"paths": {
|
"paths": {
|
||||||
"@/*": ["./src/*"]
|
"@/*": ["./src/*"]
|
||||||
},
|
}
|
||||||
"strict": true,
|
},
|
||||||
"noEmit": true
|
"include": ["./src", "./test"]
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
24
tsconfig.json
Normal file
24
tsconfig.json
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json.schemastore.org/tsconfig",
|
||||||
|
"extends": "@tsconfig/bun/tsconfig.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"lib": ["dom", "ESNext"],
|
||||||
|
"target": "ESNext",
|
||||||
|
"module": "preserve",
|
||||||
|
"moduleResolution": "bundler",
|
||||||
|
"strict": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"moduleDetection": "force",
|
||||||
|
"jsx": "react-jsx",
|
||||||
|
"allowJs": true,
|
||||||
|
"allowImportingTsExtensions": true,
|
||||||
|
"verbatimModuleSyntax": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
"noUncheckedIndexedAccess": true,
|
||||||
|
"noImplicitOverride": true,
|
||||||
|
"noUnusedLocals": false,
|
||||||
|
"noUnusedParameters": false,
|
||||||
|
"noPropertyAccessFromIndexSignature": false
|
||||||
|
}
|
||||||
|
}
|
||||||
14
turbo.json
Normal file
14
turbo.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://turbo.build/schema.json",
|
||||||
|
"tasks": {
|
||||||
|
"typecheck": {},
|
||||||
|
"build": {
|
||||||
|
"dependsOn": ["^build"],
|
||||||
|
"outputs": ["../../dist/**"]
|
||||||
|
},
|
||||||
|
"test": {
|
||||||
|
"dependsOn": ["^build"],
|
||||||
|
"outputs": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user