Compare commits

...

12 Commits

Author SHA1 Message Date
d77a006ded chore: ignore .turbo cache dirs 2026-04-28 19:56:39 -04:00
56b2198df1 chore: fix turbo build outputs path to match actual dist location 2026-04-28 19:56:29 -04:00
63716272c5 chore: slim per-package tsconfigs to extend root 2026-04-28 19:55:59 -04:00
1d21c66945 chore: use exports field and catalog refs in all packages 2026-04-28 19:55:37 -04:00
f2f78225f3 chore: add workspace catalog and turbo to root package.json 2026-04-28 19:54:46 -04:00
43d15fce5f chore: add shared root tsconfig.json 2026-04-28 19:53:58 -04:00
fef2f1968a chore: add bunfig.toml and turbo.json 2026-04-28 19:53:47 -04:00
01081f6b2e docs: add opencode monorepo config adoption implementation plan 2026-04-28 19:52:28 -04:00
d10d5305a3 docs: add opencode monorepo config adoption design spec 2026-04-28 19:50:51 -04:00
bf393eacae chore: setup typecheck scripts for each package
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-04-28 19:37:18 -04:00
79bb249603 chore: replace any cast by asserting tool schema property types
Tightens the type assertion for the `unstableFilter` schema property in tests to ensure correct structural checking of its `type` and `description` fields.
2026-04-28 19:24:39 -04:00
957e0f137b chore: biome lint and formatting
Signed-off-by: Dmytro Stanchiev <git@dmytros.dev>
2026-04-28 19:21:16 -04:00
25 changed files with 1238 additions and 391 deletions

1
.gitignore vendored
View File

@@ -33,6 +33,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
.eslintcache
.cache
*.tsbuildinfo
.turbo
# IntelliJ based IDEs
.idea

View File

@@ -6,6 +6,8 @@
"name": "marketplace-scrapers-monorepo",
"devDependencies": {
"@biomejs/biome": "2.3.11",
"@tsconfig/bun": "catalog:",
"turbo": "2.5.4",
},
},
"packages/api-server": {
@@ -13,9 +15,10 @@
"version": "1.0.0",
"dependencies": {
"@marketplace-scrapers/core": "workspace:*",
"@typescript/native-preview": "catalog:",
},
"devDependencies": {
"@types/bun": "latest",
"@types/bun": "catalog:",
},
"peerDependencies": {
"typescript": "^5",
@@ -25,14 +28,15 @@
"name": "@marketplace-scrapers/core",
"version": "1.0.0",
"dependencies": {
"@typescript/native-preview": "catalog:",
"cli-progress": "^3.12.0",
"linkedom": "^0.18.12",
"unidecode": "^1.1.0",
},
"devDependencies": {
"@types/bun": "latest",
"@types/cli-progress": "^3.11.6",
"@types/unidecode": "^1.1.0",
"@types/bun": "catalog:",
"@types/cli-progress": "catalog:",
"@types/unidecode": "catalog:",
},
"peerDependencies": {
"typescript": "^5",
@@ -43,15 +47,23 @@
"version": "1.0.0",
"dependencies": {
"@marketplace-scrapers/core": "workspace:*",
"@typescript/native-preview": "catalog:",
},
"devDependencies": {
"@types/bun": "latest",
"@types/bun": "catalog:",
},
"peerDependencies": {
"typescript": "^5",
},
},
},
"catalog": {
"@tsconfig/bun": "1.0.9",
"@types/bun": "1.3.13",
"@types/cli-progress": "3.11.6",
"@types/unidecode": "1.1.0",
"@typescript/native-preview": "7.0.0-dev.20260428.1",
},
"packages": {
"@biomejs/biome": ["@biomejs/biome@2.3.11", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.11", "@biomejs/cli-darwin-x64": "2.3.11", "@biomejs/cli-linux-arm64": "2.3.11", "@biomejs/cli-linux-arm64-musl": "2.3.11", "@biomejs/cli-linux-x64": "2.3.11", "@biomejs/cli-linux-x64-musl": "2.3.11", "@biomejs/cli-win32-arm64": "2.3.11", "@biomejs/cli-win32-x64": "2.3.11" }, "bin": { "biome": "bin/biome" } }, "sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ=="],
@@ -77,7 +89,9 @@
"@marketplace-scrapers/mcp-server": ["@marketplace-scrapers/mcp-server@workspace:packages/mcp-server"],
"@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="],
"@tsconfig/bun": ["@tsconfig/bun@1.0.9", "", {}, "sha512-4M0/Ivfwcpz325z6CwSifOBZYji3DFOEpY6zEUt0+Xi2qRhzwvmqQN9XAHJh3OVvRJuAqVTLU2abdCplvp6mwQ=="],
"@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="],
"@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="],
@@ -85,11 +99,27 @@
"@types/unidecode": ["@types/unidecode@1.1.0", "", {}, "sha512-NTIsFsTe9WRek39/8DDj7KiQ0nU33DHMrKwNHcD1rKlUvn4N0Rc4Di8q/Xavs8bsDZmBa4MMtQA8+HNgwfxC/A=="],
"@typescript/native-preview": ["@typescript/native-preview@7.0.0-dev.20260428.1", "", { "optionalDependencies": { "@typescript/native-preview-darwin-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-darwin-x64": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-arm": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-linux-x64": "7.0.0-dev.20260428.1", "@typescript/native-preview-win32-arm64": "7.0.0-dev.20260428.1", "@typescript/native-preview-win32-x64": "7.0.0-dev.20260428.1" }, "bin": { "tsgo": "bin/tsgo.js" } }, "sha512-JiM4PYWDGs57TT0mV2KArmaW7BnTkk3XRid79NdG17tfvDbRyg4hBCpKI7vARiQPtxjKrHlxyzxOGDpv5W5T7Q=="],
"@typescript/native-preview-darwin-arm64": ["@typescript/native-preview-darwin-arm64@7.0.0-dev.20260428.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Lll6WmXfgTEj1G3QBIoHlabQwUtJiyhlRgSLksa06QFL5BoA7V+Lu1waa9PtPNZbGsXLDMHodtk/bRQABKuPiw=="],
"@typescript/native-preview-darwin-x64": ["@typescript/native-preview-darwin-x64@7.0.0-dev.20260428.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-WbsBNSHlo+4sGrTxDWdmI7r8x48tCtSCuKdmK62FvVOq58UWAs6sL13Z4Rev4ohLcGHdXC5E/8AIdpLPqDYQpw=="],
"@typescript/native-preview-linux-arm": ["@typescript/native-preview-linux-arm@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "arm" }, "sha512-/d/NnZFvEJU67L5mHh+cO3gsfwNCvJ9HGtxGq1KGz1VwTabOIcwLdpTpfsAR39WXzzfh9GJHL28n6GSGZInPow=="],
"@typescript/native-preview-linux-arm64": ["@typescript/native-preview-linux-arm64@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-cgcBX/ZBMdepkamLT8g8jQdHe7DZS/s6zTZRof6mvcrnJHlMeUnKoC9UO8/c22IrUMV3n0XPh7R8FYjUP0ll+Q=="],
"@typescript/native-preview-linux-x64": ["@typescript/native-preview-linux-x64@7.0.0-dev.20260428.1", "", { "os": "linux", "cpu": "x64" }, "sha512-4gJCE7wzenx1BH2Vtx2uKWUo8rFxnhGkxNEH1zxbYy/6ASwo+PnOPYmKHAzNE1C3yB5lzw71/vR5p5zyO57Y4A=="],
"@typescript/native-preview-win32-arm64": ["@typescript/native-preview-win32-arm64@7.0.0-dev.20260428.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-yn6Rzbn62L4QTWrp0QgG8al6l/VG7PCPRdbE0vuGDSlKhInlC+Flo4QSc1qA8KHTbpHgl+nEsq9DymiitI4G4g=="],
"@typescript/native-preview-win32-x64": ["@typescript/native-preview-win32-x64@7.0.0-dev.20260428.1", "", { "os": "win32", "cpu": "x64" }, "sha512-T9z13mcMowXmwGjprA2FIR2EEdYZxgqH8+qk7dFZVBlo5vfk41AN/qJfAdN7IsAhEb640MJ8cMN/aiczweZKmA=="],
"ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="],
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
"bun-types": ["bun-types@1.3.4", "", { "dependencies": { "@types/node": "*" } }, "sha512-5ua817+BZPZOlNaRgGBpZJOSAQ9RQ17pkwPD0yR7CfJg+r8DgIILByFifDTa+IPDDxzf5VNhtNlcKqFzDgJvlQ=="],
"bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="],
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
@@ -125,6 +155,20 @@
"strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="],
"turbo": ["turbo@2.5.4", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.4", "turbo-darwin-arm64": "2.5.4", "turbo-linux-64": "2.5.4", "turbo-linux-arm64": "2.5.4", "turbo-windows-64": "2.5.4", "turbo-windows-arm64": "2.5.4" }, "bin": { "turbo": "bin/turbo" } }, "sha512-kc8ZibdRcuWUG1pbYSBFWqmIjynlD8Lp7IB6U3vIzvOv9VG+6Sp8bzyeBWE3Oi8XV5KsQrznyRTBPvrf99E4mA=="],
"turbo-darwin-64": ["turbo-darwin-64@2.5.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-ah6YnH2dErojhFooxEzmvsoZQTMImaruZhFPfMKPBq8sb+hALRdvBNLqfc8NWlZq576FkfRZ/MSi4SHvVFT9PQ=="],
"turbo-darwin-arm64": ["turbo-darwin-arm64@2.5.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2+Nx6LAyuXw2MdXb7pxqle3MYignLvS7OwtsP9SgtSBaMlnNlxl9BovzqdYAgkUW3AsYiQMJ/wBRb7d+xemM5A=="],
"turbo-linux-64": ["turbo-linux-64@2.5.4", "", { "os": "linux", "cpu": "x64" }, "sha512-5May2kjWbc8w4XxswGAl74GZ5eM4Gr6IiroqdLhXeXyfvWEdm2mFYCSWOzz0/z5cAgqyGidF1jt1qzUR8hTmOA=="],
"turbo-linux-arm64": ["turbo-linux-arm64@2.5.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-/2yqFaS3TbfxV3P5yG2JUI79P7OUQKOUvAnx4MV9Bdz6jqHsHwc9WZPpO4QseQm+NvmgY6ICORnoVPODxGUiJg=="],
"turbo-windows-64": ["turbo-windows-64@2.5.4", "", { "os": "win32", "cpu": "x64" }, "sha512-EQUO4SmaCDhO6zYohxIjJpOKRN3wlfU7jMAj3CgcyTPvQR/UFLEKAYHqJOnJtymbQmiiM/ihX6c6W6Uq0yC7mA=="],
"turbo-windows-arm64": ["turbo-windows-arm64@2.5.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-oQ8RrK1VS8lrxkLriotFq+PiF7iiGgkZtfLKF4DDKsmdbPo0O9R2mQxm7jHLuXraRCuIQDWMIw6dpcr7Iykf4A=="],
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"uhyphen": ["uhyphen@0.2.0", "", {}, "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="],

5
bunfig.toml Normal file
View File

@@ -0,0 +1,5 @@
[install]
exact = true
[test]
root = "./do-not-run-tests-from-root"

View File

@@ -0,0 +1,477 @@
# opencode Monorepo Config Adoption Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Adopt opencode-style monorepo config: Turbo task orchestration, workspace dep catalog, shared root tsconfig, bunfig.toml, and `exports` field in all packages.
**Architecture:** Pure config changes across 10 files — no source code touched. Root config files are added/updated first, then per-package files updated to reference them. Changes are independent within each task and safe to commit atomically.
**Tech Stack:** Bun workspaces, Turbo 2.x, @tsconfig/bun, TypeScript (tsgo / @typescript/native-preview)
---
## File Map
| File | Action | Responsible for |
|---|---|---|
| `package.json` | Modify | Workspace catalog, turbo devDep, @tsconfig/bun devDep, updated scripts |
| `turbo.json` | Create | Task graph: typecheck, build, test |
| `tsconfig.json` | Create | Shared TS compiler options for all packages |
| `bunfig.toml` | Create | Exact installs, root test guard |
| `packages/core/package.json` | Modify | exports field, catalog refs, script rename |
| `packages/api-server/package.json` | Modify | exports field, catalog refs, script rename |
| `packages/mcp-server/package.json` | Modify | exports field, catalog refs, script rename |
| `packages/core/tsconfig.json` | Modify | Slim — extends root, paths only |
| `packages/api-server/tsconfig.json` | Modify | Slim — extends root, paths only |
| `packages/mcp-server/tsconfig.json` | Modify | Slim — extends root, paths only |
---
### Task 1: Add `bunfig.toml` and `turbo.json`
Two new root config files with no dependencies on other tasks.
**Files:**
- Create: `bunfig.toml`
- Create: `turbo.json`
- [ ] **Step 1: Create `bunfig.toml`**
Write this file at repo root (`/path/to/ca-marketplace-scraper/bunfig.toml`):
```toml
[install]
exact = true
[test]
root = "./do-not-run-tests-from-root"
```
- [ ] **Step 2: Create `turbo.json`**
Write this file at repo root:
```json
{
"$schema": "https://turbo.build/schema.json",
"tasks": {
"typecheck": {},
"build": {
"dependsOn": ["^build"],
"outputs": ["dist/**"]
},
"test": {
"dependsOn": ["^build"],
"outputs": []
}
}
}
```
- [ ] **Step 3: Verify files exist**
Run:
```bash
ls bunfig.toml turbo.json
```
Expected: both files listed, no errors.
- [ ] **Step 4: Commit**
```bash
git add bunfig.toml turbo.json
git commit -m "chore: add bunfig.toml and turbo.json"
```
---
### Task 2: Create root `tsconfig.json`
Shared base tsconfig all packages will extend. Extracts the common options currently duplicated in all 3 per-package tsconfigs.
**Files:**
- Create: `tsconfig.json`
- [ ] **Step 1: Create root `tsconfig.json`**
Write this file at repo root:
```json
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"lib": ["dom", "ESNext"],
"target": "ESNext",
"module": "preserve",
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"moduleDetection": "force",
"jsx": "react-jsx",
"allowJs": true,
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedIndexedAccess": true,
"noImplicitOverride": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false
}
}
```
- [ ] **Step 2: Commit**
```bash
git add tsconfig.json
git commit -m "chore: add shared root tsconfig.json"
```
---
### Task 3: Update root `package.json`
Add workspace catalog, `turbo` + `@tsconfig/bun` devDependencies, and update scripts to use `turbo run`.
**Files:**
- Modify: `package.json`
- [ ] **Step 1: Replace root `package.json`**
Write this complete file:
```json
{
"name": "marketplace-scrapers-monorepo",
"version": "1.0.0",
"private": true,
"type": "module",
"packageManager": "bun@1.3.13",
"scripts": {
"typecheck": "turbo run typecheck",
"build": "bun run clean && turbo run build",
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
"build:all": "bun run build:api && bun run build:mcp",
"ci": "biome ci",
"clean": "rm -rf dist",
"start": "./scripts/start.sh"
},
"workspaces": {
"packages": [
"packages/*"
],
"catalog": {
"@tsconfig/bun": "1.0.9",
"@typescript/native-preview": "7.0.0-dev.20260428.1",
"@types/bun": "1.2.18",
"@types/cli-progress": "3.11.6",
"@types/unidecode": "1.1.0"
}
},
"devDependencies": {
"@biomejs/biome": "2.3.11",
"@tsconfig/bun": "catalog:",
"turbo": "2.5.4"
}
}
```
> **Note on catalog versions:** The catalog pins exact versions. The values above are taken from the current package installs. If `@types/bun` was `latest`, check `node_modules/@types/bun/package.json` for the actual installed version and use that. Same for `@typescript/native-preview`.
- [ ] **Step 2: Check actual installed versions**
Run:
```bash
cat node_modules/@types/bun/package.json | grep '"version"'
cat node_modules/@typescript/native-preview/package.json | grep '"version"'
cat node_modules/@types/cli-progress/package.json | grep '"version"'
cat node_modules/@types/unidecode/package.json | grep '"version"'
```
Update the catalog values in `package.json` to match the exact installed versions.
- [ ] **Step 3: Install turbo and @tsconfig/bun**
```bash
bun install
```
Expected: lock file updated, `turbo` and `@tsconfig/bun` appear in `node_modules`.
- [ ] **Step 4: Verify turbo works**
```bash
bunx turbo run typecheck --dry
```
Expected: output lists the `typecheck` task for each package (even if no `typecheck` script exists yet — turbo will note them as skipped/missing).
- [ ] **Step 5: Commit**
```bash
git add package.json bun.lock
git commit -m "chore: add workspace catalog and turbo to root package.json"
```
---
### Task 4: Update per-package `package.json` files
Rename `type:check``typecheck`, replace `main`/`module` with `exports`, swap pinned dep versions for `catalog:` references.
**Files:**
- Modify: `packages/core/package.json`
- Modify: `packages/api-server/package.json`
- Modify: `packages/mcp-server/package.json`
- [ ] **Step 1: Replace `packages/core/package.json`**
```json
{
"name": "@marketplace-scrapers/core",
"version": "1.0.0",
"type": "module",
"exports": {
".": "./src/index.ts"
},
"private": true,
"scripts": {
"typecheck": "bun tsgo"
},
"dependencies": {
"@typescript/native-preview": "catalog:",
"cli-progress": "^3.12.0",
"linkedom": "^0.18.12",
"unidecode": "^1.1.0"
},
"devDependencies": {
"@types/bun": "catalog:",
"@types/cli-progress": "catalog:",
"@types/unidecode": "catalog:"
},
"peerDependencies": {
"typescript": "^5"
}
}
```
- [ ] **Step 2: Replace `packages/api-server/package.json`**
```json
{
"name": "@marketplace-scrapers/api-server",
"version": "1.0.0",
"type": "module",
"exports": {
".": "./src/index.ts"
},
"private": true,
"scripts": {
"start": "bun ./src/index.ts",
"dev": "bun --watch ./src/index.ts",
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api",
"typecheck": "bun tsgo"
},
"dependencies": {
"@marketplace-scrapers/core": "workspace:*",
"@typescript/native-preview": "catalog:"
},
"devDependencies": {
"@types/bun": "catalog:"
},
"peerDependencies": {
"typescript": "^5"
}
}
```
- [ ] **Step 3: Replace `packages/mcp-server/package.json`**
```json
{
"name": "@marketplace-scrapers/mcp-server",
"version": "1.0.0",
"type": "module",
"exports": {
".": "./src/index.ts"
},
"private": true,
"scripts": {
"start": "bun ./src/index.ts",
"dev": "bun --watch ./src/index.ts",
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp",
"typecheck": "bun tsgo"
},
"dependencies": {
"@marketplace-scrapers/core": "workspace:*",
"@typescript/native-preview": "catalog:"
},
"devDependencies": {
"@types/bun": "catalog:"
},
"peerDependencies": {
"typescript": "^5"
}
}
```
- [ ] **Step 4: Run `bun install` to sync lockfile**
```bash
bun install
```
Expected: no errors. Catalog refs resolved. `bun.lock` updated.
- [ ] **Step 5: Verify typecheck still works per-package**
```bash
cd packages/core && bun run typecheck
cd ../api-server && bun run typecheck
cd ../mcp-server && bun run typecheck
cd ../..
```
Expected: each exits 0 (or same errors as before — no new errors introduced).
- [ ] **Step 6: Commit**
```bash
git add packages/core/package.json packages/api-server/package.json packages/mcp-server/package.json bun.lock
git commit -m "chore: use exports field and catalog refs in all packages"
```
---
### Task 5: Slim per-package `tsconfig.json` files
Replace the duplicated full tsconfig in each package with a slim `extends`-based one pointing to root.
**Files:**
- Modify: `packages/core/tsconfig.json`
- Modify: `packages/api-server/tsconfig.json`
- Modify: `packages/mcp-server/tsconfig.json`
- [ ] **Step 1: Replace `packages/core/tsconfig.json`**
```json
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["./src", "./test"]
}
```
- [ ] **Step 2: Replace `packages/api-server/tsconfig.json`**
```json
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["./src", "./test"]
}
```
- [ ] **Step 3: Replace `packages/mcp-server/tsconfig.json`**
```json
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["./src", "./test"]
}
```
- [ ] **Step 4: Verify `@tsconfig/bun` is resolvable**
The root tsconfig extends `@tsconfig/bun/tsconfig.json`. Confirm the package is installed:
```bash
ls node_modules/@tsconfig/bun/tsconfig.json
```
Expected: file exists.
- [ ] **Step 5: Run typecheck via Turbo**
```bash
bun run typecheck
```
Expected: Turbo runs `typecheck` for all 3 packages in parallel, all pass (or same pre-existing errors — no new ones).
- [ ] **Step 6: Commit**
```bash
git add packages/core/tsconfig.json packages/api-server/tsconfig.json packages/mcp-server/tsconfig.json
git commit -m "chore: slim per-package tsconfigs to extend root"
```
---
### Task 6: Smoke test full build pipeline
Verify everything works end-to-end.
**Files:** none (verification only)
- [ ] **Step 1: Run turbo typecheck**
```bash
bun run typecheck
```
Expected: Turbo runs `typecheck` across all packages. Exit 0.
- [ ] **Step 2: Run full build**
```bash
bun run build
```
Expected: `dist/` cleaned, Turbo runs `build` (core first, then api-server and mcp-server in parallel), build artifacts appear in `dist/api/` and `dist/mcp/`.
- [ ] **Step 3: Verify dist artifacts**
```bash
ls dist/api/ dist/mcp/
```
Expected: compiled output files in both directories.
- [ ] **Step 4: Verify `bun install` is exact**
```bash
grep -c '\^' bun.lock | head -5
```
With `exact = true` in bunfig.toml, new installs won't add `^` ranges. Existing `^` ranges in `bun.lock` from before are fine — they'll be resolved to exact on next fresh install.
- [ ] **Step 5: Final commit if any loose files**
```bash
git status
```
If clean: done. If any files modified by `bun install` (e.g. `bun.lock`):
```bash
git add bun.lock
git commit -m "chore: sync lockfile after monorepo config adoption"
```

View File

@@ -0,0 +1,136 @@
# Design: Adopt opencode Monorepo Config
**Date:** 2025-07-14
**Status:** Approved
**Approach:** Full adoption (A)
## Context
Current repo (`marketplace-scrapers-monorepo`) has basic bun workspaces with 3 packages (`core`, `api-server`, `mcp-server`). Reference: `anomalyco/opencode` monorepo patterns.
**Gaps vs opencode:**
- No Turbo (task orchestration, caching, dep graph)
- No workspace catalog (shared dep versions duplicated across packages)
- No root tsconfig (identical tsconfigs duplicated in all 3 packages)
- No `bunfig.toml` (no exact installs, no root test guard)
- `main`/`module` fields instead of `exports` field
## Changes
### 1. Root `package.json`
- Add `workspaces.catalog` block with shared deps:
- `@typescript/native-preview`, `@types/bun`, `@types/unidecode`, `@types/cli-progress`
- Add `turbo` to `devDependencies`
- Add `@tsconfig/bun` to `devDependencies` + catalog
- Update root scripts: `typecheck` and `build` delegate to `turbo run`
- Keep `build:api`, `build:mcp`, `build:all`, `start` as-is (deployment-specific)
- Rename `type:check``typecheck` in all packages (Turbo convention)
### 2. `turbo.json` (new file)
Tasks:
```json
{
"tasks": {
"typecheck": {},
"build": { "dependsOn": ["^build"], "outputs": ["dist/**"] },
"test": { "dependsOn": ["^build"], "outputs": [] }
}
}
```
`core` builds before `api-server`/`mcp-server` due to `^build` dep.
### 3. Root `tsconfig.json` (new file)
```json
{
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"lib": ["dom", "ESNext"],
"target": "ESNext",
"module": "preserve",
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"moduleDetection": "force",
"jsx": "react-jsx",
"allowJs": true,
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedIndexedAccess": true,
"noImplicitOverride": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false
}
}
```
### 4. Per-package `tsconfig.json` (slim)
All 3 packages slim to:
```json
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"paths": { "@/*": ["./src/*"] }
},
"include": ["./src", "./test"]
}
```
### 5. `bunfig.toml` (new file)
```toml
[install]
exact = true
[test]
root = "./do-not-run-tests-from-root"
```
Exact installs = reproducible. Root test guard prevents accidental root-level test runs.
### 6. Package `exports` field
Replace `main`/`module` with `exports` in all 3 packages:
```json
"exports": { ".": "./src/index.ts" }
```
Remove `main` and `module` fields. Bun resolves `.ts` directly.
### 7. Catalog references in per-package `package.json`
Replace pinned versions with `"catalog:"` for shared deps:
- `@typescript/native-preview: "catalog:"`
- `@types/bun: "catalog:"`
- `@types/unidecode: "catalog:"` (core only)
- `@types/cli-progress: "catalog:"` (core only)
## Files Changed
| File | Action |
|---|---|
| `package.json` | Update (catalog, turbo dep, scripts) |
| `turbo.json` | Create |
| `tsconfig.json` | Create |
| `bunfig.toml` | Create |
| `packages/core/package.json` | Update (exports, catalog refs, script rename) |
| `packages/api-server/package.json` | Update (exports, catalog refs, script rename) |
| `packages/mcp-server/package.json` | Update (exports, catalog refs, script rename) |
| `packages/core/tsconfig.json` | Update (slim, extends root) |
| `packages/api-server/tsconfig.json` | Update (slim, extends root) |
| `packages/mcp-server/tsconfig.json` | Update (slim, extends root) |
## Non-Goals
- No Husky/git hooks (not needed yet)
- No SST/cloud infra (not applicable)
- No prettier (keep biome as formatter)
- No patches mechanism
- No `postinstall` scripts

View File

@@ -1,21 +1,35 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "marketplace-scrapers-monorepo",
"version": "1.0.0",
"private": true,
"type": "module",
"packageManager": "bun@1.3.13",
"scripts": {
"ci": "biome ci",
"clean": "rm -rf dist",
"typecheck": "turbo run typecheck",
"build": "bun run clean && turbo run build",
"build:api": "bun build ./packages/api-server/src/index.ts --target=bun --outdir=./dist/api --minify",
"build:mcp": "bun build ./packages/mcp-server/src/index.ts --target=bun --outdir=./dist/mcp --minify",
"build:all": "bun run build:api && bun run build:mcp",
"build": "bun run clean && bun run build:all",
"ci": "biome ci",
"clean": "rm -rf dist",
"start": "./scripts/start.sh"
},
"private": true,
"type": "module",
"workspaces": [
"workspaces": {
"packages": [
"packages/*"
],
"catalog": {
"@tsconfig/bun": "1.0.9",
"@typescript/native-preview": "7.0.0-dev.20260428.1",
"@types/bun": "1.3.13",
"@types/cli-progress": "3.11.6",
"@types/unidecode": "1.1.0"
}
},
"devDependencies": {
"@biomejs/biome": "2.3.11"
"@biomejs/biome": "2.3.11",
"@tsconfig/bun": "catalog:",
"turbo": "2.5.4"
}
}

View File

@@ -2,18 +2,22 @@
"name": "@marketplace-scrapers/api-server",
"version": "1.0.0",
"type": "module",
"module": "./src/index.ts",
"exports": {
".": "./src/index.ts"
},
"private": true,
"scripts": {
"start": "bun ./src/index.ts",
"dev": "bun --watch ./src/index.ts",
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api"
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/api",
"typecheck": "bun tsgo"
},
"dependencies": {
"@marketplace-scrapers/core": "workspace:*"
"@marketplace-scrapers/core": "workspace:*",
"@typescript/native-preview": "catalog:"
},
"devDependencies": {
"@types/bun": "latest"
"@types/bun": "catalog:"
},
"peerDependencies": {
"typescript": "^5"

View File

@@ -48,16 +48,16 @@ export async function kijijiRoute(req: Request): Promise<Response> {
location: reqUrl.searchParams.get("location") || undefined,
category: reqUrl.searchParams.get("category") || undefined,
keywords: reqUrl.searchParams.get("keywords") || undefined,
sortBy: (reqUrl.searchParams.get("sortBy") as
sortBy:
(reqUrl.searchParams.get("sortBy") as
| "relevancy"
| "date"
| "price"
| "distance"
| undefined) || undefined,
sortOrder: (reqUrl.searchParams.get("sortOrder") as
| "desc"
| "asc"
| undefined) || undefined,
sortOrder:
(reqUrl.searchParams.get("sortOrder") as "desc" | "asc" | undefined) ||
undefined,
maxPages,
priceMin,
priceMax,

View File

@@ -1,4 +1,4 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import { beforeEach, describe, expect, mock, test } from "bun:test";
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
@@ -123,7 +123,10 @@ describe("API routes", () => {
),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
{
minPrice: undefined,
maxPrice: undefined,
strictMode: false,
@@ -131,9 +134,11 @@ describe("API routes", () => {
keywords: ["laptop"],
buyItNowOnly: true,
canadaOnly: true,
}, {
},
{
hideUnstableResults: true,
});
},
);
});
test("kijijiRoute forwards unstableFilter=true to core", async () => {
@@ -202,9 +207,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&buyItNowOnly=true",
),
new Request("http://localhost/api/ebay?q=laptop&buyItNowOnly=true"),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
@@ -242,9 +245,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=5",
),
new Request("http://localhost/api/kijiji?q=laptop&maxPages=5"),
);
expect(fetchKijijiItems).toHaveBeenCalledWith(
@@ -385,17 +386,17 @@ describe("API routes", () => {
test("ebayRoute forwards maxItems to core in default mode", async () => {
const { ebayRoute } = await import("../src/routes/ebay");
fetchEbayItems.mockImplementation(() =>
Promise.resolve([{ title: "a" }]),
);
fetchEbayItems.mockImplementation(() => Promise.resolve([{ title: "a" }]));
await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxItems=2",
),
new Request("http://localhost/api/ebay?q=laptop&maxItems=2"),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }));
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
expect.objectContaining({ maxItems: 2 }),
);
});
test("ebayRoute passes through scraper payload unchanged in unstable mode", async () => {
@@ -419,9 +420,14 @@ describe("API routes", () => {
expect(body.unstableResults).toHaveLength(2);
expect(body.results[0].title).toBe("a");
expect(body.unstableResults[0].title).toBe("d");
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 4 }), {
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
expect.objectContaining({ maxItems: 4 }),
{
hideUnstableResults: true,
});
},
);
});
test("ebayRoute forwards maxItems to core in unstable mode", async () => {
@@ -440,9 +446,14 @@ describe("API routes", () => {
),
);
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, expect.objectContaining({ maxItems: 2 }), {
expect(fetchEbayItems).toHaveBeenCalledWith(
"laptop",
1,
expect.objectContaining({ maxItems: 2 }),
{
hideUnstableResults: true,
});
},
);
});
test("ebayRoute returns 404 when unstable results are empty", async () => {
@@ -456,9 +467,7 @@ describe("API routes", () => {
);
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&unstableFilter=true",
),
new Request("http://localhost/api/ebay?q=laptop&unstableFilter=true"),
);
expect(response.status).toBe(404);
@@ -470,9 +479,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxItems=abc",
),
new Request("http://localhost/api/ebay?q=laptop&maxItems=abc"),
);
expect(response.status).toBe(400);
@@ -484,9 +491,7 @@ describe("API routes", () => {
const { facebookRoute } = await import("../src/routes/facebook");
const response = await facebookRoute(
new Request(
"http://localhost/api/facebook?q=laptop&maxItems=abc",
),
new Request("http://localhost/api/facebook?q=laptop&maxItems=abc"),
);
expect(response.status).toBe(400);
@@ -498,9 +503,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&minPrice=abc",
),
new Request("http://localhost/api/ebay?q=laptop&minPrice=abc"),
);
expect(response.status).toBe(400);
@@ -512,9 +515,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxPrice=abc",
),
new Request("http://localhost/api/ebay?q=laptop&maxPrice=abc"),
);
expect(response.status).toBe(400);
@@ -526,9 +527,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=abc",
),
new Request("http://localhost/api/kijiji?q=laptop&maxPages=abc"),
);
expect(response.status).toBe(400);
@@ -540,9 +539,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMin=abc",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMin=abc"),
);
expect(response.status).toBe(400);
@@ -554,9 +551,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMax=abc",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMax=abc"),
);
expect(response.status).toBe(400);
@@ -568,9 +563,7 @@ describe("API routes", () => {
const { facebookRoute } = await import("../src/routes/facebook");
const response = await facebookRoute(
new Request(
"http://localhost/api/facebook?q=laptop&maxItems=-1",
),
new Request("http://localhost/api/facebook?q=laptop&maxItems=-1"),
);
expect(response.status).toBe(400);
@@ -582,9 +575,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxItems=-1",
),
new Request("http://localhost/api/ebay?q=laptop&maxItems=-1"),
);
expect(response.status).toBe(400);
@@ -596,9 +587,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&minPrice=-5",
),
new Request("http://localhost/api/ebay?q=laptop&minPrice=-5"),
);
expect(response.status).toBe(400);
@@ -610,9 +599,7 @@ describe("API routes", () => {
const { ebayRoute } = await import("../src/routes/ebay");
const response = await ebayRoute(
new Request(
"http://localhost/api/ebay?q=laptop&maxPrice=-10",
),
new Request("http://localhost/api/ebay?q=laptop&maxPrice=-10"),
);
expect(response.status).toBe(400);
@@ -624,9 +611,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&maxPages=-2",
),
new Request("http://localhost/api/kijiji?q=laptop&maxPages=-2"),
);
expect(response.status).toBe(400);
@@ -638,9 +623,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMin=-5",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMin=-5"),
);
expect(response.status).toBe(400);
@@ -652,9 +635,7 @@ describe("API routes", () => {
const { kijijiRoute } = await import("../src/routes/kijiji");
const response = await kijijiRoute(
new Request(
"http://localhost/api/kijiji?q=laptop&priceMax=-10",
),
new Request("http://localhost/api/kijiji?q=laptop&priceMax=-10"),
);
expect(response.status).toBe(400);

View File

@@ -1,13 +1,9 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"paths": {
"@/*": ["./src/*"]
}
},
"strict": true,
"noEmit": true
}
"include": ["./src", "./test"]
}

View File

@@ -2,18 +2,23 @@
"name": "@marketplace-scrapers/core",
"version": "1.0.0",
"type": "module",
"main": "./src/index.ts",
"module": "./src/index.ts",
"exports": {
".": "./src/index.ts"
},
"private": true,
"scripts": {
"typecheck": "bun tsgo"
},
"dependencies": {
"@typescript/native-preview": "catalog:",
"cli-progress": "^3.12.0",
"linkedom": "^0.18.12",
"unidecode": "^1.1.0"
},
"devDependencies": {
"@types/bun": "latest",
"@types/unidecode": "^1.1.0",
"@types/cli-progress": "^3.11.6"
"@types/bun": "catalog:",
"@types/cli-progress": "catalog:",
"@types/unidecode": "catalog:"
},
"peerDependencies": {
"typescript": "^5"

View File

@@ -4,13 +4,13 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type CookieConfig,
ensureCookies,
formatCookiesForHeader,
} from "../utils/cookies";
import { delay } from "../utils/delay";
import { classifyUnstableListings } from "../utils/unstable";
// eBay cookie configuration
const EBAY_COOKIE_CONFIG: CookieConfig = {
@@ -44,7 +44,9 @@ function canonicalizeEbayItemUrl(url: string): string {
try {
const parsed = new URL(url, "https://www.ebay.ca");
const match = parsed.pathname.match(/\/itm\/(?:[^/?#]+\/)?\d+/);
return match ? `${parsed.origin}${match[0]}` : `${parsed.origin}${parsed.pathname}`;
return match
? `${parsed.origin}${match[0]}`
: `${parsed.origin}${parsed.pathname}`;
} catch {
return url;
}
@@ -275,11 +277,7 @@ function parseEbayListings(
const actualPrices: HTMLElement[] = [];
for (const el of allPriceElements) {
const text = el.textContent?.trim();
if (
text &&
EBAY_PRICE_TEXT_RE.test(text) &&
text.length < 50
) {
if (text && EBAY_PRICE_TEXT_RE.test(text) && text.length < 50) {
actualPrices.push(el);
}
}
@@ -386,7 +384,8 @@ async function loadEbayCookies(): Promise<string | undefined> {
export default async function fetchEbayItems(
SEARCH_QUERY: string,
REQUESTS_PER_SECOND: number | undefined,
opts: {
opts:
| {
minPrice?: number;
maxPrice?: number;
strictMode?: boolean;
@@ -395,7 +394,8 @@ export default async function fetchEbayItems(
buyItNowOnly?: boolean;
canadaOnly?: boolean;
maxItems?: number;
} | undefined,
}
| undefined,
unstableMode: { hideUnstableResults: true },
): Promise<UnstableListingBuckets<EbayListingDetails>>;
export default async function fetchEbayItems(
@@ -529,7 +529,9 @@ export default async function fetchEbayItems(
// Filter by price range (additional safety check)
const filteredListings = listings.filter((listing) => {
const cents = listing.listingPrice?.cents;
return typeof cents === "number" && cents >= minPrice && cents <= maxPrice;
return (
typeof cents === "number" && cents >= minPrice && cents <= maxPrice
);
});
console.log(`Parsed ${filteredListings.length} eBay listings.`);

View File

@@ -5,7 +5,6 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type Cookie,
type CookieConfig,
@@ -16,6 +15,7 @@ import {
import { delay } from "../utils/delay";
import { formatCentsToCurrency } from "../utils/format";
import { isRecord } from "../utils/http";
import { classifyUnstableListings } from "../utils/unstable";
/**
* Facebook Marketplace Scraper
@@ -408,7 +408,11 @@ export function classifyFacebookResponse(
htmlString.includes("This listing is no longer available") ||
htmlString.includes("listing has been removed");
if (unavailable) {
return { kind: "unavailable" as const, authGated: false, unavailable: true };
return {
kind: "unavailable" as const,
authGated: false,
unavailable: true,
};
}
if (responseUrl.includes("/marketplace/item/")) {
@@ -455,7 +459,8 @@ function isFacebookSearchEdgeArray(value: unknown): value is FacebookEdge[] {
Array.isArray(value) &&
value.length > 0 &&
value.every(
(edge) => isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
(edge) =>
isRecord(edge) && isRecord(edge.node) && isRecord(edge.node.listing),
)
);
}
@@ -552,8 +557,7 @@ function scoreMarketplaceItemPath(path: string[]): number {
if (
path.some(
(segment) =>
segment.includes("recommend") || segment.includes("related"),
(segment) => segment.includes("recommend") || segment.includes("related"),
)
) {
score -= 10;
@@ -567,7 +571,9 @@ function collectMarketplaceItemCandidates(
path: string[] = [],
): FacebookMarketplaceItemMatch[] {
if (Array.isArray(candidate)) {
return candidate.flatMap((item) => collectMarketplaceItemCandidates(item, path));
return candidate.flatMap((item) =>
collectMarketplaceItemCandidates(item, path),
);
}
if (!isRecord(candidate)) {
@@ -628,7 +634,9 @@ function extractRenderedText(node: ParentNode, selector: string): string[] {
.filter((text): text is string => Boolean(text));
}
function extractMarketplaceItemIdFromElement(element: Element | null): string | null {
function extractMarketplaceItemIdFromElement(
element: Element | null,
): string | null {
const href = element?.getAttribute("href") || "";
return href.match(FACEBOOK_ITEM_HREF_RE)?.[1] ?? null;
}
@@ -666,7 +674,9 @@ function extractFacebookPermalinkItemId(document: Document): string | null {
return extractMarketplaceItemIdFromElement(itemLinks.at(-1) ?? null);
}
function extractFacebookDescriptionText(document: Document): string | undefined {
function extractFacebookDescriptionText(
document: Document,
): string | undefined {
const labels = Array.from(document.querySelectorAll("div, span, h2, h3, p"));
for (const label of labels) {
@@ -759,7 +769,10 @@ function extractFacebookItemHtmlFallback(
const priceText = texts.find((text) => FACEBOOK_PRICE_TEXT_RE.test(text));
const parsedPrice = priceText ? parseFacebookRenderedPrice(priceText) : null;
const location = texts.find(
(text) => text !== title && text !== priceText && FACEBOOK_LOCATION_TEXT_RE.test(text),
(text) =>
text !== title &&
text !== priceText &&
FACEBOOK_LOCATION_TEXT_RE.test(text),
);
const description = extractFacebookDescriptionText(document);
@@ -841,7 +854,8 @@ export function extractFacebookItemData(
if (
!bestMatch ||
match.score > bestMatch.score ||
(match.score === bestMatch.score && match.path.length < bestMatch.path.length)
(match.score === bestMatch.score &&
match.path.length < bestMatch.path.length)
) {
bestMatch = match;
}
@@ -1101,7 +1115,9 @@ export default async function fetchFacebookItems(
const finalizeResults = (
listings: FacebookListingDetails[],
): FacebookListingDetails[] | UnstableListingBuckets<FacebookListingDetails> => {
):
| FacebookListingDetails[]
| UnstableListingBuckets<FacebookListingDetails> => {
if (!unstableMode.hideUnstableResults) {
return listings.slice(0, MAX_ITEMS);
}
@@ -1166,9 +1182,14 @@ export default async function fetchFacebookItems(
throw err;
}
const classification = classifyFacebookResponse(searchHtml, searchResponseUrl);
const classification = classifyFacebookResponse(
searchHtml,
searchResponseUrl,
);
if (classification.authGated) {
console.warn("Facebook marketplace search redirected to login. Cookies may be expired.");
console.warn(
"Facebook marketplace search redirected to login. Cookies may be expired.",
);
return finalizeResults([]);
}
@@ -1204,7 +1225,8 @@ export default async function fetchFacebookItems(
// Filter to only priced items (already done in parseFacebookAds)
const pricedItems = items.filter(
(item) =>
typeof item.listingPrice?.cents === "number" && item.listingPrice.cents >= 0,
typeof item.listingPrice?.cents === "number" &&
item.listingPrice.cents >= 0,
);
progressBar?.update(totalProgress);
@@ -1293,7 +1315,9 @@ export async function fetchFacebookItem(
if (classification.authGated) {
logExtractionMetrics(false, itemId);
console.warn(`Authentication failed for item ${itemId}. Cookies may be expired.`);
console.warn(
`Authentication failed for item ${itemId}. Cookies may be expired.`,
);
return null;
}
@@ -1301,7 +1325,9 @@ export async function fetchFacebookItem(
if (classification.unavailable && !itemData) {
logExtractionMetrics(false, itemId);
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}
@@ -1317,7 +1343,9 @@ export async function fetchFacebookItem(
logExtractionMetrics(false, itemId);
if (itemHtml.includes("This item has been sold")) {
console.warn(`Item ${itemId} appears to be sold or removed from marketplace.`);
console.warn(
`Item ${itemId} appears to be sold or removed from marketplace.`,
);
return null;
}

View File

@@ -6,7 +6,6 @@ import type {
UnstableListingBuckets,
UnstableListingModeOptions,
} from "../types/common";
import { classifyUnstableListings } from "../utils/unstable";
import {
type CookieConfig,
formatCookiesForHeader,
@@ -22,6 +21,7 @@ import {
RateLimitError,
ValidationError,
} from "../utils/http";
import { classifyUnstableListings } from "../utils/unstable";
// Kijiji cookie configuration
const KIJIJI_COOKIE_CONFIG: CookieConfig = {
@@ -203,11 +203,17 @@ const SORT_MAPPINGS: Record<string, string> = {
};
const LOCATION_SLUGS = Object.fromEntries(
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
Object.entries(LOCATION_MAPPINGS).map(([slug, id]) => [
id,
slug.replace(/\s+/g, "-"),
]),
) as Record<number, string>;
const CATEGORY_SLUGS = Object.fromEntries(
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [id, slug.replace(/\s+/g, "-")]),
Object.entries(CATEGORY_MAPPINGS).map(([slug, id]) => [
id,
slug.replace(/\s+/g, "-"),
]),
) as Record<number, string>;
// ----------------------------- Utilities -----------------------------
@@ -816,7 +822,10 @@ export default async function fetchKijijiItems(
: undefined;
// Set defaults for configuration
const finalSearchOptions: Omit<Required<SearchOptions>, "priceMin" | "priceMax"> & {
const finalSearchOptions: Omit<
Required<SearchOptions>,
"priceMin" | "priceMax"
> & {
priceMin?: number;
priceMax?: number;
} = {
@@ -903,7 +912,9 @@ export default async function fetchKijijiItems(
const batchPromises = batch.map(async (link, batchIndex) => {
try {
if (batchIndex > 0) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS * batchIndex));
await new Promise((resolve) =>
setTimeout(resolve, DELAY_MS * batchIndex),
);
}
const html = await fetchHtml(link, 0, {
@@ -949,7 +960,6 @@ export default async function fetchKijijiItems(
if (i + CONCURRENT_REQUESTS < newListingLinks.length) {
await new Promise((resolve) => setTimeout(resolve, DELAY_MS));
}
}
allListings.push(
@@ -968,9 +978,7 @@ export default async function fetchKijijiItems(
matchesPriceFilters(listing, finalSearchOptions),
);
console.log(
`\nParsed ${filteredListings.length} detailed listings.`,
);
console.log(`\nParsed ${filteredListings.length} detailed listings.`);
return finalizeResults(filteredListings);
}

View File

@@ -136,7 +136,9 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toHaveLength(1);
expect(results[0]).toEqual(
expect.objectContaining({ url: "https://www.ebay.ca/itm/123?_trkparms=foo" }),
expect.objectContaining({
url: "https://www.ebay.ca/itm/123?_trkparms=foo",
}),
);
});
@@ -229,7 +231,10 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "USD",
cents: 12345,
}),
}),
]);
});
@@ -255,7 +260,10 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "USD", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "USD",
cents: 12345,
}),
}),
]);
});
@@ -281,7 +289,10 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "GBP", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "GBP",
cents: 12345,
}),
}),
]);
});
@@ -314,10 +325,16 @@ describe("eBay Scraper Cookie Handling", () => {
expect(results).toEqual([
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "EUR", cents: 12345 }),
listingPrice: expect.objectContaining({
currency: "EUR",
cents: 12345,
}),
}),
expect.objectContaining({
listingPrice: expect.objectContaining({ currency: "JPY", cents: 12300 }),
listingPrice: expect.objectContaining({
currency: "JPY",
cents: 12300,
}),
}),
]);
});

View File

@@ -2,13 +2,13 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import cliProgress from "cli-progress";
import {
classifyFacebookResponse,
type FacebookListingDetails,
ensureFacebookCookies,
extractFacebookBootstrapCandidates,
extractFacebookItemData,
extractFacebookMarketplaceData,
default as fetchFacebookItems,
type FacebookListingDetails,
fetchFacebookItem,
default as fetchFacebookItems,
parseFacebookAds,
parseFacebookCookieString,
parseFacebookItem,
@@ -30,9 +30,13 @@ type IsExact<T, U> =
const getDefaultFacebookItems = async () => fetchFacebookItems("chair");
const getUnstableFacebookItems = async (): Promise<
UnstableListingBuckets<FacebookListingDetails>
> => fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
> =>
fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true });
type _FacebookDefaultReturn = Assert<
IsExact<Awaited<ReturnType<typeof getDefaultFacebookItems>>, FacebookListingDetails[]>
IsExact<
Awaited<ReturnType<typeof getDefaultFacebookItems>>,
FacebookListingDetails[]
>
>;
type _FacebookUnstableReturn = Assert<
IsExact<
@@ -533,7 +537,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("returns an array by default", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -556,7 +561,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -576,7 +582,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("preserves free listings through the public fetch entrypoint", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -599,7 +606,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -626,7 +634,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("does not start a progress bar when stdout is not a TTY", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -649,7 +658,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
process.stdout.isTTY = false;
const startSpy = mock(() => {});
@@ -688,7 +698,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("returns results and unstableResults when unstable mode is enabled", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -739,7 +750,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -768,7 +780,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
});
test("unstable mode classifies before the final MAX_ITEMS limit", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -791,7 +804,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
node: {
listing: {
id: "2",
marketplace_listing_title: "Second Boundary Stable Chair",
marketplace_listing_title:
"Second Boundary Stable Chair",
listing_price: {
amount: "110.00",
formatted_amount: "CA$110",
@@ -819,7 +833,8 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({
@@ -869,7 +884,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
},
redacted_description: { text: "Solid wood chair" },
location_text: { text: "Toronto, ON" },
marketplace_listing_seller: { id: "seller-1", name: "Alex" },
marketplace_listing_seller: {
id: "seller-1",
name: "Alex",
},
condition: "USED",
is_live: true,
},

View File

@@ -1,5 +1,7 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import fetchFacebookItems, { fetchFacebookItem } from "../src/scrapers/facebook";
import fetchFacebookItems, {
fetchFacebookItem,
} from "../src/scrapers/facebook";
// Mock fetch globally
const originalFetch = global.fetch;
@@ -27,7 +29,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
describe("Main Search Function", () => {
test("should successfully fetch search results", async () => {
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify(
{
payload: {
resultGroups: [
{
@@ -55,7 +58,8 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
},
],
},
})}</script></body></html>`;
},
)}</script></body></html>`;
global.fetch = mock(() =>
Promise.resolve({

View File

@@ -1,12 +1,12 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import {
buildSearchUrl,
default as fetchKijijiItems,
type DetailedListing,
default as fetchKijijiItems,
NetworkError,
parseSearch,
parseDetailedListing,
ParseError,
parseDetailedListing,
parseSearch,
RateLimitError,
resolveCategoryId,
resolveLocationId,
@@ -282,7 +282,8 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-low/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
text: () =>
Promise.resolve(listingHtml("Low Listing", 7000, "v-low/k0l0")),
headers: { get: () => null },
url,
});
@@ -291,7 +292,8 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-mid/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
text: () =>
Promise.resolve(listingHtml("Mid Listing", 9000, "v-mid/k0l0")),
headers: { get: () => null },
url,
});
@@ -300,7 +302,8 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-high/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
text: () =>
Promise.resolve(listingHtml("High Listing", 12000, "v-high/k0l0")),
headers: { get: () => null },
url,
});
@@ -534,9 +537,18 @@ describe("fetchKijijiItems", () => {
props: {
pageProps: {
__APOLLO_STATE__: {
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
"Listing:3": { url: "/v-unstable/k0l0", title: "Unstable Listing" },
"Listing:1": {
url: "/v-stable-one/k0l0",
title: "Stable Listing One",
},
"Listing:2": {
url: "/v-stable-two/k0l0",
title: "Stable Listing Two",
},
"Listing:3": {
url: "/v-unstable/k0l0",
title: "Unstable Listing",
},
},
},
},
@@ -582,7 +594,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-one/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -591,7 +606,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-two/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -600,7 +618,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-unstable/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Unstable Listing", 7000, "v-unstable/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Unstable Listing", 7000, "v-unstable/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -635,10 +656,22 @@ describe("fetchKijijiItems", () => {
props: {
pageProps: {
__APOLLO_STATE__: {
"Listing:1": { url: "/v-stable-one/k0l0", title: "Stable Listing One" },
"Listing:2": { url: "/v-stable-two/k0l0", title: "Stable Listing Two" },
"Listing:3": { url: "/v-out-of-range-high/k0l0", title: "Out Of Range High" },
"Listing:4": { url: "/v-out-of-range-low/k0l0", title: "Out Of Range Low" },
"Listing:1": {
url: "/v-stable-one/k0l0",
title: "Stable Listing One",
},
"Listing:2": {
url: "/v-stable-two/k0l0",
title: "Stable Listing Two",
},
"Listing:3": {
url: "/v-out-of-range-high/k0l0",
title: "Out Of Range High",
},
"Listing:4": {
url: "/v-out-of-range-low/k0l0",
title: "Out Of Range Low",
},
},
},
},
@@ -672,7 +705,11 @@ describe("fetchKijijiItems", () => {
global.fetch = mock((input: string | URL | Request) => {
const url = typeof input === "string" ? input : input.toString();
if (url.includes("/k0c0l1700272") && url.includes("priceMin=80") && url.includes("priceMax=150")) {
if (
url.includes("/k0c0l1700272") &&
url.includes("priceMin=80") &&
url.includes("priceMax=150")
) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(searchHtml),
@@ -684,7 +721,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-one/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing One", 10000, "v-stable-one/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -693,7 +733,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-stable-two/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Stable Listing Two", 11000, "v-stable-two/k0l0"),
),
headers: { get: () => null },
url,
});
@@ -702,7 +745,14 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-out-of-range-high/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Out Of Range High", 20000, "v-out-of-range-high/k0l0")),
text: () =>
Promise.resolve(
listingHtml(
"Out Of Range High",
20000,
"v-out-of-range-high/k0l0",
),
),
headers: { get: () => null },
url,
});
@@ -711,7 +761,10 @@ describe("fetchKijijiItems", () => {
if (url.endsWith("/v-out-of-range-low/k0l0")) {
return Promise.resolve({
ok: true,
text: () => Promise.resolve(listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0")),
text: () =>
Promise.resolve(
listingHtml("Out Of Range Low", 7000, "v-out-of-range-low/k0l0"),
),
headers: { get: () => null },
url,
});

View File

@@ -31,8 +31,13 @@ describe("classifyUnstableListings", () => {
const buckets = classifyUnstableListings(listings);
expect(buckets.results.map((listing) => listing.id)).toEqual(["stable-1", "stable-2"]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
expect(buckets.results.map((listing) => listing.id)).toEqual([
"stable-1",
"stable-2",
]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
"unstable",
]);
});
test("uses the midpoint median for even-sized priced inputs", () => {
@@ -45,8 +50,14 @@ describe("classifyUnstableListings", () => {
const buckets = classifyUnstableListings(listings);
expect(buckets.results.map((listing) => listing.id)).toEqual(["mid-low", "mid-high", "high"]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["low"]);
expect(buckets.results.map((listing) => listing.id)).toEqual([
"mid-low",
"mid-high",
"high",
]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
"low",
]);
});
test("keeps non-positive prices in results and excludes them from the median input", () => {
@@ -66,7 +77,9 @@ describe("classifyUnstableListings", () => {
"stable-1",
"stable-2",
]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual(["unstable"]);
expect(buckets.unstableResults.map((listing) => listing.id)).toEqual([
"unstable",
]);
});
test("returns all listings in results when fewer than two valid prices are present", () => {
@@ -78,7 +91,11 @@ describe("classifyUnstableListings", () => {
const buckets = classifyUnstableListings(listings);
expect(buckets.results.map((listing) => listing.id)).toEqual(["zero", "negative", "only-valid"]);
expect(buckets.results.map((listing) => listing.id)).toEqual([
"zero",
"negative",
"only-valid",
]);
expect(buckets.unstableResults).toEqual([]);
});
});

View File

@@ -1,13 +1,9 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"paths": {
"@/*": ["./src/*"]
}
},
"strict": true,
"noEmit": true
}
"include": ["./src", "./test"]
}

View File

@@ -2,18 +2,22 @@
"name": "@marketplace-scrapers/mcp-server",
"version": "1.0.0",
"type": "module",
"module": "./src/index.ts",
"exports": {
".": "./src/index.ts"
},
"private": true,
"scripts": {
"start": "bun ./src/index.ts",
"dev": "bun --watch ./src/index.ts",
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp"
"build": "bun build ./src/index.ts --target=bun --outdir=../../dist/mcp",
"typecheck": "bun tsgo"
},
"dependencies": {
"@marketplace-scrapers/core": "workspace:*"
"@marketplace-scrapers/core": "workspace:*",
"@typescript/native-preview": "catalog:"
},
"devDependencies": {
"@types/bun": "latest"
"@types/bun": "catalog:"
},
"peerDependencies": {
"typescript": "^5"

View File

@@ -72,7 +72,10 @@ describe("MCP protocol unstableFilter", () => {
const tool = tools.find((t) => t.name === toolName);
expect(tool).toBeDefined();
expect(tool?.inputSchema.properties).toHaveProperty("unstableFilter");
const prop = tool?.inputSchema.properties.unstableFilter as any;
const prop = tool?.inputSchema.properties.unstableFilter as {
type: string;
description: string;
};
expect(prop.type).toBe("boolean");
expect(prop.description).toContain("optional");
expect(prop.description).toContain("20%");

View File

@@ -1,13 +1,9 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"lib": ["dom"],
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "bundler",
"paths": {
"@/*": ["./src/*"]
}
},
"strict": true,
"noEmit": true
}
"include": ["./src", "./test"]
}

24
tsconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"lib": ["dom", "ESNext"],
"target": "ESNext",
"module": "preserve",
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"moduleDetection": "force",
"jsx": "react-jsx",
"allowJs": true,
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedIndexedAccess": true,
"noImplicitOverride": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false
}
}

14
turbo.json Normal file
View File

@@ -0,0 +1,14 @@
{
"$schema": "https://turbo.build/schema.json",
"tasks": {
"typecheck": {},
"build": {
"dependsOn": ["^build"],
"outputs": ["../../dist/**"]
},
"test": {
"dependsOn": ["^build"],
"outputs": []
}
}
}