From 5d99e984e0ac08d4f05744387d941fefbe708ef7 Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Thu, 30 Apr 2026 22:41:41 -0400 Subject: [PATCH] docs: plan live parser tests --- .../plans/2026-04-30-live-parser-tests.md | 166 ++++++++++++++++++ .../2026-04-30-live-parser-tests-design.md | 37 ++++ 2 files changed, 203 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-30-live-parser-tests.md create mode 100644 docs/superpowers/specs/2026-04-30-live-parser-tests-design.md diff --git a/docs/superpowers/plans/2026-04-30-live-parser-tests.md b/docs/superpowers/plans/2026-04-30-live-parser-tests.md new file mode 100644 index 0000000..8ad117d --- /dev/null +++ b/docs/superpowers/plans/2026-04-30-live-parser-tests.md @@ -0,0 +1,166 @@ +# Live Parser Tests Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add explicit live endpoint test suites for each core marketplace scraper, excluded from default tests and runnable through one script. + +**Architecture:** Live tests live under `packages/core/test/live/` and import public scraper entry points directly. Normal package tests remain offline because the new files are outside current explicit test commands and run only through `bun run test:live`. + +**Tech Stack:** Bun `1.3.13`, `bun:test`, TypeScript, existing core scraper APIs. + +--- + +## File Structure + +- Create `packages/core/test/live/ebay.live.test.ts`: live eBay search smoke test against `fetchEbayItems`. +- Create `packages/core/test/live/kijiji.live.test.ts`: live Kijiji search smoke test against `fetchKijijiItems`. +- Create `packages/core/test/live/facebook.live.test.ts`: strict live Facebook search smoke test against `fetchFacebookItems` and `FACEBOOK_COOKIE`. +- Modify `package.json`: add root script `test:live` running all files under `packages/core/test/live`. + +### Task 1: Add eBay Live Suite + +**Files:** +- Create: `packages/core/test/live/ebay.live.test.ts` + +- [ ] **Step 1: Write the live test file** + +```ts +import { describe, expect, test } from "bun:test"; +import fetchEbayItems from "../../src/scrapers/ebay"; + +describe("eBay live parser", () => { + test("scrapes live search results into listing details", async () => { + const results = await fetchEbayItems("iphone", 1, { maxItems: 3 }); + + expect(results.length).toBeGreaterThan(0); + for (const listing of results) { + expect(listing.url).toStartWith("https://"); + expect(listing.title.length).toBeGreaterThan(0); + expect(listing.listingPrice.cents).toBeGreaterThanOrEqual(0); + expect(listing.listingPrice.currency.length).toBeGreaterThan(0); + } + }); +}); +``` + +- [ ] **Step 2: Run eBay live test** + +Run: `bun test packages/core/test/live/ebay.live.test.ts` +Expected: PASS when eBay returns parseable search results; FAIL on endpoint/rate-limit/parser breakage. + +### Task 2: Add Kijiji Live Suite + +**Files:** +- Create: `packages/core/test/live/kijiji.live.test.ts` + +- [ ] **Step 1: Write the live test file** + +```ts +import { describe, expect, test } from "bun:test"; +import fetchKijijiItems from "../../src/scrapers/kijiji"; + +describe("Kijiji live parser", () => { + test("scrapes live search results into detailed listings", async () => { + const results = await fetchKijijiItems( + "iphone", + 1, + "https://www.kijiji.ca", + { maxPages: 1 }, + { includeImages: false, sellerDataDepth: "basic" }, + ); + + expect(results.length).toBeGreaterThan(0); + for (const listing of results) { + expect(listing.url).toStartWith("https://www.kijiji.ca/"); + expect(listing.title.length).toBeGreaterThan(0); + expect(listing.listingPrice.cents).toBeGreaterThanOrEqual(0); + expect(listing.listingPrice.currency.length).toBeGreaterThan(0); + } + }); +}); +``` + +- [ ] **Step 2: Run Kijiji live test** + +Run: `bun test packages/core/test/live/kijiji.live.test.ts` +Expected: PASS when Kijiji returns parseable search and detail pages; FAIL on endpoint/parser breakage. + +### Task 3: Add Facebook Live Suite + +**Files:** +- Create: `packages/core/test/live/facebook.live.test.ts` + +- [ ] **Step 1: Write the live test file** + +```ts +import { describe, expect, test } from "bun:test"; +import fetchFacebookItems from "../../src/scrapers/facebook"; + +describe("Facebook live parser", () => { + test("requires FACEBOOK_COOKIE for strict live testing", () => { + expect(process.env.FACEBOOK_COOKIE?.trim().length ?? 0).toBeGreaterThan(0); + }); + + test("scrapes live marketplace search results into listing details", async () => { + const results = await fetchFacebookItems("iphone", 1, "toronto", 3); + + expect(results.length).toBeGreaterThan(0); + for (const listing of results) { + expect(listing.url).toStartWith("https://www.facebook.com/marketplace/item/"); + expect(listing.title.length).toBeGreaterThan(0); + expect(listing.listingPrice.cents).toBeGreaterThanOrEqual(0); + expect(listing.listingPrice.currency.length).toBeGreaterThan(0); + } + }); +}); +``` + +- [ ] **Step 2: Run Facebook live test** + +Run: `bun test packages/core/test/live/facebook.live.test.ts` +Expected: PASS with valid `FACEBOOK_COOKIE`; FAIL when `FACEBOOK_COOKIE` is missing, expired, or parser output is empty. + +### Task 4: Add Root Live Test Script + +**Files:** +- Modify: `package.json` + +- [ ] **Step 1: Add script** + +Change root `scripts` to include: + +```json +{ + "test:live": "bun test packages/core/test/live" +} +``` + +- [ ] **Step 2: Run all live tests through script** + +Run: `bun run test:live` +Expected: runs eBay, Kijiji, and Facebook live suites. Facebook fails if `FACEBOOK_COOKIE` is unset. + +### Task 5: Verify Default Suite Exclusion + +**Files:** +- No code files modified. + +- [ ] **Step 1: Run existing core tests** + +Run: `bun test packages/core/test` +Expected: existing mocked tests run. If Bun discovers `packages/core/test/live`, change normal verification command to explicit glob `bun test packages/core/test/*.test.ts` and document that in final notes. + +- [ ] **Step 2: Run static checks** + +Run: `bun run ci` +Expected: typecheck and Biome pass. Fix code issues without changing lint or TypeScript rules. + +## Commit Note + +Do not commit during execution unless user explicitly requests a commit. This repo session policy overrides generic plan commit steps. + +## Self-Review + +- Spec coverage: eBay, Kijiji, Facebook live suites; explicit script; strict Facebook auth; excluded from default flow. +- Placeholder scan: no `TBD`, `TODO`, or underspecified implementation steps. +- Type consistency: tests use current exported scraper signatures and shared listing fields from `ListingDetails`. diff --git a/docs/superpowers/specs/2026-04-30-live-parser-tests-design.md b/docs/superpowers/specs/2026-04-30-live-parser-tests-design.md new file mode 100644 index 0000000..964ddae --- /dev/null +++ b/docs/superpowers/specs/2026-04-30-live-parser-tests-design.md @@ -0,0 +1,37 @@ +# Live Parser Tests Design + +## Summary + +Add explicit live endpoint tests for each core scraper parser path. These tests are excluded from normal deterministic test commands and run only through a dedicated package script. + +## Scope + +- Add one live suite per parser: eBay, Kijiji, Facebook. +- Place suites under `packages/core/test/live/` so normal `bun test packages/core/test/*.test.ts` patterns do not include them accidentally. +- Add a root `test:live` script that runs all live suites together. +- Keep existing mocked tests unchanged. + +## Behavior + +- Each suite calls the public scraper entry point for that marketplace with a narrow query and low max item count. +- Assertions verify scrape output shape and parser viability, not exact listing identity. +- eBay and Kijiji require live network access and fail on endpoint/parser breakage. +- Facebook is strict: missing or expired `FACEBOOK_COOKIE` fails the live suite instead of skipping. + +## Test Data + +- Use stable broad Canadian queries such as `iphone` or `laptop` to reduce empty-result risk. +- Use low limits to avoid unnecessary load and rate-limit pressure. +- Avoid exact prices, titles, listing IDs, or ordering assumptions. + +## Failure Meaning + +- Empty result arrays fail because live parser logic did not produce usable listings. +- Missing required fields fail because adapter contracts depend on those fields. +- Authentication failures fail for Facebook because selected scope is strict. + +## Verification + +- Normal suite remains offline: `bun test packages/core/test`. +- Live suite runs by explicit script: `bun run test:live`. +- Full static checks remain via `bun run ci`.