From f8975fa91d087a387e1abc996f52f85c9163116f Mon Sep 17 00:00:00 2001 From: Dmytro Stanchiev Date: Wed, 22 Apr 2026 17:53:45 -0400 Subject: [PATCH] docs: add unstable listing mode plan --- .../plans/2026-04-22-unstable-listing-mode.md | 672 ++++++++++++++++++ 1 file changed, 672 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-22-unstable-listing-mode.md diff --git a/docs/superpowers/plans/2026-04-22-unstable-listing-mode.md b/docs/superpowers/plans/2026-04-22-unstable-listing-mode.md new file mode 100644 index 0000000..5756337 --- /dev/null +++ b/docs/superpowers/plans/2026-04-22-unstable-listing-mode.md @@ -0,0 +1,672 @@ +# Unstable Listing Mode Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add an optional shared mode across Facebook, eBay, and Kijiji that moves listings priced below 80% of the median into `unstableResults`, while preserving current default response shapes. + +**Architecture:** Introduce a shared generic classifier in `packages/core` that splits any listing array into `results` and `unstableResults` using the same median-based rule. Then thread one opt-in flag through the scraper entrypoints, API routes, and MCP tool definitions so all surfaces expose the same behavior without changing existing defaults. + +**Tech Stack:** Bun, TypeScript, Bun test, workspace packages, JSON-RPC MCP server + +--- + +## File Map + +- Create: `packages/core/src/utils/unstable.ts` + Purpose: shared generic median/cutoff classifier for listing arrays. +- Modify: `packages/core/src/types/common.ts` + Purpose: add shared mode types used by scrapers and adapters. +- Modify: `packages/core/src/index.ts` + Purpose: export the new shared classifier/types. +- Modify: `packages/core/src/scrapers/facebook.ts` + Purpose: add the optional mode flag and return bucketed results when enabled. +- Modify: `packages/core/src/scrapers/ebay.ts` + Purpose: add the optional mode flag and return bucketed results when enabled. +- Modify: `packages/core/src/scrapers/kijiji.ts` + Purpose: add the optional mode flag and return bucketed results when enabled. +- Create: `packages/core/test/unstable-listing-mode.test.ts` + Purpose: lock the shared classifier behavior with direct unit tests. +- Modify: `packages/core/test/facebook-core.test.ts` + Purpose: prove Facebook preserves default arrays and returns buckets when enabled. +- Modify: `packages/core/test/ebay-core.test.ts` + Purpose: prove eBay preserves default arrays and returns buckets when enabled. +- Modify: `packages/core/test/kijiji-core.test.ts` + Purpose: prove Kijiji preserves default arrays and returns buckets when enabled. +- Modify: `packages/api-server/src/routes/facebook.ts` + Purpose: expose a shared opt-in query parameter and preserve default response shape. +- Modify: `packages/api-server/src/routes/ebay.ts` + Purpose: expose the same query parameter and preserve default response shape. +- Modify: `packages/api-server/src/routes/kijiji.ts` + Purpose: expose the same query parameter and preserve default response shape. +- Modify: `packages/api-server/test/routes.test.ts` + Purpose: verify route forwarding and route response-shape switching. +- Modify: `packages/mcp-server/src/protocol/tools.ts` + Purpose: document the optional unstable mode in all search tools. +- Modify: `packages/mcp-server/src/protocol/handler.ts` + Purpose: forward the optional mode to API routes for all search tools. +- Modify: `packages/mcp-server/test/protocol.test.ts` + Purpose: verify MCP tool metadata and forwarded URLs include the new option. + +### Task 1: Add the shared unstable-listing classifier + +**Files:** +- Create: `packages/core/src/utils/unstable.ts` +- Modify: `packages/core/src/types/common.ts` +- Modify: `packages/core/src/index.ts` +- Test: `packages/core/test/unstable-listing-mode.test.ts` + +- [ ] **Step 1: Write the failing test** + +Create `packages/core/test/unstable-listing-mode.test.ts` with focused shared-behavior coverage: + +```ts +import { describe, expect, test } from "bun:test"; +import { + classifyUnstableListings, + type ListingDetails, +} from "../src/index"; + +function makeListing(title: string, cents?: number): ListingDetails { + return { + url: `https://example.com/${title}`, + title, + listingPrice: { + amountFormatted: cents ? `$${(cents / 100).toFixed(2)}` : "$0.00", + cents: cents ?? 0, + currency: "CAD", + }, + listingType: "item", + listingStatus: "ACTIVE", + }; +} + +describe("classifyUnstableListings", () => { + test("moves listings below 80% of the median into unstableResults", () => { + const output = classifyUnstableListings([ + makeListing("cheap", 1000), + makeListing("mid", 2000), + makeListing("high", 3000), + ]); + + expect(output.results.map((item) => item.title)).toEqual(["mid", "high"]); + expect(output.unstableResults.map((item) => item.title)).toEqual(["cheap"]); + }); + + test("uses the midpoint median for even-sized priced inputs", () => { + const output = classifyUnstableListings([ + makeListing("a", 1000), + makeListing("b", 2000), + makeListing("c", 3000), + makeListing("d", 4000), + ]); + + expect(output.results.map((item) => item.title)).toEqual(["b", "c", "d"]); + expect(output.unstableResults.map((item) => item.title)).toEqual(["a"]); + }); + + test("keeps non-positive prices in results while excluding them from median input", () => { + const output = classifyUnstableListings([ + makeListing("free", 0), + makeListing("cheap", 1000), + makeListing("mid", 2000), + makeListing("high", 3000), + ]); + + expect(output.results.map((item) => item.title)).toEqual(["free", "mid", "high"]); + expect(output.unstableResults.map((item) => item.title)).toEqual(["cheap"]); + }); + + test("returns all listings as results when fewer than two valid prices exist", () => { + const output = classifyUnstableListings([makeListing("only", 2500)]); + + expect(output.results.map((item) => item.title)).toEqual(["only"]); + expect(output.unstableResults).toEqual([]); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test packages/core/test/unstable-listing-mode.test.ts` +Expected: FAIL because `classifyUnstableListings` and the shared mode types do not exist yet. + +- [ ] **Step 3: Write minimal implementation** + +Add shared types in `packages/core/src/types/common.ts`: + +```ts +export interface UnstableListingBuckets { + results: T[]; + unstableResults: T[]; +} + +export interface UnstableListingModeOptions { + hideUnstableResults?: boolean; +} +``` + +Create `packages/core/src/utils/unstable.ts` with the shared classifier: + +```ts +import type { ListingDetails, UnstableListingBuckets } from "../types/common"; + +function getMedian(values: number[]): number | null { + if (values.length < 2) return null; + + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + + if (sorted.length % 2 === 0) { + return (sorted[middle - 1] + sorted[middle]) / 2; + } + + return sorted[middle]; +} + +export function classifyUnstableListings( + listings: T[], +): UnstableListingBuckets { + const pricedValues = listings + .map((listing) => listing.listingPrice?.cents) + .filter((cents): cents is number => Number.isFinite(cents) && cents > 0); + + const median = getMedian(pricedValues); + if (median == null) { + return { results: listings, unstableResults: [] }; + } + + const threshold = median * 0.8; + const results: T[] = []; + const unstableResults: T[] = []; + + for (const listing of listings) { + const cents = listing.listingPrice?.cents; + if (Number.isFinite(cents) && cents > 0 && cents < threshold) { + unstableResults.push(listing); + continue; + } + + results.push(listing); + } + + return { results, unstableResults }; +} +``` + +Export the new symbols from `packages/core/src/index.ts`: + +```ts +export * from "./types/common"; +export { classifyUnstableListings } from "./utils/unstable"; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test packages/core/test/unstable-listing-mode.test.ts` +Expected: PASS with 4 passing tests. + +- [ ] **Step 5: Commit** + +```bash +git add packages/core/src/utils/unstable.ts packages/core/src/types/common.ts packages/core/src/index.ts packages/core/test/unstable-listing-mode.test.ts +git commit -m "feat: add shared unstable listing classifier" +``` + +### Task 2: Thread the optional mode through all core scrapers + +**Files:** +- Modify: `packages/core/src/scrapers/facebook.ts` +- Modify: `packages/core/src/scrapers/ebay.ts` +- Modify: `packages/core/src/scrapers/kijiji.ts` +- Modify: `packages/core/test/facebook-core.test.ts` +- Modify: `packages/core/test/ebay-core.test.ts` +- Modify: `packages/core/test/kijiji-core.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Add one focused opt-in test per scraper. Use the new shared classifier through the public scraper entrypoints instead of testing internal helpers. + +In `packages/core/test/facebook-core.test.ts`, add: + +```ts +test("fetchFacebookItems returns stable and unstable buckets when unstable mode is enabled", async () => { + process.env.FACEBOOK_COOKIE = "c_user=123; xs=abc"; + + global.fetch = mock(() => + Promise.resolve({ + ok: true, + text: () => Promise.resolve(facebookSearchHtmlFixture), + headers: { get: () => null }, + }), + ); + + const result = await fetchFacebookItems("bike", 1, "toronto", 25, { + hideUnstableResults: true, + }); + + expect(result).toHaveProperty("results"); + expect(result).toHaveProperty("unstableResults"); +}); +``` + +In `packages/core/test/ebay-core.test.ts`, add: + +```ts +test("fetchEbayItems returns stable and unstable buckets when unstable mode is enabled", async () => { + const result = await fetchEbayItems("bike", 1, { + keywords: ["bike"], + exclusions: [], + strictMode: false, + buyItNowOnly: true, + canadaOnly: true, + }, { + hideUnstableResults: true, + }); + + expect(result).toHaveProperty("results"); + expect(result).toHaveProperty("unstableResults"); +}); +``` + +In `packages/core/test/kijiji-core.test.ts`, add: + +```ts +test("fetchKijijiItems returns stable and unstable buckets when unstable mode is enabled", async () => { + const result = await fetchKijijiItems( + "bike", + 1, + "https://www.kijiji.ca", + { maxPages: 1 }, + {}, + { hideUnstableResults: true }, + ); + + expect(result).toHaveProperty("results"); + expect(result).toHaveProperty("unstableResults"); +}); +``` + +Also add one default-mode assertion in one existing scraper test file, for example in `packages/core/test/facebook-core.test.ts`: + +```ts +test("fetchFacebookItems keeps returning an array by default", async () => { + process.env.FACEBOOK_COOKIE = "c_user=123; xs=abc"; + + global.fetch = mock(() => + Promise.resolve({ + ok: true, + text: () => Promise.resolve(facebookSearchHtmlFixture), + headers: { get: () => null }, + }), + ); + + const result = await fetchFacebookItems("bike"); + expect(Array.isArray(result)).toBe(true); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test packages/core/test/facebook-core.test.ts packages/core/test/ebay-core.test.ts packages/core/test/kijiji-core.test.ts` +Expected: FAIL because the scraper signatures do not yet accept the new option and still always return arrays. + +- [ ] **Step 3: Write minimal implementation** + +Add a small shared helper type import to each scraper: + +```ts +import { + classifyUnstableListings, + type UnstableListingBuckets, + type UnstableListingModeOptions, +} from "../index"; +``` + +In `packages/core/src/scrapers/facebook.ts`, extend the default export signature and branch at the end: + +```ts +export default async function fetchFacebookItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND = 1, + LOCATION = "toronto", + MAX_ITEMS = 25, + unstableOptions: UnstableListingModeOptions = {}, +): Promise> { + // existing fetch/parsing logic + + const limitedItems = pricedItems.slice(0, MAX_ITEMS); + if (!unstableOptions.hideUnstableResults) { + return limitedItems; + } + + const classified = classifyUnstableListings(pricedItems); + return { + results: classified.results.slice(0, MAX_ITEMS), + unstableResults: classified.unstableResults, + }; +} +``` + +In `packages/core/src/scrapers/ebay.ts`, extend the entrypoint the same way: + +```ts +export default async function fetchEbayItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND = 1, + options: EbaySearchOptions = {}, + unstableOptions: UnstableListingModeOptions = {}, +): Promise> { + // existing fetch/parsing logic + + const limitedResults = maxItems ? listings.slice(0, maxItems) : listings; + if (!unstableOptions.hideUnstableResults) { + return limitedResults; + } + + const classified = classifyUnstableListings(listings); + return { + results: maxItems ? classified.results.slice(0, maxItems) : classified.results, + unstableResults: classified.unstableResults, + }; +} +``` + +In `packages/core/src/scrapers/kijiji.ts`, add the same final argument after `listingOptions`: + +```ts +export default async function fetchKijijiItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND = 1, + BASE_URL = "https://www.kijiji.ca", + searchOptions: SearchOptions = {}, + listingOptions: ListingFetchOptions = {}, + unstableOptions: UnstableListingModeOptions = {}, +): Promise> { + // existing fetch/parsing logic + + if (!unstableOptions.hideUnstableResults) { + return allListings; + } + + return classifyUnstableListings(allListings); +} +``` + +Keep the default branch untouched in all three files so existing callers still receive arrays. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test packages/core/test/unstable-listing-mode.test.ts packages/core/test/facebook-core.test.ts packages/core/test/ebay-core.test.ts packages/core/test/kijiji-core.test.ts` +Expected: PASS, including the new opt-in bucket assertions and the default-array regression assertion. + +- [ ] **Step 5: Commit** + +```bash +git add packages/core/src/scrapers/facebook.ts packages/core/src/scrapers/ebay.ts packages/core/src/scrapers/kijiji.ts packages/core/test/facebook-core.test.ts packages/core/test/ebay-core.test.ts packages/core/test/kijiji-core.test.ts +git commit -m "feat: add unstable mode to scraper results" +``` + +### Task 3: Expose unstable mode in API routes + +**Files:** +- Modify: `packages/api-server/src/routes/facebook.ts` +- Modify: `packages/api-server/src/routes/ebay.ts` +- Modify: `packages/api-server/src/routes/kijiji.ts` +- Modify: `packages/api-server/test/routes.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Extend `packages/api-server/test/routes.test.ts` with route-forwarding coverage for the new query parameter: + +```ts +test("facebookRoute forwards unstableFilter=true to core", async () => { + const { facebookRoute } = await import("../src/routes/facebook"); + + await facebookRoute( + new Request( + "http://localhost/api/facebook?q=laptop&location=toronto&maxItems=3&unstableFilter=true", + ), + ); + + expect(fetchFacebookItems).toHaveBeenCalledWith( + "laptop", + 1, + "toronto", + 3, + { hideUnstableResults: true }, + ); +}); + +test("ebayRoute forwards unstableFilter=true to core", async () => { + const { ebayRoute } = await import("../src/routes/ebay"); + + await ebayRoute( + new Request("http://localhost/api/ebay?q=laptop&unstableFilter=true"), + ); + + expect(fetchEbayItems).toHaveBeenCalledWith( + "laptop", + 1, + { + minPrice: undefined, + maxPrice: undefined, + strictMode: false, + exclusions: [], + keywords: ["laptop"], + buyItNowOnly: true, + canadaOnly: true, + }, + { hideUnstableResults: true }, + ); +}); + +test("kijijiRoute forwards unstableFilter=true to core", async () => { + const { kijijiRoute } = await import("../src/routes/kijiji"); + + await kijijiRoute( + new Request("http://localhost/api/kijiji?q=laptop&unstableFilter=true"), + ); + + expect(fetchKijijiItems).toHaveBeenCalledWith( + "laptop", + 4, + "https://www.kijiji.ca", + expect.any(Object), + {}, + { hideUnstableResults: true }, + ); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test packages/api-server/test/routes.test.ts` +Expected: FAIL because the routes do not yet parse or forward `unstableFilter`. + +- [ ] **Step 3: Write minimal implementation** + +In each route, parse the shared boolean once: + +```ts +const hideUnstableResults = reqUrl.searchParams.get("unstableFilter") === "true"; +``` + +Update the core calls to forward the shared option. + +In `packages/api-server/src/routes/facebook.ts`: + +```ts +const items = await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems, { + hideUnstableResults, +}); +``` + +In `packages/api-server/src/routes/ebay.ts`: + +```ts +const items = await fetchEbayItems( + SEARCH_QUERY, + 1, + { + minPrice, + maxPrice, + strictMode, + exclusions, + keywords, + buyItNowOnly, + canadaOnly, + }, + { hideUnstableResults }, +); +``` + +In `packages/api-server/src/routes/kijiji.ts`: + +```ts +const items = await fetchKijijiItems( + SEARCH_QUERY, + 4, + "https://www.kijiji.ca", + searchOptions, + {}, + { hideUnstableResults }, +); +``` + +Do not add any response wrapper logic in the routes; simply return whatever the core scraper returns so the default array path remains unchanged. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test packages/api-server/test/routes.test.ts` +Expected: PASS, including existing cookie-parameter regression tests and the new unstable-mode forwarding assertions. + +- [ ] **Step 5: Commit** + +```bash +git add packages/api-server/src/routes/facebook.ts packages/api-server/src/routes/ebay.ts packages/api-server/src/routes/kijiji.ts packages/api-server/test/routes.test.ts +git commit -m "feat: expose unstable mode in api routes" +``` + +### Task 4: Document and forward unstable mode in MCP tools + +**Files:** +- Modify: `packages/mcp-server/src/protocol/tools.ts` +- Modify: `packages/mcp-server/src/protocol/handler.ts` +- Modify: `packages/mcp-server/test/protocol.test.ts` + +- [ ] **Step 1: Write the failing tests** + +Extend `packages/mcp-server/test/protocol.test.ts` with metadata and forwarding coverage: + +```ts +test("search tools document unstable listing mode", () => { + for (const toolName of ["search_kijiji", "search_facebook", "search_ebay"]) { + const tool = tools.find((entry) => entry.name === toolName); + + expect(tool?.inputSchema.properties).toHaveProperty("unstableFilter"); + expect(tool?.inputSchema.properties.unstableFilter.description).toContain( + "20% below the median", + ); + expect(tool?.inputSchema.properties.unstableFilter.description).toContain( + "unstableResults", + ); + } +}); + +test("search_facebook forwards unstableFilter to the API", async () => { + await handleMcpRequest( + new Request("http://localhost", { + method: "POST", + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "search_facebook", + arguments: { + query: "laptop", + unstableFilter: true, + }, + }, + }), + }), + ); + + const calledUrl = (global.fetch as ReturnType).mock.calls[0]?.[0]; + expect(String(calledUrl)).toContain("unstableFilter=true"); +}); +``` + +Mirror the forwarding assertion for `search_kijiji` and `search_ebay` in the same file. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test packages/mcp-server/test/protocol.test.ts` +Expected: FAIL because the tools do not yet describe `unstableFilter` and the handler does not append it to API URLs. + +- [ ] **Step 3: Write minimal implementation** + +In `packages/mcp-server/src/protocol/tools.ts`, add the same optional property to all three tools: + +```ts +unstableFilter: { + type: "boolean", + description: + "Optional: move listings priced more than 20% below the median into unstableResults instead of the main results. When enabled, the response shape changes from a plain list to an object with results and unstableResults.", + default: false, +}, +``` + +In `packages/mcp-server/src/protocol/handler.ts`, append the shared flag in each search branch: + +```ts +if (args.unstableFilter !== undefined) { + params.append("unstableFilter", args.unstableFilter.toString()); +} +``` + +Add that snippet to the `search_kijiji`, `search_facebook`, and `search_ebay` branches. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test packages/mcp-server/test/protocol.test.ts` +Expected: PASS, including the new tool-schema assertions and URL-forwarding assertions. + +- [ ] **Step 5: Commit** + +```bash +git add packages/mcp-server/src/protocol/tools.ts packages/mcp-server/src/protocol/handler.ts packages/mcp-server/test/protocol.test.ts +git commit -m "docs: expose unstable mode in mcp tools" +``` + +### Task 5: Verify the full cross-package feature end to end + +**Files:** +- No code changes expected. + +- [ ] **Step 1: Run the focused package tests** + +Run: `bun test packages/core/test/unstable-listing-mode.test.ts packages/core/test/facebook-core.test.ts packages/core/test/ebay-core.test.ts packages/core/test/kijiji-core.test.ts packages/api-server/test/routes.test.ts packages/mcp-server/test/protocol.test.ts` +Expected: PASS with zero failing tests. + +- [ ] **Step 2: Run the broader workspace verification** + +Run: `bun run ci` +Expected: PASS with clean workspace validation. + +- [ ] **Step 3: Commit verification-only follow-ups if needed** + +If verification forced any tiny fixes, commit them immediately after the fix with a focused message, for example: + +```bash +git add +git commit -m "fix: align unstable mode verification" +``` + +If no files changed during verification, skip this commit step. + +## Self-Review + +- Spec coverage: shared classifier, all three scrapers, API exposure, MCP documentation, and tests are each mapped to a task. +- Placeholder scan: no `TODO`, `TBD`, or "write tests later" placeholders remain. +- Type consistency: the plan uses one shared flag name, `unstableFilter`, and one shared core option, `hideUnstableResults`, across all tasks.