docs: align cookie setup with env-only auth
This commit is contained in:
543
docs/superpowers/plans/2026-04-21-cookie-env-only.md
Normal file
543
docs/superpowers/plans/2026-04-21-cookie-env-only.md
Normal file
@@ -0,0 +1,543 @@
|
||||
# Cookie Env-Only Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Remove cookie files and request-provided cookie overrides so all authenticated marketplace scraping reads raw `Cookie` header strings only from environment variables.
|
||||
|
||||
**Architecture:** Collapse shared cookie loading to a single env-var reader in `packages/core/src/utils/cookies.ts`, then tighten Facebook and eBay core signatures to stop accepting request/file cookie inputs. Update the API and MCP adapters so they no longer advertise or forward cookie parameters, and rewrite docs/tests to match the env-only contract.
|
||||
|
||||
**Tech Stack:** Bun, TypeScript, Bun test, Biome, workspace package exports
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
- Modify: `packages/core/src/utils/cookies.ts`
|
||||
Purpose: remove JSON/file/request-source loading and keep env-only cookie parsing/formatting.
|
||||
- Modify: `packages/core/src/scrapers/facebook.ts`
|
||||
Purpose: drop `cookiesSource` / `cookiePath` arguments and env-only error text.
|
||||
- Modify: `packages/core/src/scrapers/ebay.ts`
|
||||
Purpose: remove `opts.cookies` request override and use env-only cookie loading.
|
||||
- Modify: `packages/core/src/index.ts`
|
||||
Purpose: keep exports aligned with tightened core signatures.
|
||||
- Modify: `packages/core/test/facebook-core.test.ts`
|
||||
Purpose: replace missing-file coverage with env-only auth tests.
|
||||
- Create: `packages/core/test/ebay-core.test.ts`
|
||||
Purpose: add dedicated eBay auth regression coverage instead of mixing it into Facebook tests.
|
||||
- Modify: `packages/api-server/src/routes/facebook.ts`
|
||||
Purpose: stop parsing/forwarding `cookies` query params.
|
||||
- Modify: `packages/api-server/src/routes/ebay.ts`
|
||||
Purpose: stop parsing/forwarding `cookies` query params.
|
||||
- Create: `packages/api-server/test/routes.test.ts`
|
||||
Purpose: verify Facebook/eBay routes ignore cookie query params and still call core correctly.
|
||||
- Modify: `packages/mcp-server/src/protocol/tools.ts`
|
||||
Purpose: remove Facebook/eBay cookie tool inputs and descriptions.
|
||||
- Modify: `packages/mcp-server/src/protocol/handler.ts`
|
||||
Purpose: stop mapping removed cookie tool inputs into API URLs.
|
||||
- Create: `packages/mcp-server/test/protocol.test.ts`
|
||||
Purpose: verify tool schemas and handler URL building no longer include Facebook/eBay cookie fields.
|
||||
- Modify: `cookies/AGENTS.md`
|
||||
Purpose: document env vars as the only supported cookie input.
|
||||
|
||||
### Task 1: Lock core cookie utilities to env-only loading
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/core/src/utils/cookies.ts:19-227`
|
||||
- Test: `packages/core/test/facebook-core.test.ts`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add or replace the auth-source test block in `packages/core/test/facebook-core.test.ts` with env-only expectations:
|
||||
|
||||
```ts
|
||||
test("should load Facebook cookies from FACEBOOK_COOKIE env var", async () => {
|
||||
const previous = process.env.FACEBOOK_COOKIE;
|
||||
process.env.FACEBOOK_COOKIE = "c_user=123; xs=abc";
|
||||
|
||||
try {
|
||||
const cookies = await ensureFacebookCookies();
|
||||
|
||||
expect(cookies.map((cookie) => cookie.name)).toEqual(["c_user", "xs"]);
|
||||
} finally {
|
||||
if (previous === undefined) {
|
||||
delete process.env.FACEBOOK_COOKIE;
|
||||
} else {
|
||||
process.env.FACEBOOK_COOKIE = previous;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("should reject missing Facebook auth env var", async () => {
|
||||
const previous = process.env.FACEBOOK_COOKIE;
|
||||
delete process.env.FACEBOOK_COOKIE;
|
||||
|
||||
try {
|
||||
await expect(ensureFacebookCookies()).rejects.toThrow(
|
||||
"Provide cookies via FACEBOOK_COOKIE environment variable as a raw Cookie header string",
|
||||
);
|
||||
} finally {
|
||||
if (previous !== undefined) {
|
||||
process.env.FACEBOOK_COOKIE = previous;
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `bun test packages/core/test/facebook-core.test.ts`
|
||||
Expected: FAIL because the current implementation still allows missing env values to fall through to file/request-based behavior and does not emit the new env-only error.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Replace the multi-source loader in `packages/core/src/utils/cookies.ts` with an env-only loader. The target shape is:
|
||||
|
||||
```ts
|
||||
export interface CookieConfig {
|
||||
name: string;
|
||||
domain: string;
|
||||
envVar: string;
|
||||
}
|
||||
|
||||
export async function ensureCookies(config: CookieConfig): Promise<Cookie[]> {
|
||||
const envValue = process.env[config.envVar];
|
||||
const cookies = parseCookieString(envValue ?? "", config.domain);
|
||||
|
||||
if (cookies.length > 0) {
|
||||
console.log(
|
||||
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
|
||||
);
|
||||
return cookies;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`No valid ${config.name} cookies found. Provide cookies via ${config.envVar} environment variable as a raw Cookie header string.`,
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
Delete the now-dead helpers and types that exist only for JSON/file/request loading:
|
||||
|
||||
```ts
|
||||
// Remove:
|
||||
// - parseJsonCookies
|
||||
// - parseCookiesAuto
|
||||
// - loadCookiesFromFile
|
||||
// - loadCookiesOptional
|
||||
// - CookieConfig.filePath
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `bun test packages/core/test/facebook-core.test.ts`
|
||||
Expected: PASS for the new env-only tests.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/core/src/utils/cookies.ts packages/core/test/facebook-core.test.ts
|
||||
git commit -m "refactor: make cookie loading env-only"
|
||||
```
|
||||
|
||||
### Task 2: Tighten Facebook core APIs to the new contract
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/core/src/scrapers/facebook.ts:23-29`
|
||||
- Modify: `packages/core/src/scrapers/facebook.ts:214-228`
|
||||
- Modify: `packages/core/src/scrapers/facebook.ts:823-929`
|
||||
- Modify: `packages/core/src/index.ts:5-15`
|
||||
- Test: `packages/core/test/facebook-core.test.ts`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Add a focused test proving Facebook item fetch now depends only on env auth:
|
||||
|
||||
```ts
|
||||
test("should fail Facebook item fetch when FACEBOOK_COOKIE is unset", async () => {
|
||||
const previous = process.env.FACEBOOK_COOKIE;
|
||||
delete process.env.FACEBOOK_COOKIE;
|
||||
|
||||
try {
|
||||
await expect(fetchFacebookItem("123")).rejects.toThrow(
|
||||
"Provide cookies via FACEBOOK_COOKIE environment variable as a raw Cookie header string",
|
||||
);
|
||||
} finally {
|
||||
if (previous !== undefined) {
|
||||
process.env.FACEBOOK_COOKIE = previous;
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `bun test packages/core/test/facebook-core.test.ts`
|
||||
Expected: FAIL because the current function signatures and error text still mention parameter/file-based auth paths.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Tighten the Facebook signatures and messages:
|
||||
|
||||
```ts
|
||||
const FACEBOOK_COOKIE_CONFIG: CookieConfig = {
|
||||
name: "Facebook",
|
||||
domain: ".facebook.com",
|
||||
envVar: "FACEBOOK_COOKIE",
|
||||
};
|
||||
|
||||
export async function ensureFacebookCookies(): Promise<Cookie[]> {
|
||||
return ensureCookies(FACEBOOK_COOKIE_CONFIG);
|
||||
}
|
||||
|
||||
export default async function fetchFacebookItems(
|
||||
SEARCH_QUERY: string,
|
||||
REQUESTS_PER_SECOND = 1,
|
||||
LOCATION = "toronto",
|
||||
MAX_ITEMS = 25,
|
||||
) {
|
||||
const cookies = await ensureFacebookCookies();
|
||||
```
|
||||
|
||||
Also change the stale auth warnings:
|
||||
|
||||
```ts
|
||||
console.warn(
|
||||
"This might indicate invalid or expired cookies. Update FACEBOOK_COOKIE with a fresh raw Cookie header string.",
|
||||
);
|
||||
```
|
||||
|
||||
Remove the extra cookie arguments from `fetchFacebookItem(...)` and keep `packages/core/src/index.ts` exporting the tightened functions without the old parameter contract.
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `bun test packages/core/test/facebook-core.test.ts`
|
||||
Expected: PASS with the new env-only Facebook API surface.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/core/src/scrapers/facebook.ts packages/core/src/index.ts packages/core/test/facebook-core.test.ts
|
||||
git commit -m "refactor: remove facebook cookie overrides"
|
||||
```
|
||||
|
||||
### Task 3: Tighten eBay core APIs to env-only auth
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/core/src/scrapers/ebay.ts:9-15`
|
||||
- Modify: `packages/core/src/scrapers/ebay.ts:337-389`
|
||||
- Create: `packages/core/test/ebay-core.test.ts`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `packages/core/test/ebay-core.test.ts` with a dedicated auth regression test:
|
||||
|
||||
```ts
|
||||
test("should warn and continue without eBay cookies when EBAY_COOKIE is unset", async () => {
|
||||
const previous = process.env.EBAY_COOKIE;
|
||||
delete process.env.EBAY_COOKIE;
|
||||
|
||||
try {
|
||||
const cookies = await loadEbayCookies();
|
||||
expect(cookies).toBeUndefined();
|
||||
} finally {
|
||||
if (previous !== undefined) {
|
||||
process.env.EBAY_COOKIE = previous;
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `bun test packages/core/test/ebay-core.test.ts`
|
||||
Expected: FAIL because `loadEbayCookies` still accepts request overrides and mentions file/json sources.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Remove file/request branches from the eBay cookie path:
|
||||
|
||||
```ts
|
||||
const EBAY_COOKIE_CONFIG: CookieConfig = {
|
||||
name: "eBay",
|
||||
domain: ".ebay.ca",
|
||||
envVar: "EBAY_COOKIE",
|
||||
};
|
||||
|
||||
async function loadEbayCookies(): Promise<string | undefined> {
|
||||
try {
|
||||
const cookies = await ensureCookies(EBAY_COOKIE_CONFIG);
|
||||
return formatCookiesForHeader(cookies, "www.ebay.ca");
|
||||
} catch {
|
||||
console.warn(
|
||||
"No valid eBay cookies found in EBAY_COOKIE. eBay may block requests without a raw Cookie header string.",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then remove `cookies` from `fetchEbayItems(..., opts)` and the destructuring that feeds it into `loadEbayCookies()`.
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `bun test packages/core/test/ebay-core.test.ts`
|
||||
Expected: PASS for the eBay env-only regression coverage.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/core/src/scrapers/ebay.ts packages/core/test/ebay-core.test.ts
|
||||
git commit -m "refactor: make ebay auth env-only"
|
||||
```
|
||||
|
||||
### Task 4: Remove cookie query parameters from the API adapter
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/api-server/src/routes/facebook.ts:3-33`
|
||||
- Modify: `packages/api-server/src/routes/ebay.ts:3-52`
|
||||
- Create: `packages/api-server/test/routes.test.ts`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `packages/api-server/test/routes.test.ts` and mock `@marketplace-scrapers/core` so the route contract is explicit:
|
||||
|
||||
```ts
|
||||
import { afterEach, describe, expect, mock, test } from "bun:test";
|
||||
|
||||
const fetchFacebookItems = mock(() => Promise.resolve([{ title: "item" }]));
|
||||
const fetchEbayItems = mock(() => Promise.resolve([{ title: "item" }]));
|
||||
|
||||
mock.module("@marketplace-scrapers/core", () => ({
|
||||
fetchFacebookItems,
|
||||
fetchEbayItems,
|
||||
}));
|
||||
|
||||
import { ebayRoute } from "../src/routes/ebay";
|
||||
import { facebookRoute } from "../src/routes/facebook";
|
||||
|
||||
afterEach(() => {
|
||||
fetchFacebookItems.mockReset();
|
||||
fetchEbayItems.mockReset();
|
||||
});
|
||||
|
||||
test("facebookRoute ignores cookies query parameter", async () => {
|
||||
await facebookRoute(
|
||||
new Request("http://localhost/api/facebook?q=laptop&location=toronto&maxItems=3&cookies=c_user=1"),
|
||||
);
|
||||
|
||||
expect(fetchFacebookItems).toHaveBeenCalledWith("laptop", 1, "toronto", 3);
|
||||
});
|
||||
|
||||
test("ebayRoute ignores cookies query parameter", async () => {
|
||||
await ebayRoute(
|
||||
new Request("http://localhost/api/ebay?q=laptop&cookies=s%3D1&buyItNowOnly=true"),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
||||
minPrice: undefined,
|
||||
maxPrice: undefined,
|
||||
strictMode: false,
|
||||
exclusions: [],
|
||||
keywords: ["laptop"],
|
||||
buyItNowOnly: true,
|
||||
canadaOnly: true,
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `bun test packages/api-server/test/routes.test.ts`
|
||||
Expected: FAIL because the current routes still parse `reqUrl.searchParams.get("cookies")` and forward it downstream.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Delete the cookie query-parameter handling from both routes:
|
||||
|
||||
```ts
|
||||
// packages/api-server/src/routes/facebook.ts
|
||||
/**
|
||||
* GET /api/facebook?q={query}&location={location}
|
||||
* Search Facebook Marketplace for listings
|
||||
*/
|
||||
|
||||
const items = await fetchFacebookItems(SEARCH_QUERY, 1, LOCATION, maxItems);
|
||||
```
|
||||
|
||||
```ts
|
||||
// packages/api-server/src/routes/ebay.ts
|
||||
/**
|
||||
* GET /api/ebay?q={query}&minPrice={minPrice}&maxPrice={maxPrice}&strictMode={strictMode}&exclusions={exclusions}&keywords={keywords}&buyItNowOnly={buyItNowOnly}&canadaOnly={canadaOnly}
|
||||
*/
|
||||
|
||||
const items = await fetchEbayItems(SEARCH_QUERY, 1, {
|
||||
minPrice,
|
||||
maxPrice,
|
||||
strictMode,
|
||||
exclusions,
|
||||
keywords,
|
||||
buyItNowOnly,
|
||||
canadaOnly,
|
||||
});
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `bun test packages/api-server/test/routes.test.ts`
|
||||
Expected: PASS for route coverage and no remaining adapter references to `cookies` for Facebook/eBay.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/api-server/src/routes/facebook.ts packages/api-server/src/routes/ebay.ts packages/api-server/test/routes.test.ts
|
||||
git commit -m "refactor: remove api cookie query overrides"
|
||||
```
|
||||
|
||||
### Task 5: Remove cookie inputs from MCP tool schemas and request mapping
|
||||
|
||||
**Files:**
|
||||
- Modify: `packages/mcp-server/src/protocol/tools.ts:65-148`
|
||||
- Modify: `packages/mcp-server/src/protocol/handler.ts:154-211`
|
||||
- Create: `packages/mcp-server/test/protocol.test.ts`
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Create `packages/mcp-server/test/protocol.test.ts` with schema and URL-building assertions:
|
||||
|
||||
```ts
|
||||
import { expect, mock, test } from "bun:test";
|
||||
|
||||
import { TOOLS } from "../src/protocol/tools";
|
||||
import { handleJsonRpcRequest } from "../src/protocol/handler";
|
||||
|
||||
const searchFacebookTool = TOOLS.find((tool) => tool.name === "search_facebook");
|
||||
const searchEbayTool = TOOLS.find((tool) => tool.name === "search_ebay");
|
||||
|
||||
expect(searchFacebookTool.inputSchema.properties).not.toHaveProperty("cookiesSource");
|
||||
expect(searchEbayTool.inputSchema.properties).not.toHaveProperty("cookies");
|
||||
```
|
||||
|
||||
And handler URL construction should omit cookie params:
|
||||
|
||||
```ts
|
||||
const fetchMock = mock(() =>
|
||||
Promise.resolve(new Response(JSON.stringify([]), { status: 200 })),
|
||||
);
|
||||
|
||||
global.fetch = fetchMock as typeof fetch;
|
||||
|
||||
await handleJsonRpcRequest(
|
||||
new Request("http://localhost", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: 1,
|
||||
method: "tools/call",
|
||||
params: { name: "search_facebook", arguments: { query: "laptop" } },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const calledUrl = fetchMock.mock.calls[0]?.[0] as string;
|
||||
expect(calledUrl).toContain("/facebook?q=laptop");
|
||||
expect(calledUrl).not.toContain("cookies=");
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run test to verify it fails**
|
||||
|
||||
Run: `bun test packages/mcp-server/test/protocol.test.ts`
|
||||
Expected: FAIL because the current MCP schema and handler still expose and forward those inputs.
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Delete the Facebook/eBay cookie tool properties and handler mapping:
|
||||
|
||||
```ts
|
||||
// tools.ts
|
||||
// Remove `cookiesSource` from search_facebook
|
||||
// Remove `cookies` from search_ebay
|
||||
```
|
||||
|
||||
```ts
|
||||
// handler.ts
|
||||
// Remove:
|
||||
// if (args.cookiesSource) params.append("cookies", args.cookiesSource);
|
||||
// if (args.cookies) params.append("cookies", args.cookies);
|
||||
```
|
||||
|
||||
Leave Kijiji alone; this plan only changes Facebook/eBay env-only auth paths defined by the approved spec.
|
||||
|
||||
- [ ] **Step 4: Run test to verify it passes**
|
||||
|
||||
Run: `bun test packages/mcp-server/test/protocol.test.ts`
|
||||
Expected: PASS with MCP definitions and handler mapping in sync.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add packages/mcp-server/src/protocol/tools.ts packages/mcp-server/src/protocol/handler.ts packages/mcp-server/test/protocol.test.ts
|
||||
git commit -m "refactor: remove mcp cookie parameters"
|
||||
```
|
||||
|
||||
### Task 6: Rewrite cookie documentation and run full verification
|
||||
|
||||
**Files:**
|
||||
- Modify: `cookies/AGENTS.md:9-85`
|
||||
- Modify: `docs/superpowers/specs/2026-04-21-cookie-env-only-design.md` only if implementation reveals a spec mismatch
|
||||
|
||||
- [ ] **Step 1: Write the failing test**
|
||||
|
||||
Treat docs drift as a contract failure. Capture the required state before editing:
|
||||
|
||||
```md
|
||||
- Cookie setup docs mention env vars only for Facebook and eBay
|
||||
- No examples remain that show `cookies=` request params
|
||||
- No examples remain that show `facebook.json` or `ebay.json`
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run verification to prove current docs are stale**
|
||||
|
||||
Run: `rg -n "facebook\.json|ebay\.json|cookies=" cookies/AGENTS.md`
|
||||
Expected: matches found
|
||||
|
||||
- [ ] **Step 3: Write minimal implementation**
|
||||
|
||||
Rewrite the cookie setup doc so Facebook and eBay each show only env-var setup:
|
||||
|
||||
```md
|
||||
## Cookie Configuration
|
||||
|
||||
All supported authenticated scrapers read cookies only from environment variables.
|
||||
|
||||
### Facebook Marketplace
|
||||
|
||||
```bash
|
||||
export FACEBOOK_COOKIE='c_user=123; xs=token; fr=request'
|
||||
```
|
||||
|
||||
### eBay
|
||||
|
||||
```bash
|
||||
export EBAY_COOKIE='s=VALUE; ds2=VALUE; ebay=VALUE'
|
||||
```
|
||||
```
|
||||
|
||||
Remove the file-based and request-parameter sections entirely.
|
||||
|
||||
- [ ] **Step 4: Run full verification**
|
||||
|
||||
Run: `bun test && bun run ci && bun run build`
|
||||
Expected: all commands pass
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add cookies/AGENTS.md docs/superpowers/specs/2026-04-21-cookie-env-only-design.md
|
||||
git commit -m "docs: align cookie setup with env-only auth"
|
||||
```
|
||||
|
||||
## Self-Review
|
||||
|
||||
- Spec coverage check: shared cookie utils, Facebook, eBay, API adapter, MCP adapter, tests, and docs each have explicit tasks.
|
||||
- Placeholder scan: concrete test files are now named for eBay core, API routes, and MCP protocol coverage.
|
||||
- Type consistency check: `ensureCookies(config)` is the single shared loader name used across Tasks 1-3, and Facebook/eBay route signatures stay aligned with the core changes.
|
||||
131
docs/superpowers/specs/2026-04-21-cookie-env-only-design.md
Normal file
131
docs/superpowers/specs/2026-04-21-cookie-env-only-design.md
Normal file
@@ -0,0 +1,131 @@
|
||||
# Cookie Env-Only Design
|
||||
|
||||
## Summary
|
||||
|
||||
Remove all file-based and request-provided cookie inputs across the repo.
|
||||
The only supported authentication input becomes a raw `Cookie` header string supplied through scraper-specific environment variables such as `FACEBOOK_COOKIE` and `EBAY_COOKIE`.
|
||||
|
||||
## Goals
|
||||
|
||||
- Remove cookie file fallback from shared and marketplace-specific code.
|
||||
- Remove request-level cookie overrides from public scraper entrypoints.
|
||||
- Remove deprecated cookie-path parameters from Facebook APIs.
|
||||
- Keep cookie parsing deterministic and limited to raw header-string input.
|
||||
- Update tests and docs so the public contract matches the implementation.
|
||||
|
||||
## Non-Goals
|
||||
|
||||
- Changing scraper behavior unrelated to authentication input.
|
||||
- Adding new cookie formats or migration helpers.
|
||||
- Preserving backward compatibility for cookie files, JSON cookie arrays, or request overrides.
|
||||
|
||||
## Current State
|
||||
|
||||
The current shared cookie utilities support three sources in priority order:
|
||||
|
||||
1. Request parameter
|
||||
2. Environment variable
|
||||
3. Cookie file
|
||||
|
||||
`packages/core/src/utils/cookies.ts` includes file loading, JSON array parsing, and auto-detection between JSON and header-string formats.
|
||||
Facebook also exposes deprecated `cookiePath` arguments that still reach shared loading logic.
|
||||
Docs in `cookies/AGENTS.md` still describe file-based setup and request-level overrides.
|
||||
|
||||
## Chosen Approach
|
||||
|
||||
Use the hard-reset approach.
|
||||
Delete the shared multi-source cookie-loading model and reduce the cookie surface to env-header parsing only.
|
||||
This is a larger diff than a surgical removal, but it avoids leaving behind abstractions that imply unsupported inputs still exist.
|
||||
|
||||
## Design
|
||||
|
||||
### Shared Cookie Utilities
|
||||
|
||||
`packages/core/src/utils/cookies.ts` will keep only the pieces needed for env-header-based auth:
|
||||
|
||||
- `Cookie` type
|
||||
- A reduced cookie config shape containing only `name`, `domain`, and `envVar`
|
||||
- `parseCookieString()` for raw `Cookie` header strings
|
||||
- `formatCookiesForHeader()` for domain filtering and request formatting
|
||||
- An env-only loader that reads `process.env[config.envVar]`, parses it, and throws a targeted error when missing or invalid
|
||||
|
||||
The following shared utilities will be removed:
|
||||
|
||||
- JSON cookie-array parsing
|
||||
- Auto-detection between JSON and header-string formats
|
||||
- File loading helpers
|
||||
- Optional loaders whose behavior depends on file fallback or request input
|
||||
|
||||
### Marketplace Scrapers
|
||||
|
||||
Marketplace scrapers that require auth will read cookies only from their env vars.
|
||||
|
||||
For Facebook this means:
|
||||
|
||||
- Remove `_cookiePath` / `cookiePath` parameters from helper and public functions
|
||||
- Remove any docs/comments that mention parameter > env > file precedence
|
||||
- Update auth failure messaging to name only `FACEBOOK_COOKIE`
|
||||
|
||||
For eBay this means:
|
||||
|
||||
- Remove any remaining fallback/file-oriented behavior from shared calls and error strings
|
||||
- Keep the existing env-var auth path, but make it the only path
|
||||
|
||||
### Public API Surface
|
||||
|
||||
Exports from `packages/core/src/index.ts` should reflect the new contract.
|
||||
If exported functions currently advertise cookie-source or cookie-path arguments, their signatures will be tightened so callers cannot pass unsupported inputs.
|
||||
|
||||
Downstream adapter packages should continue calling core through the simplified signatures without adding their own cookie-loading behavior.
|
||||
|
||||
### Error Handling
|
||||
|
||||
There are now only two auth failure modes:
|
||||
|
||||
1. The required env var is missing or empty.
|
||||
2. The env var does not contain any valid `name=value` cookie pairs.
|
||||
|
||||
Errors should be blunt and specific:
|
||||
|
||||
- identify the missing env var by name
|
||||
- state that the value must be a raw `Cookie` header string
|
||||
- stop mentioning request parameters, cookie paths, JSON arrays, or `./cookies/*.json`
|
||||
|
||||
### Testing Strategy
|
||||
|
||||
Follow TDD.
|
||||
Start by changing or adding core tests so the old file/request behavior is no longer accepted.
|
||||
|
||||
Coverage targets:
|
||||
|
||||
1. Valid env header strings still parse into cookies correctly.
|
||||
2. Missing env vars fail with the new env-only error.
|
||||
3. Invalid env strings fail without falling back to files or request data.
|
||||
4. Facebook APIs no longer expose or honor cookie-path/request-cookie behavior.
|
||||
5. Existing tests that depended on missing files or JSON cookie arrays are rewritten to the env-only contract.
|
||||
|
||||
Verification target after implementation:
|
||||
|
||||
- `bun test packages/core/test`
|
||||
- `bun run ci`
|
||||
- `bun run build` if any cross-package signature changes require downstream verification
|
||||
|
||||
## Documentation Changes
|
||||
|
||||
Update cookie-related docs to match the new contract:
|
||||
|
||||
- remove file-based setup instructions
|
||||
- remove request-parameter cookie examples
|
||||
- document env vars as the only supported auth input
|
||||
- show raw `Cookie` header-string examples only
|
||||
|
||||
## Risks
|
||||
|
||||
- External callers using request cookie overrides will break at compile time or runtime, depending on how they consume the package.
|
||||
- Recent work added support for custom Facebook cookie paths, so removing that path intentionally reverses a newly introduced behavior.
|
||||
- Tests that currently model missing-file behavior must be rewritten rather than preserved.
|
||||
|
||||
## Rollout Notes
|
||||
|
||||
This is an intentional contract break.
|
||||
The code, tests, and docs should all land together so there is no mixed messaging about supported cookie sources.
|
||||
@@ -11,9 +11,13 @@ mock.module("@marketplace-scrapers/core", () => ({
|
||||
describe("API routes", () => {
|
||||
beforeEach(() => {
|
||||
fetchFacebookItems.mockReset();
|
||||
fetchFacebookItems.mockImplementation(() => Promise.resolve([{ title: "item" }]));
|
||||
fetchFacebookItems.mockImplementation(() =>
|
||||
Promise.resolve([{ title: "item" }]),
|
||||
);
|
||||
fetchEbayItems.mockReset();
|
||||
fetchEbayItems.mockImplementation(() => Promise.resolve([{ title: "item" }]));
|
||||
fetchEbayItems.mockImplementation(() =>
|
||||
Promise.resolve([{ title: "item" }]),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -37,7 +41,9 @@ describe("API routes", () => {
|
||||
const { ebayRoute } = await import("../src/routes/ebay");
|
||||
|
||||
await ebayRoute(
|
||||
new Request("http://localhost/api/ebay?q=laptop&cookies=s%3D1&buyItNowOnly=true"),
|
||||
new Request(
|
||||
"http://localhost/api/ebay?q=laptop&cookies=s%3D1&buyItNowOnly=true",
|
||||
),
|
||||
);
|
||||
|
||||
expect(fetchEbayItems).toHaveBeenCalledWith("laptop", 1, {
|
||||
|
||||
@@ -93,7 +93,10 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
try {
|
||||
const cookies = await ensureFacebookCookies();
|
||||
|
||||
expect(cookies.map((cookie) => cookie.name)).toEqual(["c_user", "xs"]);
|
||||
expect(cookies.map((cookie) => cookie.name)).toEqual([
|
||||
"c_user",
|
||||
"xs",
|
||||
]);
|
||||
} finally {
|
||||
if (previous === undefined) {
|
||||
delete process.env.FACEBOOK_COOKIE;
|
||||
|
||||
@@ -3,24 +3,29 @@ import fetchFacebookItems from "../src/scrapers/facebook";
|
||||
|
||||
// Mock fetch globally
|
||||
const originalFetch = global.fetch;
|
||||
const facebookCookie = "c_user=12345; xs=abc123";
|
||||
|
||||
describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
let previousCookie: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
previousCookie = process.env.FACEBOOK_COOKIE;
|
||||
process.env.FACEBOOK_COOKIE = facebookCookie;
|
||||
global.fetch = mock(() => {
|
||||
throw new Error("fetch should be mocked in individual tests");
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (previousCookie === undefined) {
|
||||
delete process.env.FACEBOOK_COOKIE;
|
||||
} else {
|
||||
process.env.FACEBOOK_COOKIE = previousCookie;
|
||||
}
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
describe("Main Search Function", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should successfully fetch search results", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
@@ -99,13 +104,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"iPhone",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("iPhone", 1, "toronto", 25);
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].title).toBe("iPhone 13 Pro");
|
||||
expect(results[1].title).toBe("Samsung Galaxy");
|
||||
@@ -172,13 +171,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("With Price");
|
||||
});
|
||||
@@ -233,13 +226,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
5,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 5);
|
||||
expect(results).toHaveLength(5);
|
||||
});
|
||||
|
||||
@@ -285,7 +272,6 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
@@ -302,13 +288,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -316,7 +296,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
global.fetch = mock(() => Promise.reject(new Error("Network error")));
|
||||
|
||||
await expect(
|
||||
fetchFacebookItems("test", 1, "toronto", 25, mockCookies),
|
||||
fetchFacebookItems("test", 1, "toronto", 25),
|
||||
).rejects.toThrow("Network error");
|
||||
});
|
||||
|
||||
@@ -386,24 +366,13 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||
expect(attempts).toBe(2);
|
||||
expect(results).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Vehicle Listing Integration", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should correctly identify and parse vehicle listings", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
@@ -470,13 +439,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"cars",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("cars", 1, "toronto", 25);
|
||||
expect(results).toHaveLength(2);
|
||||
// Both should be classified as "item" type in search results (vehicle detection is for item details)
|
||||
expect(results[0].title).toBe("2006 Honda Civic");
|
||||
@@ -485,11 +448,6 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
});
|
||||
|
||||
describe("Different Categories", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should handle electronics listings", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
@@ -555,7 +513,6 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("Nintendo Switch");
|
||||
@@ -622,13 +579,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"table",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("table", 1, "toronto", 25);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("Dining Table");
|
||||
expect(results[0].categoryId).toBe("1569171756675761");
|
||||
@@ -636,11 +587,6 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
});
|
||||
|
||||
describe("Error Scenarios", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should handle malformed HTML responses", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
@@ -655,13 +601,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -677,13 +617,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -699,13 +633,7 @@ describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
const results = await fetchFacebookItems("test", 1, "toronto", 25);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,13 +16,17 @@ describe("MCP protocol cookie inputs", () => {
|
||||
});
|
||||
|
||||
test("search tools should not expose Facebook or eBay cookie inputs", () => {
|
||||
const searchFacebookTool = tools.find((tool) => tool.name === "search_facebook");
|
||||
const searchFacebookTool = tools.find(
|
||||
(tool) => tool.name === "search_facebook",
|
||||
);
|
||||
const searchEbayTool = tools.find((tool) => tool.name === "search_ebay");
|
||||
|
||||
expect(searchFacebookTool?.inputSchema.properties).not.toHaveProperty(
|
||||
"cookiesSource",
|
||||
);
|
||||
expect(searchEbayTool?.inputSchema.properties).not.toHaveProperty("cookies");
|
||||
expect(searchEbayTool?.inputSchema.properties).not.toHaveProperty(
|
||||
"cookies",
|
||||
);
|
||||
});
|
||||
|
||||
test("search_facebook should not forward cookies query parameters", async () => {
|
||||
@@ -44,7 +48,8 @@ describe("MCP protocol cookie inputs", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock.calls[0]?.[0];
|
||||
const calledUrl = (global.fetch as ReturnType<typeof mock>).mock
|
||||
.calls[0]?.[0];
|
||||
expect(String(calledUrl)).toContain("/facebook?q=laptop");
|
||||
expect(String(calledUrl)).not.toContain("cookies=");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user