old4ever/ca-marketplace-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make cookie loading env-only

Browse Source
This commit is contained in:
Dmytro Stanchiev
2026-04-21 21:44:12 -04:00
parent 7b4b656868
commit 847ce28590
2 changed files with 58 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
packages/core/src/utils/cookies.ts
View File

@@ -23,8 +23,6 @@ export interface CookieConfig {
domain: string;
/** Environment variable name (e.g., "FACEBOOK_COOKIE") */
envVar: string;
/** Path to cookie file (e.g., "./cookies/facebook.json") */
filePath: string;
}
/**
@@ -66,61 +64,6 @@ export function parseCookieString(
.filter((cookie): cookie is Cookie => cookie !== null);
}
/**
* Parse JSON array format into Cookie array
* Supports format: [{"name": "foo", "value": "bar", ...}]
*/
export function parseJsonCookies(jsonString: string): Cookie[] {
const parsed = JSON.parse(jsonString);
if (!Array.isArray(parsed)) {
return [];
}
return parsed.filter(
(cookie): cookie is Cookie =>
cookie &&
typeof cookie.name === "string" &&
typeof cookie.value === "string",
);
}
/**
* Try to parse cookies from a string (tries JSON first, then cookie string format)
*/
export function parseCookiesAuto(
input: string,
defaultDomain: string,
): Cookie[] {
// Try JSON array format first
try {
const cookies = parseJsonCookies(input);
if (cookies.length > 0) {
return cookies;
}
} catch {
// JSON parse failed, try cookie string format
}
// Try cookie string format
return parseCookieString(input, defaultDomain);
}
/**
* Load cookies from file (supports both JSON array and cookie string formats)
*/
export async function loadCookiesFromFile(
filePath: string,
defaultDomain: string,
): Promise<Cookie[]> {
const file = Bun.file(filePath);
if (!(await file.exists())) {
return [];
}
const content = await file.text();
return parseCookiesAuto(content.trim(), defaultDomain);
}
/**
* Format cookies array into Cookie header string for HTTP requests
*/
@@ -155,60 +98,21 @@ export function formatCookiesForHeader(
}
/**
* Load cookies with priority: URL param > ENV var > file
* Supports both JSON array and cookie string formats for all sources
* Load cookies from the configured environment variable
*/
export async function ensureCookies(
config: CookieConfig,
cookiesSource?: string,
): Promise<Cookie[]> {
// Priority 1: URL/API parameter (if provided)
if (cookiesSource) {
const cookies = parseCookiesAuto(cookiesSource, config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from parameter`,
);
return cookies;
}
console.warn(
`${config.name} cookies parameter provided but no valid cookies extracted`,
);
}
// Priority 2: Environment variable
export async function ensureCookies(config: CookieConfig): Promise<Cookie[]> {
const envValue = process.env[config.envVar];
if (envValue?.trim()) {
const cookies = parseCookiesAuto(envValue, config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
);
return cookies;
}
console.warn(`${config.envVar} env var contains no valid cookies`);
const cookies = parseCookieString(envValue ?? "", config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.envVar} env var`,
);
return cookies;
}
// Priority 3: Cookie file (fallback)
try {
const cookies = await loadCookiesFromFile(config.filePath, config.domain);
if (cookies.length > 0) {
console.log(
`Loaded ${cookies.length} ${config.name} cookies from ${config.filePath}`,
);
return cookies;
}
} catch (e) {
console.warn(`Could not load cookies from ${config.filePath}: ${e}`);
}
// No cookies found from any source
throw new Error(
`No valid ${config.name} cookies found. Provide cookies via (in priority order):\n` +
` 1. 'cookies' parameter (highest priority), or\n` +
` 2. ${config.envVar} environment variable, or\n` +
` 3. ${config.filePath} file (lowest priority)\n` +
'Format: JSON array or cookie string like "name1=value1; name2=value2"',
`No valid ${config.name} cookies found. Provide cookies via ${config.envVar} environment variable as a raw Cookie header string.`,
);
}

62
packages/core/test/facebook-core.test.ts
View File

@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import {
ensureFacebookCookies,
extractFacebookItemData,
extractFacebookMarketplaceData,
fetchFacebookItem,
@@ -84,15 +85,58 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
expect(result[1].name).toBe("xs");
expect(result[1].value).toBe("abc");
});
test("should load Facebook cookies from FACEBOOK_COOKIE env var", async () => {
const previous = process.env.FACEBOOK_COOKIE;
process.env.FACEBOOK_COOKIE = "c_user=123; xs=abc";
try {
const cookies = await ensureFacebookCookies();
expect(cookies.map((cookie) => cookie.name)).toEqual(["c_user", "xs"]);
} finally {
if (previous === undefined) {
delete process.env.FACEBOOK_COOKIE;
} else {
process.env.FACEBOOK_COOKIE = previous;
}
}
});
test("should reject missing Facebook auth env var", async () => {
const previous = process.env.FACEBOOK_COOKIE;
delete process.env.FACEBOOK_COOKIE;
try {
await expect(ensureFacebookCookies()).rejects.toThrow(
"Provide cookies via FACEBOOK_COOKIE environment variable as a raw Cookie header string",
);
} finally {
if (previous !== undefined) {
process.env.FACEBOOK_COOKIE = previous;
}
}
});
});
});
describe("Facebook Item Fetching", () => {
describe("fetchFacebookItem", () => {
const mockCookies = JSON.stringify([
{ name: "c_user", value: "12345", domain: ".facebook.com" },
{ name: "xs", value: "abc123", domain: ".facebook.com" },
]);
const mockCookies = "c_user=12345; xs=abc123";
let previousCookie: string | undefined;
beforeEach(() => {
previousCookie = process.env.FACEBOOK_COOKIE;
process.env.FACEBOOK_COOKIE = mockCookies;
});
afterEach(() => {
if (previousCookie === undefined) {
delete process.env.FACEBOOK_COOKIE;
} else {
process.env.FACEBOOK_COOKIE = previousCookie;
}
});
test("should handle authentication errors", async () => {
global.fetch = mock(() =>
@@ -234,16 +278,6 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
expect(result?.listingStatus).toBe("SOLD");
});
test("should handle missing authentication cookies", async () => {
// Use a test-specific cookie file that doesn't exist
const testCookiePath = "./cookies/facebook-test.json";
// Test with no cookies available (test file doesn't exist)
await expect(
fetchFacebookItem("123", undefined, testCookiePath),
).rejects.toThrow("No valid Facebook cookies found");
});
test("should handle successful item extraction", async () => {
const mockData = {
require: [