feat: port upstream scraper improvements to monorepo
Kijiji improvements: - Add error classes: NetworkError, ParseError, RateLimitError, ValidationError - Add exponential backoff with jitter for retries - Add request timeout (30s abort) - Add pagination support (SearchOptions.maxPages) - Add location/category mappings and resolution functions - Add enhanced DetailedListing interface with images, seller info, attributes - Add GraphQL client for seller details Facebook improvements: - Add parseFacebookCookieString() for parsing cookie strings - Add ensureFacebookCookies() with env var fallback - Add extractFacebookItemData() with multiple extraction paths - Add fetchFacebookItem() for individual item fetching - Add extraction metrics and API stability monitoring - Add vehicle-specific field extraction - Improve error handling with specific guidance for auth errors Shared utilities: - Update http.ts with new error classes and improved fetchHtml Documentation: - Port KIJIJI.md, FMARKETPLACE.md, AGENTS.md from upstream Tests: - Port kijiji-core, kijiji-integration, kijiji-utils tests - Port facebook-core, facebook-integration tests - Add test setup file Scripts: - Port parse-facebook-cookies.ts script Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
834
packages/core/test/facebook-core.test.ts
Normal file
834
packages/core/test/facebook-core.test.ts
Normal file
@@ -0,0 +1,834 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import {
|
||||
extractFacebookItemData,
|
||||
extractFacebookMarketplaceData,
|
||||
fetchFacebookItem,
|
||||
formatCentsToCurrency,
|
||||
formatCookiesForHeader,
|
||||
loadFacebookCookies,
|
||||
parseFacebookAds,
|
||||
parseFacebookCookieString,
|
||||
parseFacebookItem,
|
||||
} from "../src/scrapers/facebook";
|
||||
|
||||
// Mock fetch globally
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
describe("Facebook Marketplace Scraper Core Tests", () => {
|
||||
beforeEach(() => {
|
||||
global.fetch = mock(() => {
|
||||
throw new Error("fetch should be mocked in individual tests");
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
describe("Cookie Parsing", () => {
|
||||
describe("parseFacebookCookieString", () => {
|
||||
test("should parse valid cookie string", () => {
|
||||
const cookieString = "c_user=123456789; xs=abcdef123456; fr=xyz789";
|
||||
const result = parseFacebookCookieString(cookieString);
|
||||
|
||||
expect(result).toHaveLength(3);
|
||||
expect(result[0]).toEqual({
|
||||
name: "c_user",
|
||||
value: "123456789",
|
||||
domain: ".facebook.com",
|
||||
path: "/",
|
||||
secure: true,
|
||||
httpOnly: false,
|
||||
sameSite: "lax",
|
||||
expirationDate: undefined,
|
||||
});
|
||||
expect(result[1]).toEqual({
|
||||
name: "xs",
|
||||
value: "abcdef123456",
|
||||
domain: ".facebook.com",
|
||||
path: "/",
|
||||
secure: true,
|
||||
httpOnly: false,
|
||||
sameSite: "lax",
|
||||
expirationDate: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
test("should handle URL-encoded values", () => {
|
||||
const cookieString = "c_user=123%2B456; xs=abc%3Ddef";
|
||||
const result = parseFacebookCookieString(cookieString);
|
||||
|
||||
expect(result[0].value).toBe("123+456");
|
||||
expect(result[1].value).toBe("abc=def");
|
||||
});
|
||||
|
||||
test("should filter out malformed cookies", () => {
|
||||
const cookieString = "c_user=123; invalid; xs=abc; =empty";
|
||||
const result = parseFacebookCookieString(cookieString);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.map((c) => c.name)).toEqual(["c_user", "xs"]);
|
||||
});
|
||||
|
||||
test("should handle empty input", () => {
|
||||
expect(parseFacebookCookieString("")).toEqual([]);
|
||||
expect(parseFacebookCookieString(" ")).toEqual([]);
|
||||
});
|
||||
|
||||
test("should handle extra whitespace", () => {
|
||||
const cookieString = " c_user = 123 ; xs=abc ";
|
||||
const result = parseFacebookCookieString(cookieString);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].name).toBe("c_user");
|
||||
expect(result[0].value).toBe("123");
|
||||
expect(result[1].name).toBe("xs");
|
||||
expect(result[1].value).toBe("abc");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Facebook Item Fetching", () => {
|
||||
describe("fetchFacebookItem", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com" },
|
||||
]);
|
||||
|
||||
test("should handle authentication errors", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 401,
|
||||
text: () => Promise.resolve("Authentication required"),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await fetchFacebookItem("123", mockCookies);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should handle item not found", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 404,
|
||||
text: () => Promise.resolve("Not found"),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await fetchFacebookItem("nonexistent", mockCookies);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should handle rate limiting", async () => {
|
||||
let attempts = 0;
|
||||
global.fetch = mock(() => {
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
return Promise.resolve({
|
||||
ok: false,
|
||||
status: 429,
|
||||
headers: {
|
||||
get: (header: string) => {
|
||||
if (header === "X-RateLimit-Reset") return "1";
|
||||
return null;
|
||||
},
|
||||
},
|
||||
text: () => Promise.resolve("Rate limited"),
|
||||
});
|
||||
}
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
viewer: {
|
||||
marketplace_product_details_page: {
|
||||
target: {
|
||||
id: "123",
|
||||
__typename: "GroupCommerceProductItem",
|
||||
marketplace_listing_title: "Test Item",
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
const result = await fetchFacebookItem("123", mockCookies);
|
||||
expect(attempts).toBe(2);
|
||||
// Should eventually succeed after retry
|
||||
});
|
||||
|
||||
test("should handle sold items", async () => {
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
viewer: {
|
||||
marketplace_product_details_page: {
|
||||
target: {
|
||||
id: "456",
|
||||
__typename: "GroupCommerceProductItem",
|
||||
marketplace_listing_title: "Sold Item",
|
||||
is_sold: true,
|
||||
is_live: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await fetchFacebookItem("456", mockCookies);
|
||||
expect(result?.listingStatus).toBe("SOLD");
|
||||
});
|
||||
|
||||
test("should handle missing authentication cookies", async () => {
|
||||
// Use a test-specific cookie file that doesn't exist
|
||||
const testCookiePath = "./cookies/facebook-test.json";
|
||||
|
||||
// Test with no cookies available (test file doesn't exist)
|
||||
await expect(
|
||||
fetchFacebookItem("123", undefined, testCookiePath),
|
||||
).rejects.toThrow("No valid Facebook cookies found");
|
||||
});
|
||||
|
||||
test("should handle successful item extraction", async () => {
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
viewer: {
|
||||
marketplace_product_details_page: {
|
||||
target: {
|
||||
id: "789",
|
||||
__typename: "GroupCommerceProductItem",
|
||||
marketplace_listing_title: "Working Item",
|
||||
formatted_price: { text: "$299.00" },
|
||||
listing_price: {
|
||||
amount: "299.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
creation_time: 1640995200,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await fetchFacebookItem("789", mockCookies);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.title).toBe("Working Item");
|
||||
expect(result?.listingPrice?.amountFormatted).toBe("$299.00");
|
||||
expect(result?.listingStatus).toBe("ACTIVE");
|
||||
});
|
||||
|
||||
test("should handle server errors", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: () => Promise.resolve("Internal Server Error"),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await fetchFacebookItem("error", mockCookies);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Data Extraction", () => {
|
||||
describe("extractFacebookItemData", () => {
|
||||
test("should extract item data from standard require structure", () => {
|
||||
const mockItemData = {
|
||||
id: "123456",
|
||||
__typename: "GroupCommerceProductItem",
|
||||
marketplace_listing_title: "Test Item",
|
||||
formatted_price: { text: "$100.00" },
|
||||
listing_price: { amount: "100.00", currency: "CAD" },
|
||||
is_live: true,
|
||||
};
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
viewer: {
|
||||
marketplace_product_details_page: {
|
||||
target: mockItemData,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||
|
||||
const result = extractFacebookItemData(html);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.id).toBe("123456");
|
||||
expect(result?.marketplace_listing_title).toBe("Test Item");
|
||||
});
|
||||
|
||||
test("should handle missing item data", () => {
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
viewer: {
|
||||
marketplace_product_details_page: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||
|
||||
const result = extractFacebookItemData(html);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should handle malformed HTML", () => {
|
||||
const result = extractFacebookItemData(
|
||||
"<html><body>Invalid HTML</body></html>",
|
||||
);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should handle invalid JSON in script tags", () => {
|
||||
const html =
|
||||
"<html><body><script>{invalid: json}</script></body></html>";
|
||||
const result = extractFacebookItemData(html);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should extract item with vehicle data", () => {
|
||||
const mockVehicleItem = {
|
||||
id: "789",
|
||||
__typename: "GroupCommerceProductItem",
|
||||
marketplace_listing_title: "2006 Honda Civic",
|
||||
formatted_price: { text: "$5,000" },
|
||||
listing_price: { amount: "5000.00", currency: "CAD" },
|
||||
vehicle_make_display_name: "Honda",
|
||||
vehicle_model_display_name: "Civic",
|
||||
vehicle_odometer_data: { unit: "KILOMETERS", value: 150000 },
|
||||
vehicle_transmission_type: "AUTOMATIC",
|
||||
is_live: true,
|
||||
};
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
viewer: {
|
||||
marketplace_product_details_page: {
|
||||
target: mockVehicleItem,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||
|
||||
const result = extractFacebookItemData(html);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.vehicle_make_display_name).toBe("Honda");
|
||||
expect(result?.vehicle_odometer_data?.value).toBe(150000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractFacebookMarketplaceData", () => {
|
||||
test("should extract search results from marketplace data", () => {
|
||||
const mockMarketplaceData = {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "Item 1",
|
||||
listing_price: { amount: "10.00", currency: "CAD" },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "Item 2",
|
||||
listing_price: { amount: "20.00", currency: "CAD" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: mockMarketplaceData,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||
|
||||
const result = extractFacebookMarketplaceData(html);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result?.[0].node.listing.marketplace_listing_title).toBe(
|
||||
"Item 1",
|
||||
);
|
||||
});
|
||||
|
||||
test("should handle empty search results", () => {
|
||||
const mockData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: { edges: [] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
const html = `<html><body><script>${JSON.stringify(mockData)}</script></body></html>`;
|
||||
|
||||
const result = extractFacebookMarketplaceData(html);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Data Parsing", () => {
|
||||
describe("parseFacebookItem", () => {
|
||||
test("should parse complete item with all fields", () => {
|
||||
const item = {
|
||||
id: "123456",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "iPhone 13 Pro",
|
||||
redacted_description: { text: "Excellent condition" },
|
||||
formatted_price: { text: "$800.00" },
|
||||
listing_price: { amount: "800.00", currency: "CAD" },
|
||||
location_text: { text: "Toronto, ON" },
|
||||
is_live: true,
|
||||
creation_time: 1640995200,
|
||||
marketplace_listing_seller: {
|
||||
id: "seller1",
|
||||
name: "John Doe",
|
||||
},
|
||||
delivery_types: ["IN_PERSON"],
|
||||
};
|
||||
|
||||
const result = parseFacebookItem(item);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.title).toBe("iPhone 13 Pro");
|
||||
expect(result?.description).toBe("Excellent condition");
|
||||
expect(result?.listingPrice?.amountFormatted).toBe("$800.00");
|
||||
expect(result?.listingPrice?.cents).toBe(80000);
|
||||
expect(result?.listingPrice?.currency).toBe("CAD");
|
||||
expect(result?.address).toBe("Toronto, ON");
|
||||
expect(result?.listingStatus).toBe("ACTIVE");
|
||||
expect(result?.seller?.name).toBe("John Doe");
|
||||
expect(result?.deliveryTypes).toEqual(["IN_PERSON"]);
|
||||
});
|
||||
|
||||
test("should parse FREE items", () => {
|
||||
const item = {
|
||||
id: "789",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "Free Sofa",
|
||||
formatted_price: { text: "FREE" },
|
||||
listing_price: { amount: "0.00", currency: "CAD" },
|
||||
is_live: true,
|
||||
};
|
||||
|
||||
const result = parseFacebookItem(item);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.title).toBe("Free Sofa");
|
||||
expect(result?.listingPrice?.amountFormatted).toBe("FREE");
|
||||
expect(result?.listingPrice?.cents).toBe(0);
|
||||
});
|
||||
|
||||
test("should handle missing optional fields", () => {
|
||||
const item = {
|
||||
id: "456",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "Minimal Item",
|
||||
};
|
||||
|
||||
const result = parseFacebookItem(item);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.title).toBe("Minimal Item");
|
||||
expect(result?.description).toBeUndefined();
|
||||
expect(result?.seller).toBeUndefined();
|
||||
});
|
||||
|
||||
test("should identify vehicle listings", () => {
|
||||
const vehicleItem = {
|
||||
id: "999",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "2012 Mazda 3",
|
||||
formatted_price: { text: "$8,000" },
|
||||
listing_price: { amount: "8000.00", currency: "CAD" },
|
||||
vehicle_make_display_name: "Mazda",
|
||||
vehicle_model_display_name: "3",
|
||||
is_live: true,
|
||||
};
|
||||
|
||||
const result = parseFacebookItem(vehicleItem);
|
||||
expect(result?.listingType).toBe("vehicle");
|
||||
});
|
||||
|
||||
test("should handle different listing statuses", () => {
|
||||
const soldItem = {
|
||||
id: "111",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "Sold Item",
|
||||
is_sold: true,
|
||||
is_live: false,
|
||||
};
|
||||
|
||||
const pendingItem = {
|
||||
id: "222",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "Pending Item",
|
||||
is_pending: true,
|
||||
is_live: true,
|
||||
};
|
||||
|
||||
const hiddenItem = {
|
||||
id: "333",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
marketplace_listing_title: "Hidden Item",
|
||||
is_hidden: true,
|
||||
is_live: false,
|
||||
};
|
||||
|
||||
expect(parseFacebookItem(soldItem)?.listingStatus).toBe("SOLD");
|
||||
expect(parseFacebookItem(pendingItem)?.listingStatus).toBe("PENDING");
|
||||
expect(parseFacebookItem(hiddenItem)?.listingStatus).toBe("HIDDEN");
|
||||
});
|
||||
|
||||
test("should return null for items without title", () => {
|
||||
const invalidItem = {
|
||||
id: "invalid",
|
||||
__typename: "GroupCommerceProductItem" as const,
|
||||
is_live: true,
|
||||
};
|
||||
|
||||
const result = parseFacebookItem(invalidItem);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseFacebookAds", () => {
|
||||
test("should parse search result ads", () => {
|
||||
const ads = [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "Ad 1",
|
||||
listing_price: {
|
||||
amount: "50.00",
|
||||
formatted_amount: "$50.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
location: {
|
||||
reverse_geocode: { city_page: { display_name: "Toronto" } },
|
||||
},
|
||||
creation_time: 1640995200,
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "Ad 2",
|
||||
listing_price: {
|
||||
amount: "75.00",
|
||||
formatted_amount: "$75.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
location: {
|
||||
reverse_geocode: { city_page: { display_name: "Ottawa" } },
|
||||
},
|
||||
creation_time: 1640995300,
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const results = parseFacebookAds(ads);
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].title).toBe("Ad 1");
|
||||
expect(results[0].listingPrice?.cents).toBe(5000);
|
||||
expect(results[0].address).toBe("Toronto");
|
||||
expect(results[1].title).toBe("Ad 2");
|
||||
expect(results[1].address).toBe("Ottawa");
|
||||
});
|
||||
|
||||
test("should filter out ads without price", () => {
|
||||
const ads = [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "With Price",
|
||||
listing_price: {
|
||||
amount: "100.00",
|
||||
formatted_amount: "$100.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "No Price",
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const results = parseFacebookAds(ads);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("With Price");
|
||||
});
|
||||
|
||||
test("should handle malformed ads gracefully", () => {
|
||||
const ads = [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "Valid Ad",
|
||||
listing_price: {
|
||||
amount: "50.00",
|
||||
formatted_amount: "$50.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
// Missing listing
|
||||
},
|
||||
} as { node: { listing?: unknown } },
|
||||
];
|
||||
|
||||
const results = parseFacebookAds(ads);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("Valid Ad");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Utility Functions", () => {
|
||||
describe("formatCentsToCurrency", () => {
|
||||
test("should format cents to currency string", () => {
|
||||
expect(formatCentsToCurrency(100)).toBe("$1.00");
|
||||
expect(formatCentsToCurrency(1000)).toBe("$10.00");
|
||||
expect(formatCentsToCurrency(9999)).toBe("$99.99");
|
||||
expect(formatCentsToCurrency(123456)).toBe("$1,234.56");
|
||||
});
|
||||
|
||||
test("should handle string inputs", () => {
|
||||
expect(formatCentsToCurrency("100")).toBe("$1.00");
|
||||
expect(formatCentsToCurrency("1000")).toBe("$10.00");
|
||||
});
|
||||
|
||||
test("should handle zero", () => {
|
||||
expect(formatCentsToCurrency(0)).toBe("$0.00");
|
||||
});
|
||||
|
||||
test("should handle null and undefined", () => {
|
||||
expect(formatCentsToCurrency(null)).toBe("");
|
||||
expect(formatCentsToCurrency(undefined)).toBe("");
|
||||
});
|
||||
|
||||
test("should handle invalid inputs", () => {
|
||||
expect(formatCentsToCurrency("invalid")).toBe("");
|
||||
expect(formatCentsToCurrency(Number.NaN)).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatCookiesForHeader", () => {
|
||||
const mockCookies = [
|
||||
{ name: "c_user", value: "123456", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abcdef", domain: ".facebook.com", path: "/" },
|
||||
{ name: "session_id", value: "xyz", domain: "other.com", path: "/" },
|
||||
];
|
||||
|
||||
test("should format cookies for header string", () => {
|
||||
const result = formatCookiesForHeader(mockCookies, "www.facebook.com");
|
||||
expect(result).toBe("c_user=123456; xs=abcdef");
|
||||
});
|
||||
|
||||
test("should filter expired cookies", () => {
|
||||
const cookiesWithExpiration = [
|
||||
...mockCookies,
|
||||
{
|
||||
name: "expired",
|
||||
value: "old",
|
||||
domain: ".facebook.com",
|
||||
path: "/",
|
||||
expirationDate: Date.now() / 1000 - 1000,
|
||||
},
|
||||
];
|
||||
const result = formatCookiesForHeader(
|
||||
cookiesWithExpiration,
|
||||
"www.facebook.com",
|
||||
);
|
||||
expect(result).not.toContain("expired");
|
||||
});
|
||||
|
||||
test("should handle no matching cookies", () => {
|
||||
const result = formatCookiesForHeader(mockCookies, "www.google.com");
|
||||
expect(result).toBe("");
|
||||
});
|
||||
|
||||
test("should handle empty cookie array", () => {
|
||||
const result = formatCookiesForHeader([], "www.facebook.com");
|
||||
expect(result).toBe("");
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
712
packages/core/test/facebook-integration.test.ts
Normal file
712
packages/core/test/facebook-integration.test.ts
Normal file
@@ -0,0 +1,712 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import fetchFacebookItems, { fetchFacebookItem } from "../src/scrapers/facebook";
|
||||
|
||||
// Mock fetch globally
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
describe("Facebook Marketplace Scraper Integration Tests", () => {
|
||||
beforeEach(() => {
|
||||
global.fetch = mock(() => {
|
||||
throw new Error("fetch should be mocked in individual tests");
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
describe("Main Search Function", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should successfully fetch search results", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "iPhone 13 Pro",
|
||||
listing_price: {
|
||||
amount: "800.00",
|
||||
formatted_amount: "$800.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
location: {
|
||||
reverse_geocode: {
|
||||
city_page: { display_name: "Toronto" },
|
||||
},
|
||||
},
|
||||
creation_time: 1640995200,
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "Samsung Galaxy",
|
||||
listing_price: {
|
||||
amount: "600.00",
|
||||
formatted_amount: "$600.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
location: {
|
||||
reverse_geocode: {
|
||||
city_page: { display_name: "Mississauga" },
|
||||
},
|
||||
},
|
||||
creation_time: 1640995300,
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"iPhone",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].title).toBe("iPhone 13 Pro");
|
||||
expect(results[1].title).toBe("Samsung Galaxy");
|
||||
});
|
||||
|
||||
test("should filter out items without price", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "With Price",
|
||||
listing_price: {
|
||||
amount: "100.00",
|
||||
formatted_amount: "$100.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "No Price",
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("With Price");
|
||||
});
|
||||
|
||||
test("should respect MAX_ITEMS parameter", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: Array.from({ length: 10 }, (_, i) => ({
|
||||
node: {
|
||||
listing: {
|
||||
id: String(i),
|
||||
marketplace_listing_title: `Item ${i}`,
|
||||
listing_price: {
|
||||
amount: `${(i + 1) * 10}.00`,
|
||||
formatted_amount: `$${(i + 1) * 10}.00`,
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
})),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
5,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(5);
|
||||
});
|
||||
|
||||
test("should return empty array for no results", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"nonexistent query",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
test("should handle authentication errors gracefully", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 401,
|
||||
text: () => Promise.resolve("Unauthorized"),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
test("should handle network errors", async () => {
|
||||
global.fetch = mock(() => Promise.reject(new Error("Network error")));
|
||||
|
||||
await expect(
|
||||
fetchFacebookItems("test", 1, "toronto", 25, mockCookies),
|
||||
).rejects.toThrow("Network error");
|
||||
});
|
||||
|
||||
test("should handle rate limiting with retry", async () => {
|
||||
let attempts = 0;
|
||||
global.fetch = mock(() => {
|
||||
attempts++;
|
||||
if (attempts === 1) {
|
||||
return Promise.resolve({
|
||||
ok: false,
|
||||
status: 429,
|
||||
headers: {
|
||||
get: (header: string) => {
|
||||
if (header === "X-RateLimit-Reset") return "1";
|
||||
return null;
|
||||
},
|
||||
},
|
||||
text: () => Promise.resolve("Rate limited"),
|
||||
});
|
||||
}
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "Item 1",
|
||||
listing_price: {
|
||||
amount: "100.00",
|
||||
formatted_amount: "$100.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(attempts).toBe(2);
|
||||
expect(results).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Vehicle Listing Integration", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should correctly identify and parse vehicle listings", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "2006 Honda Civic",
|
||||
listing_price: {
|
||||
amount: "8000.00",
|
||||
formatted_amount: "$8,000.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "2",
|
||||
marketplace_listing_title: "iPhone 13",
|
||||
listing_price: {
|
||||
amount: "800.00",
|
||||
formatted_amount: "$800.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"cars",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(2);
|
||||
// Both should be classified as "item" type in search results (vehicle detection is for item details)
|
||||
expect(results[0].title).toBe("2006 Honda Civic");
|
||||
expect(results[1].title).toBe("iPhone 13");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Different Categories", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should handle electronics listings", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "Nintendo Switch",
|
||||
listing_price: {
|
||||
amount: "250.00",
|
||||
formatted_amount: "$250.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
location: {
|
||||
reverse_geocode: {
|
||||
city_page: { display_name: "Toronto" },
|
||||
},
|
||||
},
|
||||
marketplace_listing_category_id:
|
||||
"479353692612078",
|
||||
condition: "USED",
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"nintendo switch",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("Nintendo Switch");
|
||||
expect(results[0].categoryId).toBe("479353692612078");
|
||||
});
|
||||
|
||||
test("should handle home goods/furniture listings", async () => {
|
||||
const mockSearchData = {
|
||||
require: [
|
||||
[
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
{
|
||||
__bbox: {
|
||||
result: {
|
||||
data: {
|
||||
marketplace_search: {
|
||||
feed_units: {
|
||||
edges: [
|
||||
{
|
||||
node: {
|
||||
listing: {
|
||||
id: "1",
|
||||
marketplace_listing_title: "Dining Table",
|
||||
listing_price: {
|
||||
amount: "150.00",
|
||||
formatted_amount: "$150.00",
|
||||
currency: "CAD",
|
||||
},
|
||||
location: {
|
||||
reverse_geocode: {
|
||||
city_page: { display_name: "Mississauga" },
|
||||
},
|
||||
},
|
||||
marketplace_listing_category_id:
|
||||
"1569171756675761",
|
||||
condition: "USED",
|
||||
is_live: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
};
|
||||
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
`<html><body><script>${JSON.stringify(mockSearchData)}</script></body></html>`,
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"table",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].title).toBe("Dining Table");
|
||||
expect(results[0].categoryId).toBe("1569171756675761");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Error Scenarios", () => {
|
||||
const mockCookies = JSON.stringify([
|
||||
{ name: "c_user", value: "12345", domain: ".facebook.com", path: "/" },
|
||||
{ name: "xs", value: "abc123", domain: ".facebook.com", path: "/" },
|
||||
]);
|
||||
|
||||
test("should handle malformed HTML responses", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
"<html><body>Invalid HTML without JSON data</body></html>",
|
||||
),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
test("should handle 404 errors gracefully", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 404,
|
||||
text: () => Promise.resolve("Not found"),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
test("should handle 500 errors gracefully", async () => {
|
||||
global.fetch = mock(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: () => Promise.resolve("Internal Server Error"),
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const results = await fetchFacebookItems(
|
||||
"test",
|
||||
1,
|
||||
"toronto",
|
||||
25,
|
||||
mockCookies,
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
166
packages/core/test/kijiji-core.test.ts
Normal file
166
packages/core/test/kijiji-core.test.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
HttpError,
|
||||
NetworkError,
|
||||
ParseError,
|
||||
RateLimitError,
|
||||
ValidationError,
|
||||
buildSearchUrl,
|
||||
resolveCategoryId,
|
||||
resolveLocationId,
|
||||
} from "../src/scrapers/kijiji";
|
||||
|
||||
describe("Location and Category Resolution", () => {
|
||||
describe("resolveLocationId", () => {
|
||||
test("should return numeric IDs as-is", () => {
|
||||
expect(resolveLocationId(1700272)).toBe(1700272);
|
||||
expect(resolveLocationId(0)).toBe(0);
|
||||
});
|
||||
|
||||
test("should resolve string location names", () => {
|
||||
expect(resolveLocationId("canada")).toBe(0);
|
||||
expect(resolveLocationId("ontario")).toBe(9004);
|
||||
expect(resolveLocationId("toronto")).toBe(1700273);
|
||||
expect(resolveLocationId("gta")).toBe(1700272);
|
||||
});
|
||||
|
||||
test("should handle case insensitive matching", () => {
|
||||
expect(resolveLocationId("Canada")).toBe(0);
|
||||
expect(resolveLocationId("ONTARIO")).toBe(9004);
|
||||
});
|
||||
|
||||
test("should default to Canada for unknown locations", () => {
|
||||
expect(resolveLocationId("unknown")).toBe(0);
|
||||
expect(resolveLocationId("")).toBe(0);
|
||||
});
|
||||
|
||||
test("should handle undefined input", () => {
|
||||
expect(resolveLocationId(undefined)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveCategoryId", () => {
|
||||
test("should return numeric IDs as-is", () => {
|
||||
expect(resolveCategoryId(132)).toBe(132);
|
||||
expect(resolveCategoryId(0)).toBe(0);
|
||||
});
|
||||
|
||||
test("should resolve string category names", () => {
|
||||
expect(resolveCategoryId("all")).toBe(0);
|
||||
expect(resolveCategoryId("phones")).toBe(132);
|
||||
expect(resolveCategoryId("electronics")).toBe(29659001);
|
||||
expect(resolveCategoryId("buy-sell")).toBe(10);
|
||||
});
|
||||
|
||||
test("should handle case insensitive matching", () => {
|
||||
expect(resolveCategoryId("All")).toBe(0);
|
||||
expect(resolveCategoryId("PHONES")).toBe(132);
|
||||
});
|
||||
|
||||
test("should default to all categories for unknown categories", () => {
|
||||
expect(resolveCategoryId("unknown")).toBe(0);
|
||||
expect(resolveCategoryId("")).toBe(0);
|
||||
});
|
||||
|
||||
test("should handle undefined input", () => {
|
||||
expect(resolveCategoryId(undefined)).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("URL Construction", () => {
|
||||
describe("buildSearchUrl", () => {
|
||||
test("should build basic search URL", () => {
|
||||
const url = buildSearchUrl("iphone", {
|
||||
location: 1700272,
|
||||
category: 132,
|
||||
sortBy: "relevancy",
|
||||
sortOrder: "desc",
|
||||
});
|
||||
|
||||
expect(url).toContain("b-buy-sell/canada/iphone/k0c132l1700272");
|
||||
expect(url).toContain("sort=relevancyDesc");
|
||||
expect(url).toContain("order=DESC");
|
||||
});
|
||||
|
||||
test("should handle pagination", () => {
|
||||
const url = buildSearchUrl("iphone", {
|
||||
location: 1700272,
|
||||
category: 132,
|
||||
page: 2,
|
||||
});
|
||||
|
||||
expect(url).toContain("&page=2");
|
||||
});
|
||||
|
||||
test("should handle different sort options", () => {
|
||||
const dateUrl = buildSearchUrl("iphone", {
|
||||
sortBy: "date",
|
||||
sortOrder: "asc",
|
||||
});
|
||||
expect(dateUrl).toContain("sort=DATE");
|
||||
expect(dateUrl).toContain("order=ASC");
|
||||
|
||||
const priceUrl = buildSearchUrl("iphone", {
|
||||
sortBy: "price",
|
||||
sortOrder: "desc",
|
||||
});
|
||||
expect(priceUrl).toContain("sort=PRICE");
|
||||
expect(priceUrl).toContain("order=DESC");
|
||||
});
|
||||
|
||||
test("should handle string location/category inputs", () => {
|
||||
const url = buildSearchUrl("iphone", {
|
||||
location: "toronto",
|
||||
category: "phones",
|
||||
});
|
||||
|
||||
expect(url).toContain("k0c132l1700273"); // phones + toronto
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Error Classes", () => {
|
||||
test("HttpError should store status and URL", () => {
|
||||
const error = new HttpError("Not found", 404, "https://example.com");
|
||||
expect(error.message).toBe("Not found");
|
||||
expect(error.statusCode).toBe(404);
|
||||
expect(error.url).toBe("https://example.com");
|
||||
expect(error.name).toBe("HttpError");
|
||||
});
|
||||
|
||||
test("NetworkError should store URL and cause", () => {
|
||||
const cause = new Error("Connection failed");
|
||||
const error = new NetworkError(
|
||||
"Network error",
|
||||
"https://example.com",
|
||||
cause
|
||||
);
|
||||
expect(error.message).toBe("Network error");
|
||||
expect(error.url).toBe("https://example.com");
|
||||
expect(error.cause).toBe(cause);
|
||||
expect(error.name).toBe("NetworkError");
|
||||
});
|
||||
|
||||
test("ParseError should store data", () => {
|
||||
const data = { invalid: "json" };
|
||||
const error = new ParseError("Invalid JSON", data);
|
||||
expect(error.message).toBe("Invalid JSON");
|
||||
expect(error.data).toBe(data);
|
||||
expect(error.name).toBe("ParseError");
|
||||
});
|
||||
|
||||
test("RateLimitError should store URL and reset time", () => {
|
||||
const error = new RateLimitError("Rate limited", "https://example.com", 60);
|
||||
expect(error.message).toBe("Rate limited");
|
||||
expect(error.url).toBe("https://example.com");
|
||||
expect(error.resetTime).toBe(60);
|
||||
expect(error.name).toBe("RateLimitError");
|
||||
});
|
||||
|
||||
test("ValidationError should work without field", () => {
|
||||
const error = new ValidationError("Invalid value");
|
||||
expect(error.message).toBe("Invalid value");
|
||||
expect(error.name).toBe("ValidationError");
|
||||
});
|
||||
});
|
||||
363
packages/core/test/kijiji-integration.test.ts
Normal file
363
packages/core/test/kijiji-integration.test.ts
Normal file
@@ -0,0 +1,363 @@
|
||||
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import {
|
||||
extractApolloState,
|
||||
parseDetailedListing,
|
||||
parseSearch,
|
||||
} from "../src/scrapers/kijiji";
|
||||
|
||||
// Mock fetch globally
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
describe("HTML Parsing Integration", () => {
|
||||
beforeEach(() => {
|
||||
// Mock fetch for all tests
|
||||
global.fetch = mock(() => {
|
||||
throw new Error("fetch should be mocked in individual tests");
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
describe("extractApolloState", () => {
|
||||
test("should extract Apollo state from valid HTML", () => {
|
||||
const mockHtml =
|
||||
'<html><head><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"__APOLLO_STATE__":{"ROOT_QUERY":{"test":"value"}}}}}</script></head></html>';
|
||||
|
||||
const result = extractApolloState(mockHtml);
|
||||
expect(result).toEqual({
|
||||
ROOT_QUERY: { test: "value" },
|
||||
});
|
||||
});
|
||||
|
||||
test("should return null for HTML without Apollo state", () => {
|
||||
const mockHtml = "<html><body>No data here</body></html>";
|
||||
const result = extractApolloState(mockHtml);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should return null for malformed JSON", () => {
|
||||
const mockHtml =
|
||||
'<html><script id="__NEXT_DATA__" type="application/json">{"invalid": json}</script></html>';
|
||||
|
||||
const result = extractApolloState(mockHtml);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should handle missing __NEXT_DATA__ element", () => {
|
||||
const mockHtml = "<html><body><div>Content</div></body></html>";
|
||||
const result = extractApolloState(mockHtml);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseSearch", () => {
|
||||
test("should parse search results from HTML", () => {
|
||||
const mockHtml = `
|
||||
<html>
|
||||
<script id="__NEXT_DATA__" type="application/json">
|
||||
${JSON.stringify({
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:123": {
|
||||
url: "/v-iphone/k0l0",
|
||||
title: "iPhone 13 Pro",
|
||||
},
|
||||
"Listing:456": {
|
||||
url: "/v-samsung/k0l0",
|
||||
title: "Samsung Galaxy",
|
||||
},
|
||||
ROOT_QUERY: { test: "value" },
|
||||
},
|
||||
},
|
||||
},
|
||||
})}
|
||||
</script>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0]).toEqual({
|
||||
name: "iPhone 13 Pro",
|
||||
listingLink: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||
});
|
||||
expect(results[1]).toEqual({
|
||||
name: "Samsung Galaxy",
|
||||
listingLink: "https://www.kijiji.ca/v-samsung/k0l0",
|
||||
});
|
||||
});
|
||||
|
||||
test("should handle absolute URLs", () => {
|
||||
const mockHtml = `
|
||||
<html>
|
||||
<script id="__NEXT_DATA__" type="application/json">
|
||||
${JSON.stringify({
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:123": {
|
||||
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||
title: "iPhone 13 Pro",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})}
|
||||
</script>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||
expect(results[0].listingLink).toBe(
|
||||
"https://www.kijiji.ca/v-iphone/k0l0",
|
||||
);
|
||||
});
|
||||
|
||||
test("should filter out invalid listings", () => {
|
||||
const mockHtml = `
|
||||
<html>
|
||||
<script id="__NEXT_DATA__" type="application/json">
|
||||
${JSON.stringify({
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:123": {
|
||||
url: "/v-iphone/k0l0",
|
||||
title: "iPhone 13 Pro",
|
||||
},
|
||||
"Listing:456": {
|
||||
url: "/v-samsung/k0l0",
|
||||
// Missing title
|
||||
},
|
||||
"Other:789": {
|
||||
url: "/v-other/k0l0",
|
||||
title: "Other Item",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})}
|
||||
</script>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const results = parseSearch(mockHtml, "https://www.kijiji.ca");
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].name).toBe("iPhone 13 Pro");
|
||||
});
|
||||
|
||||
test("should return empty array for invalid HTML", () => {
|
||||
const results = parseSearch(
|
||||
"<html><body>Invalid</body></html>",
|
||||
"https://www.kijiji.ca",
|
||||
);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseDetailedListing", () => {
|
||||
test("should parse detailed listing with all fields", async () => {
|
||||
const mockHtml = `
|
||||
<html>
|
||||
<script id="__NEXT_DATA__" type="application/json">
|
||||
${JSON.stringify({
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:123": {
|
||||
url: "/v-iphone-13-pro/k0l0",
|
||||
title: "iPhone 13 Pro 256GB",
|
||||
description: "Excellent condition iPhone 13 Pro",
|
||||
price: {
|
||||
amount: 80000,
|
||||
currency: "CAD",
|
||||
type: "FIXED",
|
||||
},
|
||||
type: "OFFER",
|
||||
status: "ACTIVE",
|
||||
activationDate: "2024-01-15T10:00:00.000Z",
|
||||
endDate: "2025-01-15T10:00:00.000Z",
|
||||
metrics: { views: 150 },
|
||||
location: {
|
||||
address: "Toronto, ON",
|
||||
id: 1700273,
|
||||
name: "Toronto",
|
||||
coordinates: {
|
||||
latitude: 43.6532,
|
||||
longitude: -79.3832,
|
||||
},
|
||||
},
|
||||
imageUrls: [
|
||||
"https://media.kijiji.ca/api/v1/image1.jpg",
|
||||
"https://media.kijiji.ca/api/v1/image2.jpg",
|
||||
],
|
||||
imageCount: 2,
|
||||
categoryId: 132,
|
||||
adSource: "ORGANIC",
|
||||
flags: {
|
||||
topAd: false,
|
||||
priceDrop: true,
|
||||
},
|
||||
posterInfo: {
|
||||
posterId: "user123",
|
||||
rating: 4.8,
|
||||
},
|
||||
attributes: [
|
||||
{
|
||||
canonicalName: "forsaleby",
|
||||
canonicalValues: ["ownr"],
|
||||
},
|
||||
{
|
||||
canonicalName: "phonecarrier",
|
||||
canonicalValues: ["unlocked"],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})}
|
||||
</script>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const result = await parseDetailedListing(
|
||||
mockHtml,
|
||||
"https://www.kijiji.ca",
|
||||
);
|
||||
expect(result).toEqual({
|
||||
url: "https://www.kijiji.ca/v-iphone-13-pro/k0l0",
|
||||
title: "iPhone 13 Pro 256GB",
|
||||
description: "Excellent condition iPhone 13 Pro",
|
||||
listingPrice: {
|
||||
amountFormatted: "$800.00",
|
||||
cents: 80000,
|
||||
currency: "CAD",
|
||||
},
|
||||
listingType: "OFFER",
|
||||
listingStatus: "ACTIVE",
|
||||
creationDate: "2024-01-15T10:00:00.000Z",
|
||||
endDate: "2025-01-15T10:00:00.000Z",
|
||||
numberOfViews: 150,
|
||||
address: "Toronto, ON",
|
||||
images: [
|
||||
"https://media.kijiji.ca/api/v1/image1.jpg",
|
||||
"https://media.kijiji.ca/api/v1/image2.jpg",
|
||||
],
|
||||
categoryId: 132,
|
||||
adSource: "ORGANIC",
|
||||
flags: {
|
||||
topAd: false,
|
||||
priceDrop: true,
|
||||
},
|
||||
attributes: {
|
||||
forsaleby: ["ownr"],
|
||||
phonecarrier: ["unlocked"],
|
||||
},
|
||||
location: {
|
||||
id: 1700273,
|
||||
name: "Toronto",
|
||||
coordinates: {
|
||||
latitude: 43.6532,
|
||||
longitude: -79.3832,
|
||||
},
|
||||
},
|
||||
sellerInfo: {
|
||||
posterId: "user123",
|
||||
rating: 4.8,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("should return null for contact-based pricing", async () => {
|
||||
const mockHtml = `
|
||||
<html>
|
||||
<script id="__NEXT_DATA__" type="application/json">
|
||||
${JSON.stringify({
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:123": {
|
||||
url: "/v-iphone/k0l0",
|
||||
title: "iPhone for Sale",
|
||||
price: {
|
||||
type: "CONTACT",
|
||||
amount: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})}
|
||||
</script>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const result = await parseDetailedListing(
|
||||
mockHtml,
|
||||
"https://www.kijiji.ca",
|
||||
);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("should handle missing optional fields", async () => {
|
||||
const mockHtml = `
|
||||
<html>
|
||||
<script id="__NEXT_DATA__" type="application/json">
|
||||
${JSON.stringify({
|
||||
props: {
|
||||
pageProps: {
|
||||
__APOLLO_STATE__: {
|
||||
"Listing:123": {
|
||||
url: "/v-iphone/k0l0",
|
||||
title: "iPhone 13",
|
||||
price: { amount: 50000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})}
|
||||
</script>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const result = await parseDetailedListing(
|
||||
mockHtml,
|
||||
"https://www.kijiji.ca",
|
||||
);
|
||||
expect(result).toEqual({
|
||||
url: "https://www.kijiji.ca/v-iphone/k0l0",
|
||||
title: "iPhone 13",
|
||||
description: undefined,
|
||||
listingPrice: {
|
||||
amountFormatted: "$500.00",
|
||||
cents: 50000,
|
||||
currency: undefined,
|
||||
},
|
||||
listingType: undefined,
|
||||
listingStatus: undefined,
|
||||
creationDate: undefined,
|
||||
endDate: undefined,
|
||||
numberOfViews: undefined,
|
||||
address: null,
|
||||
images: [],
|
||||
categoryId: 0,
|
||||
adSource: "UNKNOWN",
|
||||
flags: {
|
||||
topAd: false,
|
||||
priceDrop: false,
|
||||
},
|
||||
attributes: {},
|
||||
location: {
|
||||
id: 0,
|
||||
name: "Unknown",
|
||||
coordinates: undefined,
|
||||
},
|
||||
sellerInfo: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
54
packages/core/test/kijiji-utils.test.ts
Normal file
54
packages/core/test/kijiji-utils.test.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import { formatCentsToCurrency, slugify } from "../src/scrapers/kijiji";
|
||||
|
||||
describe("Utility Functions", () => {
|
||||
describe("slugify", () => {
|
||||
test("should convert basic strings to slugs", () => {
|
||||
expect(slugify("Hello World")).toBe("hello-world");
|
||||
expect(slugify("iPhone 13 Pro")).toBe("iphone-13-pro");
|
||||
});
|
||||
|
||||
test("should handle special characters", () => {
|
||||
expect(slugify("Café & Restaurant")).toBe("cafe-restaurant");
|
||||
expect(slugify("100% New")).toBe("100-new");
|
||||
});
|
||||
|
||||
test("should handle empty and edge cases", () => {
|
||||
expect(slugify("")).toBe("");
|
||||
expect(slugify(" ")).toBe("-");
|
||||
expect(slugify("---")).toBe("-");
|
||||
});
|
||||
|
||||
test("should preserve numbers and valid characters", () => {
|
||||
expect(slugify("iPhone 13")).toBe("iphone-13");
|
||||
expect(slugify("item123")).toBe("item123");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatCentsToCurrency", () => {
|
||||
test("should format valid cent values", () => {
|
||||
expect(formatCentsToCurrency(100)).toBe("$1.00");
|
||||
expect(formatCentsToCurrency(1999)).toBe("$19.99");
|
||||
expect(formatCentsToCurrency(0)).toBe("$0.00");
|
||||
});
|
||||
|
||||
test("should handle string inputs", () => {
|
||||
expect(formatCentsToCurrency("100")).toBe("$1.00");
|
||||
expect(formatCentsToCurrency("1999")).toBe("$19.99");
|
||||
});
|
||||
|
||||
test("should handle null/undefined inputs", () => {
|
||||
expect(formatCentsToCurrency(null)).toBe("");
|
||||
expect(formatCentsToCurrency(undefined)).toBe("");
|
||||
});
|
||||
|
||||
test("should handle invalid inputs", () => {
|
||||
expect(formatCentsToCurrency("invalid")).toBe("");
|
||||
expect(formatCentsToCurrency(Number.NaN)).toBe("");
|
||||
});
|
||||
|
||||
test("should use en-US locale formatting", () => {
|
||||
expect(formatCentsToCurrency(123456)).toBe("$1,234.56");
|
||||
});
|
||||
});
|
||||
});
|
||||
11
packages/core/test/setup.ts
Normal file
11
packages/core/test/setup.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
// Test setup for Bun test runner
|
||||
// This file is loaded before any tests run due to bunfig.toml preload
|
||||
|
||||
// Mock fetch globally for tests
|
||||
global.fetch =
|
||||
global.fetch ||
|
||||
(() => {
|
||||
throw new Error("fetch is not available in test environment");
|
||||
});
|
||||
|
||||
// Add any global test utilities here
|
||||
Reference in New Issue
Block a user