diff --git a/packages/core/src/scrapers/ebay.ts b/packages/core/src/scrapers/ebay.ts index aa6cabc..79feb2b 100644 --- a/packages/core/src/scrapers/ebay.ts +++ b/packages/core/src/scrapers/ebay.ts @@ -1,5 +1,8 @@ import { parseHTML } from "linkedom"; -import type { UnstableListingModeOptions } from "../types/common"; +import type { + UnstableListingBuckets, + UnstableListingModeOptions, +} from "../types/common"; import { classifyUnstableListings } from "../utils/unstable"; import { type CookieConfig, @@ -352,6 +355,34 @@ async function loadEbayCookies(): Promise { // ----------------------------- Main ----------------------------- +export default async function fetchEbayItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND?: number, + opts?: { + minPrice?: number; + maxPrice?: number; + strictMode?: boolean; + exclusions?: string[]; + keywords?: string[]; + buyItNowOnly?: boolean; + canadaOnly?: boolean; + }, + unstableMode?: UnstableListingModeOptions, +): Promise; +export default async function fetchEbayItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND: number | undefined, + opts: { + minPrice?: number; + maxPrice?: number; + strictMode?: boolean; + exclusions?: string[]; + keywords?: string[]; + buyItNowOnly?: boolean; + canadaOnly?: boolean; + } | undefined, + unstableMode: { hideUnstableResults: true }, +): Promise>; export default async function fetchEbayItems( SEARCH_QUERY: string, REQUESTS_PER_SECOND = 1, @@ -366,7 +397,9 @@ export default async function fetchEbayItems( } = {}, unstableMode: UnstableListingModeOptions = {}, ) { - const finalizeResults = (listings: EbayListingDetails[]) => { + const finalizeResults = ( + listings: EbayListingDetails[], + ): EbayListingDetails[] | UnstableListingBuckets => { if (!unstableMode.hideUnstableResults) { return listings; } diff --git a/packages/core/src/scrapers/facebook.ts b/packages/core/src/scrapers/facebook.ts index 187884c..01051c1 100644 --- a/packages/core/src/scrapers/facebook.ts +++ b/packages/core/src/scrapers/facebook.ts @@ -1,6 +1,10 @@ import cliProgress from "cli-progress"; import { parseHTML } from "linkedom"; -import type { HTMLString, UnstableListingModeOptions } from "../types/common"; +import type { + HTMLString, + UnstableListingBuckets, + UnstableListingModeOptions, +} from "../types/common"; import { classifyUnstableListings } from "../utils/unstable"; import { type Cookie, @@ -1061,6 +1065,20 @@ export function parseFacebookItem( // ----------------------------- Main ----------------------------- +export default async function fetchFacebookItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND?: number, + LOCATION?: string, + MAX_ITEMS?: number, + unstableMode?: UnstableListingModeOptions, +): Promise; +export default async function fetchFacebookItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND: number | undefined, + LOCATION: string | undefined, + MAX_ITEMS: number | undefined, + unstableMode: { hideUnstableResults: true }, +): Promise>; export default async function fetchFacebookItems( SEARCH_QUERY: string, REQUESTS_PER_SECOND = 1, @@ -1068,16 +1086,14 @@ export default async function fetchFacebookItems( MAX_ITEMS = 25, unstableMode: UnstableListingModeOptions = {}, ) { - const finalizeResults = (listings: FacebookListingDetails[]) => { + const finalizeResults = ( + listings: FacebookListingDetails[], + ): FacebookListingDetails[] | UnstableListingBuckets => { if (!unstableMode.hideUnstableResults) { return listings.slice(0, MAX_ITEMS); } - const classified = classifyUnstableListings(listings); - return { - results: classified.results.slice(0, MAX_ITEMS), - unstableResults: classified.unstableResults, - }; + return classifyUnstableListings(listings.slice(0, MAX_ITEMS)); }; const cookies = await ensureFacebookCookies(); diff --git a/packages/core/src/scrapers/kijiji.ts b/packages/core/src/scrapers/kijiji.ts index f0ba0c4..ff11163 100644 --- a/packages/core/src/scrapers/kijiji.ts +++ b/packages/core/src/scrapers/kijiji.ts @@ -1,7 +1,11 @@ import cliProgress from "cli-progress"; import { parseHTML } from "linkedom"; import unidecode from "unidecode"; -import type { HTMLString, UnstableListingModeOptions } from "../types/common"; +import type { + HTMLString, + UnstableListingBuckets, + UnstableListingModeOptions, +} from "../types/common"; import { classifyUnstableListings } from "../utils/unstable"; import { type CookieConfig, @@ -697,6 +701,22 @@ export async function parseDetailedListing( // ----------------------------- Main ----------------------------- +export default async function fetchKijijiItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND?: number, + BASE_URL?: string, + searchOptions?: SearchOptions, + listingOptions?: ListingFetchOptions, + unstableMode?: UnstableListingModeOptions, +): Promise; +export default async function fetchKijijiItems( + SEARCH_QUERY: string, + REQUESTS_PER_SECOND: number | undefined, + BASE_URL: string | undefined, + searchOptions: SearchOptions | undefined, + listingOptions: ListingFetchOptions | undefined, + unstableMode: { hideUnstableResults: true }, +): Promise>; export default async function fetchKijijiItems( SEARCH_QUERY: string, REQUESTS_PER_SECOND = 1, @@ -705,7 +725,9 @@ export default async function fetchKijijiItems( listingOptions: ListingFetchOptions = {}, unstableMode: UnstableListingModeOptions = {}, ) { - const finalizeResults = (listings: DetailedListing[]) => { + const finalizeResults = ( + listings: DetailedListing[], + ): DetailedListing[] | UnstableListingBuckets => { if (!unstableMode.hideUnstableResults) { return listings; } diff --git a/packages/core/test/ebay-core.test.ts b/packages/core/test/ebay-core.test.ts index f9542fd..7712f46 100644 --- a/packages/core/test/ebay-core.test.ts +++ b/packages/core/test/ebay-core.test.ts @@ -1,6 +1,20 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import type { EbayListingDetails } from "../src/scrapers/ebay"; import fetchEbayItems from "../src/scrapers/ebay"; +type Assert = T; +type IsExact = + (() => G extends T ? 1 : 2) extends () => G extends U ? 1 : 2 + ? (() => G extends U ? 1 : 2) extends () => G extends T ? 1 : 2 + ? true + : false + : false; + +const getDefaultEbayItems = async () => fetchEbayItems("laptop"); +type _EbayDefaultReturn = Assert< + IsExact>, EbayListingDetails[]> +>; + const originalFetch = global.fetch; const originalWarn = console.warn; diff --git a/packages/core/test/facebook-core.test.ts b/packages/core/test/facebook-core.test.ts index 1662054..09bc729 100644 --- a/packages/core/test/facebook-core.test.ts +++ b/packages/core/test/facebook-core.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { classifyFacebookResponse, + type FacebookListingDetails, ensureFacebookCookies, extractFacebookBootstrapCandidates, extractFacebookItemData, @@ -14,6 +15,19 @@ import { import { formatCookiesForHeader } from "../src/utils/cookies"; import { formatCentsToCurrency } from "../src/utils/format"; +type Assert = T; +type IsExact = + (() => G extends T ? 1 : 2) extends () => G extends U ? 1 : 2 + ? (() => G extends U ? 1 : 2) extends () => G extends T ? 1 : 2 + ? true + : false + : false; + +const getDefaultFacebookItems = async () => fetchFacebookItems("chair"); +type _FacebookDefaultReturn = Assert< + IsExact>, FacebookListingDetails[]> +>; + // Mock fetch globally const originalFetch = global.fetch; @@ -492,17 +506,97 @@ describe("Facebook Marketplace Scraper Core Tests", () => { }), ); - const results = await fetchFacebookItems("chair", 1, "toronto", 1, { + const results = await fetchFacebookItems("chair", 1, "toronto", 25, { hideUnstableResults: true, }); expect(results).toEqual({ - results: [expect.objectContaining({ title: "Stable Chair Listing" })], + results: [ + expect.objectContaining({ title: "Stable Chair Listing" }), + expect.objectContaining({ title: "Another Stable Chair" }), + ], unstableResults: [ expect.objectContaining({ title: "Suspiciously Cheap Chair" }), ], }); }); + + test("unstable mode keeps MAX_ITEMS as the classification boundary", async () => { + const mockSearchHtml = ``; + + global.fetch = mock(() => + Promise.resolve({ + ok: true, + text: () => Promise.resolve(mockSearchHtml), + url: "https://www.facebook.com/marketplace/toronto/search?query=chair", + headers: { + get: () => null, + }, + }), + ); + + const results = await fetchFacebookItems("chair", 1, "toronto", 2, { + hideUnstableResults: true, + }); + + expect(results).toEqual({ + results: [expect.objectContaining({ title: "Boundary Stable Chair" })], + unstableResults: [ + expect.objectContaining({ title: "Boundary Cheap Chair" }), + ], + }); + }); }); describe("Data Extraction", () => { diff --git a/packages/core/test/kijiji-core.test.ts b/packages/core/test/kijiji-core.test.ts index 760720c..f0bb2b8 100644 --- a/packages/core/test/kijiji-core.test.ts +++ b/packages/core/test/kijiji-core.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; import { buildSearchUrl, default as fetchKijijiItems, + type DetailedListing, NetworkError, ParseError, RateLimitError, @@ -10,6 +11,19 @@ import { ValidationError, } from "../src/scrapers/kijiji"; +type Assert = T; +type IsExact = + (() => G extends T ? 1 : 2) extends () => G extends U ? 1 : 2 + ? (() => G extends U ? 1 : 2) extends () => G extends T ? 1 : 2 + ? true + : false + : false; + +const getDefaultKijijiItems = async () => fetchKijijiItems("phone"); +type _KijijiDefaultReturn = Assert< + IsExact>, DetailedListing[]> +>; + const originalFetch = global.fetch; beforeEach(() => {