fix: preserve free results and request pacing

2026-04-23 05:40:42 -04:00
parent 13c0fec305
commit eb37e8814e
4 changed files with 162 additions and 10 deletions
--- a/packages/core/test/facebook-core.test.ts
+++ b/packages/core/test/facebook-core.test.ts
@@ -571,6 +571,56 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
      expect(results).toHaveLength(1);
    });

+    test("preserves free listings through the public fetch entrypoint", async () => {
+      const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
+        payload: {
+          resultGroups: [
+            {
+              edges: [
+                {
+                  node: {
+                    listing: {
+                      id: "free-1",
+                      marketplace_listing_title: "Free Chair",
+                      listing_price: {
+                        amount: "0.00",
+                        formatted_amount: "FREE",
+                        currency: "CAD",
+                      },
+                      is_live: true,
+                    },
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      })}</script></body></html>`;
+
+      global.fetch = mock(() =>
+        Promise.resolve({
+          ok: true,
+          text: () => Promise.resolve(mockSearchHtml),
+          url: "https://www.facebook.com/marketplace/toronto/search?query=chair",
+          headers: {
+            get: () => null,
+          },
+        }),
+      );
+
+      const results = await fetchFacebookItems("chair", 1, "toronto", 25);
+
+      expect(results).toEqual([
+        expect.objectContaining({
+          title: "Free Chair",
+          listingPrice: expect.objectContaining({
+            cents: 0,
+            amountFormatted: "FREE",
+          }),
+        }),
+      ]);
+    });
+
    test("returns results and unstableResults when unstable mode is enabled", async () => {
      const mockSearchHtml = `<html><body><script>"XCometMarketplaceSearchController"</script><script>${JSON.stringify({
        payload: {
--- a/packages/core/test/kijiji-core.test.ts
+++ b/packages/core/test/kijiji-core.test.ts
@@ -321,6 +321,113 @@ describe("fetchKijijiItems", () => {
    ]);
  });

+  test("respects REQUESTS_PER_SECOND without concurrent detail fetch bursts", async () => {
+    const searchHtml = `
+      <html>
+        <script id="__NEXT_DATA__" type="application/json">
+          ${JSON.stringify({
+            props: {
+              pageProps: {
+                __APOLLO_STATE__: {
+                  "Listing:1": { url: "/v-one/k0l0", title: "One" },
+                  "Listing:2": { url: "/v-two/k0l0", title: "Two" },
+                  "Listing:3": { url: "/v-three/k0l0", title: "Three" },
+                },
+              },
+            },
+          })}
+        </script>
+      </html>
+    `;
+
+    const listingHtml = (title: string, slug: string) => `
+      <html>
+        <script id="__NEXT_DATA__" type="application/json">
+          ${JSON.stringify({
+            props: {
+              pageProps: {
+                __APOLLO_STATE__: {
+                  "Listing:detail": {
+                    url: `/${slug}`,
+                    title,
+                    price: { amount: 10000, currency: "CAD", type: "FIXED" },
+                    type: "OFFER",
+                    status: "ACTIVE",
+                  },
+                },
+              },
+            },
+          })}
+        </script>
+      </html>
+    `;
+
+    let activeDetailRequests = 0;
+    let maxActiveDetailRequests = 0;
+
+    global.fetch = mock(async (input: string | URL | Request) => {
+      const url = typeof input === "string" ? input : input.toString();
+
+      if (url.includes("/k0c0l1700272")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(searchHtml),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      activeDetailRequests++;
+      maxActiveDetailRequests = Math.max(
+        maxActiveDetailRequests,
+        activeDetailRequests,
+      );
+
+      await new Promise((resolve) => setTimeout(resolve, 5));
+
+      activeDetailRequests--;
+
+      if (url.endsWith("/v-one/k0l0")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(listingHtml("One", "v-one/k0l0")),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      if (url.endsWith("/v-two/k0l0")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(listingHtml("Two", "v-two/k0l0")),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      if (url.endsWith("/v-three/k0l0")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(listingHtml("Three", "v-three/k0l0")),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      throw new Error(`Unexpected URL: ${url}`);
+    }) as typeof fetch;
+
+    const results = await fetchKijijiItems(
+      "phone",
+      1,
+      "https://www.kijiji.ca",
+      { maxPages: 1 },
+    );
+
+    expect(results).toHaveLength(3);
+    expect(maxActiveDetailRequests).toBe(1);
+  });
+
  test("classifies the filtered Kijiji result set in unstable mode", async () => {
    const searchHtml = `
      <html>