fix: tighten item price and pacing behavior

2026-04-23 10:59:33 -04:00
parent 9c8643086a
commit 10c2856bf6
4 changed files with 135 additions and 11 deletions
--- a/packages/core/test/facebook-core.test.ts
+++ b/packages/core/test/facebook-core.test.ts
@@ -1532,10 +1532,21 @@ describe("Facebook Marketplace Scraper Core Tests", () => {
        };

        const result = parseFacebookItem(item);
-        expect(result).not.toBeNull();
-        expect(result?.title).toBe("Minimal Item");
-        expect(result?.description).toBeUndefined();
-        expect(result?.seller).toBeUndefined();
+        expect(result).toBeNull();
+      });
+
+      test("returns null when item price data is present but unparseable", () => {
+        const item = {
+          id: "456b",
+          __typename: "GroupCommerceProductItem" as const,
+          marketplace_listing_title: "Broken Price Item",
+          formatted_price: { text: "price unavailable" },
+          listing_price: { amount: "not-a-number", currency: "CAD" },
+        };
+
+        const result = parseFacebookItem(item);
+
+        expect(result).toBeNull();
      });

      test("should identify vehicle listings", () => {
--- a/packages/core/test/kijiji-core.test.ts
+++ b/packages/core/test/kijiji-core.test.ts
@@ -428,6 +428,104 @@ describe("fetchKijijiItems", () => {
    expect(maxActiveDetailRequests).toBe(1);
  });

+  test("allows bounded concurrency to scale with REQUESTS_PER_SECOND", async () => {
+    const searchHtml = `
+      <html>
+        <script id="__NEXT_DATA__" type="application/json">
+          ${JSON.stringify({
+            props: {
+              pageProps: {
+                __APOLLO_STATE__: {
+                  "Listing:1": { url: "/v-one/k0l0", title: "One" },
+                  "Listing:2": { url: "/v-two/k0l0", title: "Two" },
+                },
+              },
+            },
+          })}
+        </script>
+      </html>
+    `;
+
+    const listingHtml = (title: string, slug: string) => `
+      <html>
+        <script id="__NEXT_DATA__" type="application/json">
+          ${JSON.stringify({
+            props: {
+              pageProps: {
+                __APOLLO_STATE__: {
+                  "Listing:detail": {
+                    url: `/${slug}`,
+                    title,
+                    price: { amount: 10000, currency: "CAD", type: "FIXED" },
+                    type: "OFFER",
+                    status: "ACTIVE",
+                  },
+                },
+              },
+            },
+          })}
+        </script>
+      </html>
+    `;
+
+    let activeDetailRequests = 0;
+    let maxActiveDetailRequests = 0;
+
+    global.fetch = mock(async (input: string | URL | Request) => {
+      const url = typeof input === "string" ? input : input.toString();
+
+      if (url.includes("/k0c0l1700272")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(searchHtml),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      activeDetailRequests++;
+      maxActiveDetailRequests = Math.max(
+        maxActiveDetailRequests,
+        activeDetailRequests,
+      );
+
+      await new Promise((resolve) => setTimeout(resolve, 300));
+
+      activeDetailRequests--;
+
+      if (url.endsWith("/v-one/k0l0")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(listingHtml("One", "v-one/k0l0")),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      if (url.endsWith("/v-two/k0l0")) {
+        return {
+          ok: true,
+          text: () => Promise.resolve(listingHtml("Two", "v-two/k0l0")),
+          headers: { get: () => null },
+          url,
+        };
+      }
+
+      throw new Error(`Unexpected URL: ${url}`);
+    }) as typeof fetch;
+
+    const results = await fetchKijijiItems(
+      "phone",
+      4,
+      "https://www.kijiji.ca",
+      { maxPages: 1 },
+    );
+
+    expect(results).toHaveLength(2);
+    expect(maxActiveDetailRequests).toBeGreaterThan(1);
+    expect(maxActiveDetailRequests).toBeLessThanOrEqual(4);
+  });
+
  test("classifies the filtered Kijiji result set in unstable mode", async () => {
    const searchHtml = `
      <html>