refactor: share scraper http fetching

2026-04-29 13:14:20 -04:00
parent 22eb65d4a2
commit 6e50ebf901
6 changed files with 121 additions and 173 deletions
--- a/packages/core/test/http.test.ts
+++ b/packages/core/test/http.test.ts
@@ -38,4 +38,23 @@ describe("fetchHtml", () => {

    expect(scheduledDelays).not.toContain(1000);
  });
+
+  test("fetchHtml returns responseUrl when includeResponseUrl is true", async () => {
+    process.env.NODE_ENV = "test";
+    global.fetch = mock(() =>
+      Promise.resolve({
+        ok: true,
+        status: 200,
+        url: "https://example.test/final",
+        headers: { get: () => null },
+        text: () => Promise.resolve("<html></html>"),
+      }),
+    ) as unknown as typeof fetch;
+
+    const result = await fetchHtml("https://example.test", 0, {
+      includeResponseUrl: true,
+    });
+    expect(result.html).toBe("<html></html>");
+    expect(result.responseUrl).toBe("https://example.test/final");
+  });
 });