fix: harden shared http helper

packages/core/src/utils/http.ts

@@ -71,6 +71,43 @@ function calculateBackoffDelay(
   return Math.min(exponentialDelay + jitterDelay, 30000); // Cap at 30 seconds
 }
 
+const MAX_RATE_LIMIT_WAIT_MS = 30_000;
+const MAX_DELTA_RESET_SECONDS = 86_400;
+
+function mergeHeaders(
+  defaultHeaders: Record<string, string>,
+  customHeaders?: Record<string, string>,
+): Record<string, string> {
+  const merged: Record<string, string> = {};
+
+  for (const [key, value] of Object.entries(defaultHeaders)) {
+    merged[key.toLowerCase()] = value;
+  }
+
+  for (const [key, value] of Object.entries(customHeaders ?? {})) {
+    merged[key.toLowerCase()] = value;
+  }
+
+  return merged;
+}
+
+function calculateRateLimitWaitMs(
+  resetHeader: string | null,
+  fallbackWaitMs: number,
+): number {
+  if (!resetHeader) return fallbackWaitMs;
+
+  const resetValue = Number(resetHeader);
+  if (!Number.isFinite(resetValue)) return fallbackWaitMs;
+
+  const waitMs =
+    resetValue <= MAX_DELTA_RESET_SECONDS
+      ? resetValue * 1000
+      : resetValue * 1000 - Date.now();
+
+  return Math.min(Math.max(0, waitMs), MAX_RATE_LIMIT_WAIT_MS);
+}
+
 /** Result type when includeResponseUrl is true */
 export interface FetchHtmlResult {
   html: HTMLString;
@@ -141,13 +178,17 @@ export async function fetchHtml(
       const controller = new AbortController();
       const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
 
-      const res = await fetch(url, {
-        method: "GET",
-        headers: { ...defaultHeaders, ...opts?.headers },
-        signal: controller.signal,
-      });
-
-      clearTimeout(timeoutId);
+      const res = await (async () => {
+        try {
+          return await fetch(url, {
+            method: "GET",
+            headers: mergeHeaders(defaultHeaders, opts?.headers),
+            signal: controller.signal,
+          });
+        } finally {
+          clearTimeout(timeoutId);
+        }
+      })();
 
       const rateLimitRemaining = res.headers.get("X-RateLimit-Remaining");
       const rateLimitReset = res.headers.get("X-RateLimit-Reset");
@@ -159,13 +200,14 @@ export async function fetchHtml(
           const resetSeconds = rateLimitReset
             ? Number(rateLimitReset)
             : Number.NaN;
-          const waitMs = Number.isFinite(resetSeconds)
-            ? Math.max(0, resetSeconds * 1000)
-            : calculateBackoffDelay(
-                attempt,
-                retryBaseMs,
-                opts?.jitter ?? Math.random,
-              );
+          const waitMs = calculateRateLimitWaitMs(
+            rateLimitReset,
+            calculateBackoffDelay(
+              attempt,
+              retryBaseMs,
+              opts?.jitter ?? Math.random,
+            ),
+          );
 
           if (attempt < maxRetries) {
             await delay(waitMs);