From 13bd50ad2bc9d7f10f93ba0868bf69a56e5a7fc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= <mo.geryy@gmail.com>
Date: Tue, 15 Apr 2025 21:57:05 -0700
Subject: [PATCH] feat(fetch): don't time out (for smart scrape)

---
 apps/api/src/scraper/scrapeURL/lib/fetch.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index d6bd5157..c64e7953 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";
 import { MockState, saveMock } from "./mock";
 import { TimeoutSignal } from "../../../controllers/v1/types";
 import { fireEngineURL } from "../engines/fire-engine/scrape";
+import { fetch, RequestInit, Response, FormData, Agent } from "undici";
 
 export type RobustFetchParams<Schema extends z.Schema<any>> = {
   url: string;
@@ -78,6 +79,10 @@ export async function robustFetch<
           ...(headers !== undefined ? headers : {}),
         },
         signal: abort,
+        dispatcher: new Agent({
+          headersTimeout: 0,
+          bodyTimeout: 0,
+        }),
         ...(body instanceof FormData
           ? {
               body,