AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-04-21 13:39:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(fetch): don't time out (for smart scrape)

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-15 21:57:05 -07:00
parent 0abe60085b
commit 13bd50ad2b
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/scrapeURL/lib/fetch.ts
View File

@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";
import { MockState, saveMock } from "./mock";
import { TimeoutSignal } from "../../../controllers/v1/types";
import { fireEngineURL } from "../engines/fire-engine/scrape";
import { fetch, RequestInit, Response, FormData, Agent } from "undici";
export type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string;
@ -78,6 +79,10 @@ export async function robustFetch<
...(headers !== undefined ? headers : {}),
},
signal: abort,
dispatcher: new Agent({
headersTimeout: 0,
bodyTimeout: 0,
}),
...(body instanceof FormData
? {
body,