From 2fab2d8d29c9c458a2461a7e3da4cd5510199008 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 17 Jul 2024 20:44:34 -0400 Subject: [PATCH] Update scrape.ts --- apps/api/src/controllers/scrape.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/api/src/controllers/scrape.ts b/apps/api/src/controllers/scrape.ts index f5e2c322..f076425f 100644 --- a/apps/api/src/controllers/scrape.ts +++ b/apps/api/src/controllers/scrape.ts @@ -116,11 +116,14 @@ export async function scrapeController(req: Request, res: Response) { const crawlerOptions = req.body.crawlerOptions ?? {}; const pageOptions = { ...defaultPageOptions, ...req.body.pageOptions }; const extractorOptions = { ...defaultExtractorOptions, ...req.body.extractorOptions }; + const origin = req.body.origin ?? defaultOrigin; + let timeout = req.body.timeout ?? defaultTimeout; + if (extractorOptions.mode === "llm-extraction") { pageOptions.onlyMainContent = true; + timeout = req.body.timeout ?? 90000; } - const origin = req.body.origin ?? defaultOrigin; - const timeout = req.body.timeout ?? defaultTimeout; + try { const { success: creditsCheckSuccess, message: creditsCheckMessage } =