From f8323f9b11ab6444dc86579697ee9595a6d9352e Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:02:05 -0300 Subject: [PATCH 1/2] fix: moved delete rawHtml to end of controller --- apps/api/src/controllers/v1/crawl-status.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 1fe2fd9a..7b94f07a 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -100,6 +100,14 @@ export async function crawlStatusController(req: RequestWithAuth 0) { + if (!doneJobs[0].data.pageOptions.includeRawHtml) { + for (let ii = 0; ii < doneJobs.length; ii++) { + delete data[ii].rawHtml; + } + } + } + res.status(200).json({ status, completed: doneJobsLength, From 7eefaffe48dbc7d96566c171936671041a7d5a4b Mon Sep 17 00:00:00 2001 From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:24:14 -0300 Subject: [PATCH 2/2] fix: rawHtml for v0 --- apps/api/src/controllers/v0/crawl-status.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index bda1af70..b0649cd0 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -56,6 +56,19 @@ export async function crawlStatusController(req: Request, res: Response) { const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); + if ( + jobs.length > 0 && + jobs[0].data && + jobs[0].data.pageOptions && + !jobs[0].data.pageOptions.includeRawHtml + ) { + data.forEach(item => { + if (item) { + delete item.rawHtml; + } + }); + } + res.json({ status: jobStatus, current: jobStatuses.filter(x => x === "completed" || x === "failed").length,