AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 05:38:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into v1/llm-extract

Browse Source
This commit is contained in:
Nicolas 2024-08-30 12:48:26 -03:00
commit bb4808443c
2 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/controllers/v0/crawl-status.ts
View File

@ -56,6 +56,19 @@ export async function crawlStatusController(req: Request, res: Response) {
const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
if (
jobs.length > 0 &&
jobs[0].data &&
jobs[0].data.pageOptions &&
!jobs[0].data.pageOptions.includeRawHtml
) {
data.forEach(item => {
if (item) {
delete item.rawHtml;
}
});
}
res.json({ res.json({
status: jobStatus, status: jobStatus,
current: jobStatuses.filter(x => x === "completed" || x === "failed").length, current: jobStatuses.filter(x => x === "completed" || x === "failed").length,

8
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -100,6 +100,14 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
nextURL.searchParams.set("limit", req.query.limit); nextURL.searchParams.set("limit", req.query.limit);
} }
if (data.length > 0) {
if (!doneJobs[0].data.pageOptions.includeRawHtml) {
for (let ii = 0; ii < doneJobs.length; ii++) {
delete data[ii].rawHtml;
}
}
}
res.status(200).json({ res.status(200).json({
status, status,
completed: doneJobsLength, completed: doneJobsLength,