AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-16 17:25:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(playwright): add contentType relaying

Browse Source
This commit is contained in:
Gergő Móricz 2025-06-04 01:00:44 +02:00
parent 017a39f143
commit 1c86e80a46
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
View File

@ -28,6 +28,7 @@ export async function scrapeURLWithPlaywright(
content: z.string(), content: z.string(),
pageStatusCode: z.number(), pageStatusCode: z.number(),
pageError: z.string().optional(), pageError: z.string().optional(),
contentType: z.string().optional(),
}), }),
mock: meta.mock, mock: meta.mock,
abort: AbortSignal.timeout(timeout), abort: AbortSignal.timeout(timeout),
@ -46,5 +47,6 @@ export async function scrapeURLWithPlaywright(
html: response.content, html: response.content,
statusCode: response.pageStatusCode, statusCode: response.pageStatusCode,
error: response.pageError, error: response.pageError,
contentType: response.contentType,
}; };
} }

5
apps/playwright-service-ts/api.ts
View File

@ -136,9 +136,10 @@ const scrapePage = async (page: Page, url: string, waitUntil: 'load' | 'networki
} }
let headers = null, content = await page.content(); let headers = null, content = await page.content();
let ct: string | undefined = undefined;
if (response) { if (response) {
headers = await response.allHeaders(); headers = await response.allHeaders();
const ct = Object.entries(headers).find(x => x[0].toLowerCase() === "content-type"); ct = Object.entries(headers).find(x => x[0].toLowerCase() === "content-type")?.[1];
if (ct && (ct[1].includes("application/json") || ct[1].includes("text/plain"))) { if (ct && (ct[1].includes("application/json") || ct[1].includes("text/plain"))) {
content = (await response.body()).toString("utf8"); // TODO: determine real encoding content = (await response.body()).toString("utf8"); // TODO: determine real encoding
} }
@ -148,6 +149,7 @@ const scrapePage = async (page: Page, url: string, waitUntil: 'load' | 'networki
content, content,
status: response ? response.status() : null, status: response ? response.status() : null,
headers, headers,
contentType: ct,
}; };
}; };
@ -214,6 +216,7 @@ app.post('/scrape', async (req: Request, res: Response) => {
res.json({ res.json({
content: result.content, content: result.content,
pageStatusCode: result.status, pageStatusCode: result.status,
contentType: result.contentType,
...(pageError && { pageError }) ...(pageError && { pageError })
}); });
}); });