AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-23 10:18:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(playwright): add contentType relaying

Browse Source
This commit is contained in:
Gergő Móricz 2025-06-04 01:00:44 +02:00
parent 017a39f143
commit 1c86e80a46
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
View File

@ -28,6 +28,7 @@ export async function scrapeURLWithPlaywright(
content: z.string(),
pageStatusCode: z.number(),
pageError: z.string().optional(),
contentType: z.string().optional(),
}),
mock: meta.mock,
abort: AbortSignal.timeout(timeout),
@ -46,5 +47,6 @@ export async function scrapeURLWithPlaywright(
html: response.content,
statusCode: response.pageStatusCode,
error: response.pageError,
contentType: response.contentType,
};
}

5
apps/playwright-service-ts/api.ts
View File

@ -136,9 +136,10 @@ const scrapePage = async (page: Page, url: string, waitUntil: 'load' | 'networki
}
let headers = null, content = await page.content();
let ct: string | undefined = undefined;
if (response) {
headers = await response.allHeaders();
const ct = Object.entries(headers).find(x => x[0].toLowerCase() === "content-type");
ct = Object.entries(headers).find(x => x[0].toLowerCase() === "content-type")?.[1];
if (ct && (ct[1].includes("application/json") || ct[1].includes("text/plain"))) {
content = (await response.body()).toString("utf8"); // TODO: determine real encoding
}
@ -148,6 +149,7 @@ const scrapePage = async (page: Page, url: string, waitUntil: 'load' | 'networki
content,
status: response ? response.status() : null,
headers,
contentType: ct,
};
};
@ -214,6 +216,7 @@ app.post('/scrape', async (req: Request, res: Response) => {
res.json({
content: result.content,
pageStatusCode: result.status,
contentType: result.contentType,
...(pageError && { pageError })
});
});