AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-04-19 12:39:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scraper): improve charset detection regex to accurately parse meta tags (#1265)

Browse Source
This commit is contained in:
Grass Huang 2025-02-27 00:31:06 +08:00 committed by GitHub
parent bf1a79588e
commit 7bf04d409a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
View File

@ -74,7 +74,7 @@ export async function scrapeURLWithFetch(
const buf = Buffer.from(await x.arrayBuffer()); const buf = Buffer.from(await x.arrayBuffer());
let text = buf.toString("utf8"); let text = buf.toString("utf8");
const charset = (text.match(/charset=["']?(.+?)["']?>/) ?? [])[1] const charset = (text.match(/<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i) ?? [])[1]
try { try {
if (charset) { if (charset) {
text = new TextDecoder(charset.trim()).decode(buf); text = new TextDecoder(charset.trim()).decode(buf);