AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 07:59:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(removeUnwantedElements): post-includeTags excludeTags

Browse Source 
Fixes #700
This commit is contained in:
Móricz Gergő 2025-01-17 12:41:00 +01:00
parent de08b37480
commit faf58dfca7
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
View File

@ -54,7 +54,7 @@ export const htmlTransform = (
url: string,
scrapeOptions: ScrapeOptions,
) => {
const soup = load(html);
let soup = load(html);
// remove unwanted elements
if (
@ -68,7 +68,8 @@ export const htmlTransform = (
newRoot.append(soup(element).clone());
});
});
return newRoot.html() ?? "";
soup = load(newRoot.html() ?? "");
}
soup("script, style, noscript, meta, head").remove();