AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 01:15:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(removeUnwantedElements): post-includeTags excludeTags

Browse Source 
Fixes #700
This commit is contained in:
Móricz Gergő 2025-01-17 12:41:00 +01:00
parent de08b37480
commit faf58dfca7
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
View File

@ -54,7 +54,7 @@ export const htmlTransform = (
url: string, url: string,
scrapeOptions: ScrapeOptions, scrapeOptions: ScrapeOptions,
) => { ) => {
const soup = load(html); let soup = load(html);
// remove unwanted elements // remove unwanted elements
if ( if (
@ -68,7 +68,8 @@ export const htmlTransform = (
newRoot.append(soup(element).clone()); newRoot.append(soup(element).clone());
}); });
}); });
return newRoot.html() ?? "";
soup = load(newRoot.html() ?? "");
} }
soup("script, style, noscript, meta, head").remove(); soup("script, style, noscript, meta, head").remove();