AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 02:55:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL/removeUnwantedElements): try to fix onlyMainContent for poorly structured sites

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-04 19:05:12 +01:00
parent 88a16b18a3
commit 6b1f30e0fb
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
View File

@ -50,6 +50,10 @@ const excludeNonMainTags = [
"#cookie"
];
const forceIncludeMainTags = [
"#main"
];
export const removeUnwantedElements = (
html: string,
scrapeOptions: ScrapeOptions
@ -101,7 +105,9 @@ export const removeUnwantedElements = (
if (scrapeOptions.onlyMainContent) {
excludeNonMainTags.forEach((tag) => {
const elementsToRemove = soup(tag);
const elementsToRemove = soup(tag)
.filter(forceIncludeMainTags.map(x => ":not(:has(" + x + "))").join(""));
elementsToRemove.remove();
});
}