AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 02:55:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL/removeUnwantedElements): try to fix onlyMainContent for poorly structured sites

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-04 19:05:12 +01:00
parent 88a16b18a3
commit 6b1f30e0fb
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
View File

@ -50,6 +50,10 @@ const excludeNonMainTags = [
"#cookie" "#cookie"
]; ];
const forceIncludeMainTags = [
"#main"
];
export const removeUnwantedElements = ( export const removeUnwantedElements = (
html: string, html: string,
scrapeOptions: ScrapeOptions scrapeOptions: ScrapeOptions
@ -101,7 +105,9 @@ export const removeUnwantedElements = (
if (scrapeOptions.onlyMainContent) { if (scrapeOptions.onlyMainContent) {
excludeNonMainTags.forEach((tag) => { excludeNonMainTags.forEach((tag) => {
const elementsToRemove = soup(tag); const elementsToRemove = soup(tag)
.filter(forceIncludeMainTags.map(x => ":not(:has(" + x + "))").join(""));
elementsToRemove.remove(); elementsToRemove.remove();
}); });
} }