AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 01:56:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrapeURL): includeTags/excludeTags

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-07 21:10:27 +01:00
parent 8d467c8ca7
commit 552d55c8fc
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
View File

@ -56,7 +56,7 @@ export const removeUnwantedElements = (
) => { ) => {
const soup = load(html); const soup = load(html);
if (scrapeOptions.includeTags && scrapeOptions.includeTags.length > 0) { if (scrapeOptions.includeTags && scrapeOptions.includeTags.filter(x => x.trim().length !== 0).length > 0) {
// Create a new root element to hold the tags to keep // Create a new root element to hold the tags to keep
const newRoot = load("<div></div>")("div"); const newRoot = load("<div></div>")("div");
scrapeOptions.includeTags.forEach((tag) => { scrapeOptions.includeTags.forEach((tag) => {
@ -69,7 +69,7 @@ export const removeUnwantedElements = (
soup("script, style, noscript, meta, head").remove(); soup("script, style, noscript, meta, head").remove();
if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.length > 0) { if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.filter(x => x.trim().length !== 0).length > 0) {
scrapeOptions.excludeTags.forEach((tag) => { scrapeOptions.excludeTags.forEach((tag) => {
let elementsToRemove: Cheerio<AnyNode>; let elementsToRemove: Cheerio<AnyNode>;
if (tag.startsWith("*") && tag.endsWith("*")) { if (tag.startsWith("*") && tag.endsWith("*")) {