AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-15 16:25:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #420 from mendableai/bugfix/empty-tags

Browse Source 
Small fix for empty pageOptions
This commit is contained in:
Rafael Miller 2024-07-17 10:10:24 -03:00 committed by GitHub
commit f020048a46
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/scraper/WebScraper/utils/removeUnwantedElements.ts
View File

@ -8,7 +8,11 @@ export const removeUnwantedElements = (
) => { ) => {
const soup = cheerio.load(html); const soup = cheerio.load(html);
if (pageOptions.onlyIncludeTags) { if (
pageOptions.onlyIncludeTags &&
pageOptions.onlyIncludeTags.length > 0 &&
pageOptions.onlyIncludeTags[0] !== ''
) {
if (typeof pageOptions.onlyIncludeTags === "string") { if (typeof pageOptions.onlyIncludeTags === "string") {
pageOptions.onlyIncludeTags = [pageOptions.onlyIncludeTags]; pageOptions.onlyIncludeTags = [pageOptions.onlyIncludeTags];
} }
@ -26,7 +30,11 @@ export const removeUnwantedElements = (
soup("script, style, iframe, noscript, meta, head").remove(); soup("script, style, iframe, noscript, meta, head").remove();
if (pageOptions.removeTags) { if (
pageOptions.removeTags &&
pageOptions.removeTags.length > 0 &&
pageOptions.removeTags[0] !== ''
) {
if (typeof pageOptions.removeTags === "string") { if (typeof pageOptions.removeTags === "string") {
pageOptions.removeTags = [pageOptions.removeTags]; pageOptions.removeTags = [pageOptions.removeTags];
} }