AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-16 03:05:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: blocked sw/ra

Browse Source
This commit is contained in:
Nicolas 2024-08-29 13:55:54 -03:00
parent 02410aca36
commit 1174bc3c82
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -17,6 +17,8 @@ const socialMediaBlocklist = [
'researchhub.com', 'researchhub.com',
'youtube.com', 'youtube.com',
'corterix.com', 'corterix.com',
'southwest.com',
'ryanair.com'
]; ];
const allowedKeywords = [ const allowedKeywords = [