AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 15:49:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update blocklist.ts

Browse Source
This commit is contained in:
rafaelsideguide 2024-08-19 08:51:48 -03:00
parent 3fe82b4f12
commit b8170aaa47
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -15,7 +15,8 @@ const socialMediaBlocklist = [
'whatsapp.com', 'whatsapp.com',
'wechat.com', 'wechat.com',
'telegram.org', 'telegram.org',
'researchhub.com' 'researchhub.com',
'youtube.com'
]; ];
const allowedKeywords = [ const allowedKeywords = [