AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 12:29:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

block corterix.com for performance issues

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-25 20:05:17 +02:00
parent 6f9a2687ae
commit d591e0f51c
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -15,7 +15,8 @@ const socialMediaBlocklist = [
'wechat.com', 'wechat.com',
'telegram.org', 'telegram.org',
'researchhub.com', 'researchhub.com',
'youtube.com' 'youtube.com',
'corterix.com',
]; ];
const allowedKeywords = [ const allowedKeywords = [