AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 23:49:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(blocklist): behavioural fixes

Browse Source
This commit is contained in:
Gergő Móricz 2025-02-10 19:10:01 +01:00
parent 3608256569
commit bc787de62a
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -94,7 +94,7 @@ const allowedKeywords = [
"://www.facebook.com/ads/library", "://www.facebook.com/ads/library",
]; ];
function decryptedBlocklist(list: string[]): string[] { export function decryptedBlocklist(list: string[]): string[] {
return hashKey.length > 0 return hashKey.length > 0
? list.map((ciphertext) => decryptAES(ciphertext, hashKey)) ? list.map((ciphertext) => decryptAES(ciphertext, hashKey))
: []; : [];
@ -148,7 +148,7 @@ export function isUrlBlocked(url: string): boolean {
if ( if (
publicSuffix && publicSuffix &&
blockedlist.some( blockedlist.some(
(blocked) => blocked.startsWith(baseDomain) && blocked !== domain, (blocked) => blocked.startsWith(baseDomain + ".") && blocked !== domain,
) )
) { ) {
return true; return true;