AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 23:58:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(blocklist): behavioural fixes

Browse Source
This commit is contained in:
Gergő Móricz 2025-02-10 19:10:01 +01:00
parent 3608256569
commit bc787de62a
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -94,7 +94,7 @@ const allowedKeywords = [
"://www.facebook.com/ads/library",
];
function decryptedBlocklist(list: string[]): string[] {
export function decryptedBlocklist(list: string[]): string[] {
return hashKey.length > 0
? list.map((ciphertext) => decryptAES(ciphertext, hashKey))
: [];
@ -148,7 +148,7 @@ export function isUrlBlocked(url: string): boolean {
if (
publicSuffix &&
blockedlist.some(
(blocked) => blocked.startsWith(baseDomain) && blocked !== domain,
(blocked) => blocked.startsWith(baseDomain + ".") && blocked !== domain,
)
) {
return true;