AI/reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader synced 2025-08-18 10:35:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: fix abuse timing

Browse Source
This commit is contained in:
yanlong.wang 2024-06-11 13:57:19 +08:00
parent 70d80bbcfe
commit fd9a86bc00
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/functions/src/cloud-functions/crawler.ts
View File

@ -562,6 +562,10 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
} }
if (!uid) { if (!uid) {
if (urlToCrawl.protocol === 'http:' && (!urlToCrawl.pathname || urlToCrawl.pathname === '/') &&
crawlerOptions.respondWith !== 'default') {
throw new SecurityCompromiseError(`Your request is categorized as abuse. Please don't abuse our service. If you are sure you are not abusing, please authenticate yourself with an API key.`);
}
const blockade = (await DomainBlockade.fromFirestoreQuery( const blockade = (await DomainBlockade.fromFirestoreQuery(
DomainBlockade.COLLECTION DomainBlockade.COLLECTION
.where('domain', '==', urlToCrawl.hostname.toLowerCase()) .where('domain', '==', urlToCrawl.hostname.toLowerCase())
@ -571,10 +575,6 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
if (blockade) { if (blockade) {
throw new SecurityCompromiseError(`Domain ${urlToCrawl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`); throw new SecurityCompromiseError(`Domain ${urlToCrawl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);
} }
if (urlToCrawl.protocol === 'http:' && (!urlToCrawl.pathname || urlToCrawl.pathname === '/') &&
crawlerOptions.respondWith !== 'default') {
throw new SecurityCompromiseError(`Your request is categorized as abuse. Please don't abuse our service. If you are sure you are not abusing, please authenticate yourself with an API key.`);
}
} }
const crawlOpts = this.configure(crawlerOptions); const crawlOpts = this.configure(crawlerOptions);