diff --git a/backend/functions/src/cloud-functions/crawler.ts b/backend/functions/src/cloud-functions/crawler.ts index b71f71a..23d1f27 100644 --- a/backend/functions/src/cloud-functions/crawler.ts +++ b/backend/functions/src/cloud-functions/crawler.ts @@ -227,10 +227,6 @@ export class CrawlerHost extends RPCHost { } if (!uid) { - if (targetUrl.protocol === 'http:' && (!targetUrl.pathname || targetUrl.pathname === '/') && - crawlerOptions.respondWith !== 'default') { - throw new SecurityCompromiseError(`Your request is categorized as abuse. Please don't abuse our service. If you are sure you are not abusing, please authenticate yourself with an API key.`); - } const blockade = (await DomainBlockade.fromFirestoreQuery( DomainBlockade.COLLECTION .where('domain', '==', targetUrl.hostname.toLowerCase()) diff --git a/thinapps-shared b/thinapps-shared index efa6da6..7b3412e 160000 --- a/thinapps-shared +++ b/thinapps-shared @@ -1 +1 @@ -Subproject commit efa6da6ff1c9a4e2841c50a722a0ae2283be827c +Subproject commit 7b3412e64166599429fa38094f4abd071a15fcd6