AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-16 18:25:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick:

Browse Source
This commit is contained in:
Nicolas 2024-08-16 22:17:38 -04:00
parent 47123be783
commit f797380112
2 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/services/alerts/index.ts
View File

@ -49,7 +49,7 @@ export async function checkAlerts() {
};
const checkAll = async () => {
// await checkActiveJobs();
await checkActiveJobs();
await checkWaitingQueue();
};

14
apps/api/src/services/queue-worker.ts
View File

@ -115,6 +115,20 @@ workerFun(scrapeQueueName, processJobInternal);
async function processJob(job: Job, token: string) {
Logger.info(`🐂 Worker taking job ${job.id}`);
// Check if the job URL is researchhub and block it immediately
// TODO: remove this once solve the root issue
if (job.data.url && job.data.url.includes("researchhub.com")) {
Logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`);
const data = {
success: false,
docs: [],
project_id: job.data.project_id,
error: "URL is blocked: researchhub.com",
};
await job.moveToCompleted(data.docs, token, false);
return data;
}
try {
job.updateProgress({
current: 1,