mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-08-02 12:10:39 +08:00
fix: broken on self-host
This commit is contained in:
parent
cb3008a8af
commit
6287db8492
@ -78,5 +78,5 @@ describe("Crawl tests", () => {
|
||||
limit: 3,
|
||||
delay: 5,
|
||||
});
|
||||
}, 600000);
|
||||
}, 300000);
|
||||
});
|
||||
|
@ -674,37 +674,37 @@ const workerFun = async (
|
||||
runningJobs.delete(job.id);
|
||||
}
|
||||
|
||||
if (job.id && job.data.crawl_id && job.data.crawlerOptions?.delay) {
|
||||
await removeCrawlConcurrencyLimitActiveJob(job.data.crawl_id, job.id);
|
||||
cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);
|
||||
|
||||
const delayInSeconds = job.data.crawlerOptions.delay;
|
||||
const delayInMs = delayInSeconds * 1000;
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delayInMs));
|
||||
|
||||
const nextCrawlJob = await takeCrawlConcurrencyLimitedJob(job.data.crawl_id);
|
||||
if (nextCrawlJob !== null) {
|
||||
await pushCrawlConcurrencyLimitActiveJob(job.data.crawl_id, nextCrawlJob.id, 60 * 1000);
|
||||
|
||||
await queue.add(
|
||||
nextCrawlJob.id,
|
||||
{
|
||||
...nextCrawlJob.data,
|
||||
},
|
||||
{
|
||||
...nextCrawlJob.opts,
|
||||
jobId: nextCrawlJob.id,
|
||||
priority: nextCrawlJob.priority,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (job.id && job.data && job.data.team_id && job.data.plan) {
|
||||
await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);
|
||||
cleanOldConcurrencyLimitEntries(job.data.team_id);
|
||||
|
||||
if (job.data.crawl_id && job.data.crawlerOptions?.delay) {
|
||||
await removeCrawlConcurrencyLimitActiveJob(job.data.crawl_id, job.id);
|
||||
cleanOldCrawlConcurrencyLimitEntries(job.data.crawl_id);
|
||||
|
||||
const delayInSeconds = job.data.crawlerOptions.delay;
|
||||
const delayInMs = delayInSeconds * 1000;
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delayInMs));
|
||||
|
||||
const nextCrawlJob = await takeCrawlConcurrencyLimitedJob(job.data.crawl_id);
|
||||
if (nextCrawlJob !== null) {
|
||||
await pushCrawlConcurrencyLimitActiveJob(job.data.crawl_id, nextCrawlJob.id, 60 * 1000);
|
||||
|
||||
await queue.add(
|
||||
nextCrawlJob.id,
|
||||
{
|
||||
...nextCrawlJob.data,
|
||||
},
|
||||
{
|
||||
...nextCrawlJob.opts,
|
||||
jobId: nextCrawlJob.id,
|
||||
priority: nextCrawlJob.priority,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// No need to check if we're under the limit here -- if the current job is finished,
|
||||
// we are 1 under the limit, assuming the job insertion logic never over-inserts. - MG
|
||||
const nextJob = await takeConcurrencyLimitedJob(job.data.team_id);
|
||||
|
Loading…
x
Reference in New Issue
Block a user