diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 694b3dca..48a4a177 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -12,8 +12,7 @@ import { getDoneJobsOrdered, getDoneJobsOrderedLength, getThrottledJobs, - isCrawlFinished, - isCrawlFinishedLocked, + isCrawlKickoffFinished, } from "../../lib/crawl-redis"; import { getScrapeQueue } from "../../services/queue-service"; import { @@ -117,8 +116,7 @@ export async function crawlStatusController( const status: Exclude["status"] = sc.cancelled ? "cancelled" - : validJobStatuses.every((x) => x[1] === "completed") && - (await isCrawlFinishedLocked(req.params.jobId) || await isCrawlFinished(req.params.jobId)) + : validJobStatuses.every((x) => x[1] === "completed") && (sc.crawlerOptions ? await isCrawlKickoffFinished(req.params.jobId) : true) ? "completed" : "scraping"; diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts index add189ba..0e7e643a 100644 --- a/apps/api/src/lib/crawl-redis.ts +++ b/apps/api/src/lib/crawl-redis.ts @@ -132,6 +132,10 @@ export async function isCrawlFinished(id: string) { ); } +export async function isCrawlKickoffFinished(id: string) { + return await redisConnection.get("crawl:" + id + ":kickoff:finish") !== null +} + export async function isCrawlFinishedLocked(id: string) { return await redisConnection.exists("crawl:" + id + ":finish"); }