From ae0d705f5d3d5507447821a95eabcaef657bc42a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=B3ricz=20Gerg=C5=91?= Date: Mon, 20 Jan 2025 09:55:00 +0100 Subject: [PATCH] fix(v0/crawl): force kickoff --- apps/api/src/controllers/v0/crawl.ts | 5 ++++- apps/api/src/controllers/v0/crawlPreview.ts | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts index 67584178..5b19a0a8 100644 --- a/apps/api/src/controllers/v0/crawl.ts +++ b/apps/api/src/controllers/v0/crawl.ts @@ -18,6 +18,7 @@ import { addCrawlJob, addCrawlJobs, crawlToCrawler, + finishCrawlKickoff, lockURL, lockURLs, saveCrawl, @@ -177,6 +178,8 @@ export async function crawlController(req: Request, res: Response) { await saveCrawl(id, sc); + await finishCrawlKickoff(id); + const sitemap = sc.crawlerOptions.ignoreSitemap ? 0 : await crawler.tryGetSitemap(async (urls) => { @@ -224,7 +227,7 @@ export async function crawlController(req: Request, res: Response) { await addScrapeJob(job.data as any, {}, job.opts.jobId); } }); - + if (sitemap === 0) { await lockURL(id, sc, url); diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts index 3f1a9dd9..37c84cdc 100644 --- a/apps/api/src/controllers/v0/crawlPreview.ts +++ b/apps/api/src/controllers/v0/crawlPreview.ts @@ -7,6 +7,7 @@ import { logger } from "../../../src/lib/logger"; import { addCrawlJob, crawlToCrawler, + finishCrawlKickoff, lockURL, saveCrawl, StoredCrawl, @@ -112,6 +113,8 @@ export async function crawlPreviewController(req: Request, res: Response) { const crawler = crawlToCrawler(id, sc); + await finishCrawlKickoff(id); + const sitemap = sc.crawlerOptions?.ignoreSitemap ? 0 : await crawler.tryGetSitemap(async (urls) => {