AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 23:55:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(v0/crawl): force kickoff

Browse Source
This commit is contained in:
Móricz Gergő 2025-01-20 09:55:00 +01:00
parent 2cf7a4f57a
commit ae0d705f5d
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/controllers/v0/crawl.ts
View File

@ -18,6 +18,7 @@ import {
addCrawlJob, addCrawlJob,
addCrawlJobs, addCrawlJobs,
crawlToCrawler, crawlToCrawler,
finishCrawlKickoff,
lockURL, lockURL,
lockURLs, lockURLs,
saveCrawl, saveCrawl,
@ -177,6 +178,8 @@ export async function crawlController(req: Request, res: Response) {
await saveCrawl(id, sc); await saveCrawl(id, sc);
await finishCrawlKickoff(id);
const sitemap = sc.crawlerOptions.ignoreSitemap const sitemap = sc.crawlerOptions.ignoreSitemap
? 0 ? 0
: await crawler.tryGetSitemap(async (urls) => { : await crawler.tryGetSitemap(async (urls) => {
@ -224,7 +227,7 @@ export async function crawlController(req: Request, res: Response) {
await addScrapeJob(job.data as any, {}, job.opts.jobId); await addScrapeJob(job.data as any, {}, job.opts.jobId);
} }
}); });
if (sitemap === 0) { if (sitemap === 0) {
await lockURL(id, sc, url); await lockURL(id, sc, url);

3
apps/api/src/controllers/v0/crawlPreview.ts
View File

@ -7,6 +7,7 @@ import { logger } from "../../../src/lib/logger";
import { import {
addCrawlJob, addCrawlJob,
crawlToCrawler, crawlToCrawler,
finishCrawlKickoff,
lockURL, lockURL,
saveCrawl, saveCrawl,
StoredCrawl, StoredCrawl,
@ -112,6 +113,8 @@ export async function crawlPreviewController(req: Request, res: Response) {
const crawler = crawlToCrawler(id, sc); const crawler = crawlToCrawler(id, sc);
await finishCrawlKickoff(id);
const sitemap = sc.crawlerOptions?.ignoreSitemap const sitemap = sc.crawlerOptions?.ignoreSitemap
? 0 ? 0
: await crawler.tryGetSitemap(async (urls) => { : await crawler.tryGetSitemap(async (urls) => {