AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-04 23:00:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update crawl.ts

Browse Source
This commit is contained in:
Nicolas 2024-08-20 14:19:20 -03:00
parent de0dc20a02
commit c5ad4dedeb
1 changed files with 59 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
apps/api/src/controllers/v1/crawl.ts
View File

@ -1,21 +1,39 @@
import { Response } from "express"; import { Response } from "express";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { CrawlRequest, crawlRequestSchema, CrawlResponse, legacyCrawlerOptions, legacyScrapeOptions, RequestWithAuth } from "./types"; import {
import { addCrawlJob, addCrawlJobs, crawlToCrawler, lockURL, lockURLs, saveCrawl, StoredCrawl } from "../../lib/crawl-redis"; CrawlRequest,
crawlRequestSchema,
CrawlResponse,
legacyCrawlerOptions,
legacyScrapeOptions,
RequestWithAuth,
} from "./types";
import {
addCrawlJob,
addCrawlJobs,
crawlToCrawler,
lockURL,
lockURLs,
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
import { logCrawl } from "../../services/logging/crawl_log"; import { logCrawl } from "../../services/logging/crawl_log";
import { getScrapeQueue } from "../../services/queue-service"; import { getScrapeQueue } from "../../services/queue-service";
import { addScrapeJob } from "../../services/queue-jobs"; import { addScrapeJob } from "../../services/queue-jobs";
import { Logger } from "../../lib/logger"; import { Logger } from "../../lib/logger";
export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>, res: Response<CrawlResponse>) { export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
res: Response<CrawlResponse>
) {
req.body = crawlRequestSchema.parse(req.body); req.body = crawlRequestSchema.parse(req.body);
const id = uuidv4(); const id = uuidv4();
await logCrawl(id, req.auth.team_id); await logCrawl(id, req.auth.team_id);
const crawlerOptions = legacyCrawlerOptions(req.body.crawlerOptions), const crawlerOptions = legacyCrawlerOptions(req.body.crawlerOptions);
pageOptions = legacyScrapeOptions(req.body.scrapeOptions); const pageOptions = legacyScrapeOptions(req.body.scrapeOptions);
const sc: StoredCrawl = { const sc: StoredCrawl = {
originUrl: req.body.url, originUrl: req.body.url,
@ -30,15 +48,21 @@ export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, Cr
try { try {
sc.robots = await crawler.getRobotsTxt(); sc.robots = await crawler.getRobotsTxt();
} catch (e) { } catch (e) {
Logger.debug(`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(e)}`); Logger.debug(
`[Crawl] Failed to get robots.txt (this is probably fine!): ${JSON.stringify(
e
)}`
);
} }
await saveCrawl(id, sc); await saveCrawl(id, sc);
const sitemap = sc.crawlerOptions.ignoreSitemap ? null : await crawler.tryGetSitemap(); const sitemap = sc.crawlerOptions.ignoreSitemap
? null
: await crawler.tryGetSitemap();
if (sitemap !== null) { if (sitemap !== null) {
const jobs = sitemap.map(x => { const jobs = sitemap.map((x) => {
const url = x.url; const url = x.url;
const uuid = uuidv4(); const uuid = uuidv4();
return { return {
@ -56,33 +80,42 @@ export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, Cr
opts: { opts: {
jobId: uuid, jobId: uuid,
priority: 20, priority: 20,
} },
}; };
}) });
await lockURLs(id, jobs.map(x => x.data.url)); await lockURLs(
await addCrawlJobs(id, jobs.map(x => x.opts.jobId)); id,
jobs.map((x) => x.data.url)
);
await addCrawlJobs(
id,
jobs.map((x) => x.opts.jobId)
);
await getScrapeQueue().addBulk(jobs); await getScrapeQueue().addBulk(jobs);
} else { } else {
await lockURL(id, sc, req.body.url); await lockURL(id, sc, req.body.url);
const job = await addScrapeJob({ const job = await addScrapeJob(
url: req.body.url, {
mode: "single_urls", url: req.body.url,
crawlerOptions: crawlerOptions, mode: "single_urls",
team_id: req.auth.team_id, crawlerOptions: crawlerOptions,
pageOptions: pageOptions, team_id: req.auth.team_id,
origin: "api", pageOptions: pageOptions,
crawl_id: id, origin: "api",
webhook: req.body.webhook, crawl_id: id,
}, { webhook: req.body.webhook,
priority: 15, },
}); {
priority: 15,
}
);
await addCrawlJob(id, job.id); await addCrawlJob(id, job.id);
} }
return res.status(200).json({ return res.status(200).json({
success: true, success: true,
id, id,
url: `${req.protocol}://${req.get('host')}/v1/crawl/${id}`, url: `${req.protocol}://${req.get("host")}/v1/crawl/${id}`,
}); });
} }