AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-04 16:00:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: robots.txt laoding

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-15 19:11:07 +02:00
parent 29f0d9ec94
commit c5597bc722
1 changed files with 6 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
apps/api/src/controllers/crawl.ts
View File

@ -95,25 +95,22 @@ export async function crawlController(req: Request, res: Response) {
await logCrawl(id, team_id);
let robots;
try {
robots = await this.getRobotsTxt();
} catch (_) {}
const sc: StoredCrawl = {
originUrl: url,
crawlerOptions,
pageOptions,
team_id,
robots,
createdAt: Date.now(),
};
await saveCrawl(id, sc);
const crawler = crawlToCrawler(id, sc);
try {
sc.robots = await crawler.getRobotsTxt();
} catch (_) {}
await saveCrawl(id, sc);
const sitemap = sc.crawlerOptions?.ignoreSitemap ? null : await crawler.tryGetSitemap();
if (sitemap !== null) {