AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 08:16:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: add checkandupdateurl to crawlPreview

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-15 20:30:25 +02:00
parent 81b2479db3
commit 0c14366720
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/controllers/crawlPreview.ts
View File

@ -6,6 +6,7 @@ import { v4 as uuidv4 } from "uuid";
import { Logger } from "../../src/lib/logger";
import { addCrawlJob, crawlToCrawler, lockURL, saveCrawl, StoredCrawl } from "../../src/lib/crawl-redis";
import { addScrapeJob } from "../../src/services/queue-jobs";
import { checkAndUpdateURL } from "../../src/lib/validateUrl";
export async function crawlPreviewController(req: Request, res: Response) {
try {
@ -21,10 +22,17 @@ export async function crawlPreviewController(req: Request, res: Response) {
return res.status(status).json({ error });
}
const url = req.body.url;
let url = req.body.url;
if (!url) {
return res.status(400).json({ error: "Url is required" });
}
try {
url = checkAndUpdateURL(url).url;
} catch (e) {
return res
.status(e instanceof Error && e.message === "Invalid URL" ? 400 : 500)
.json({ error: e.message ?? e });
}
if (isUrlBlocked(url)) {
return res