AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-06 01:56:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: add jobId for scrape

Browse Source
This commit is contained in:
Gergo Moricz 2024-07-24 15:18:12 +02:00
parent 71072fef3b
commit d57dbbd0c6
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apps/api/src/controllers/scrape.ts
View File

@ -12,6 +12,7 @@ import { defaultPageOptions, defaultExtractorOptions, defaultTimeout, defaultOri
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
export async function scrapeHelper( export async function scrapeHelper(
jobId: string,
req: Request, req: Request,
team_id: string, team_id: string,
crawlerOptions: any, crawlerOptions: any,
@ -36,7 +37,7 @@ export async function scrapeHelper(
const a = new WebScraperDataProvider(); const a = new WebScraperDataProvider();
await a.setOptions({ await a.setOptions({
jobId: uuidv4(), jobId,
mode: "single_urls", mode: "single_urls",
urls: [url], urls: [url],
crawlerOptions: { crawlerOptions: {
@ -129,8 +130,11 @@ export async function scrapeController(req: Request, res: Response) {
checkCredits(); checkCredits();
} }
const jobId = uuidv4();
const startTime = new Date().getTime(); const startTime = new Date().getTime();
const result = await scrapeHelper( const result = await scrapeHelper(
jobId,
req, req,
team_id, team_id,
crawlerOptions, crawlerOptions,
@ -171,6 +175,7 @@ export async function scrapeController(req: Request, res: Response) {
} }
logJob({ logJob({
job_id: jobId,
success: result.success, success: result.success,
message: result.error, message: result.error,
num_docs: 1, num_docs: 1,