AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-04 07:11:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: offload crawl results to the DB

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-13 21:40:59 +02:00
parent 4bbc9db1df
commit 8dbac0268c
2 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
apps/api/src/controllers/crawl-status.ts
View File

@ -4,6 +4,7 @@ import { RateLimiterMode } from "../../src/types";
import { getScrapeQueue } from "../../src/services/queue-service";
import { Logger } from "../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../src/lib/crawl-redis";
import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
export async function crawlStatusController(req: Request, res: Response) {
try {
@ -21,18 +22,25 @@ export async function crawlStatusController(req: Request, res: Response) {
return res.status(404).json({ error: "Job not found" });
}
if (sc.team_id !== team_id) {
return res.status(403).json({ error: "Forbidden" });
}
const jobIDs = await getCrawlJobs(req.params.jobId);
// let data = job.returnvalue;
// if (process.env.USE_DB_AUTHENTICATION === "true") {
// const supabaseData = await supabaseGetJobById(req.params.jobId);
const jobs = await Promise.all(jobIDs.map(async x => {
const job = await getScrapeQueue().getJob(x);
if (process.env.USE_DB_AUTHENTICATION === "true") {
const supabaseData = await supabaseGetJobById(job.id);
// if (supabaseData) {
// data = supabaseData.docs;
// }
// }
if (supabaseData) {
job.returnvalue = supabaseData.docs;
}
}
const jobs = await Promise.all(jobIDs.map(x => getScrapeQueue().getJob(x)));
return job;
}));
const jobStatuses = await Promise.all(jobs.map(x => x.getState()));
const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active";

6
apps/api/src/services/queue-worker.ts
View File

@ -53,7 +53,11 @@ const processJobInternal = async (token: string, job: Job) => {
try {
const result = await processJob(job, token);
try{
await job.moveToCompleted(result.docs, token, false);
if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {
await job.moveToCompleted(null, token, false);
} else {
await job.moveToCompleted(result.docs, token, false);
}
}catch(e){
}
} catch (error) {