AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-20 05:39:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(extract-status): be able to get extract status even after TTL lapses (#1599)

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-23 22:33:09 +02:00 committed by GitHub
parent 8389a1a78d
commit a3145ccacc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 31 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -33,11 +33,12 @@ export type PseudoJob<T> = {
timestamp: number, timestamp: number,
data: { data: {
scrapeOptions: any, scrapeOptions: any,
teamId?: string,
}, },
failedReason?: string, failedReason?: string,
} }
export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null } export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string}
export async function getJob(id: string): Promise<PseudoJob<any> | null> { export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([ const [bullJob, dbJob, gcsJob] = await Promise.all([

43
apps/api/src/controllers/v1/extract-status.ts
View File

@ -18,10 +18,11 @@ export async function getExtractJob(id: string): Promise<PseudoJob<ExtractResult
const job: PseudoJob<any> = { const job: PseudoJob<any> = {
id, id,
getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"), getState: bullJob ? bullJob.getState.bind(bullJob) : (() => dbJob!.success ? "completed" : "failed"),
returnvalue: data, returnvalue: data,
data: { data: {
scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options, scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
teamId: bullJob ? bullJob.data.teamId : dbJob!.team_id,
}, },
timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(), timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),
failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined, failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
@ -36,7 +37,9 @@ export async function extractStatusController(
) { ) {
const extract = await getExtract(req.params.jobId); const extract = await getExtract(req.params.jobId);
if (!extract) { let status = extract?.status;
if (extract && extract.team_id !== req.auth.team_id) {
return res.status(404).json({ return res.status(404).json({
success: false, success: false,
error: "Extract job not found", error: "Extract job not found",
@ -45,34 +48,46 @@ export async function extractStatusController(
let data: ExtractResult | [] = []; let data: ExtractResult | [] = [];
if (extract.status === "completed") { if (!extract || extract.status === "completed") {
const jobData = await getExtractJob(req.params.jobId); const jobData = await getExtractJob(req.params.jobId);
if (!jobData) { if ((!jobData && !extract) || (jobData && jobData.data.teamId !== req.auth.team_id)) {
return res.status(404).json({ return res.status(404).json({
success: false, success: false,
error: "Job not found", error: "Extract job not found",
}); });
} }
if (!jobData.returnvalue) { if (jobData) {
const jobStatus = await jobData.getState();
if (jobStatus === "completed") {
status = "completed";
} else if (jobStatus === "failed") {
status = "failed";
} else {
status = "processing";
}
}
if (!jobData?.returnvalue) {
// if we got in the split-second where the redis is updated but the bull isn't // if we got in the split-second where the redis is updated but the bull isn't
// just pretend it's still processing - MG // just pretend it's still processing - MG
extract.status = "processing"; status = "processing";
} else { } else {
data = jobData.returnvalue ?? []; data = jobData.returnvalue ?? [];
} }
} }
return res.status(200).json({ return res.status(200).json({
success: extract.status === "failed" ? false : true, success: status === "failed" ? false : true,
data, data,
status: extract.status, status,
error: extract?.error ?? undefined, error: extract?.error ?? undefined,
expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(), expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),
steps: extract.showSteps ? extract.steps : undefined, steps: extract?.showSteps ? extract.steps : undefined,
llmUsage: extract.showLLMUsage ? extract.llmUsage : undefined, llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,
sources: extract.showSources ? extract.sources : undefined, sources: extract?.showSources ? extract.sources : undefined,
costTracking: extract.showCostTracking ? extract.costTracking : undefined, costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
sessionIds: extract.sessionIds ? extract.sessionIds : undefined, sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
}); });
} }