AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-08 13:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

revert bad hotfixes

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-07 16:34:25 +02:00
parent b7c01dcb9b
commit fe9fdb578b
4 changed files with 4 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/api/src/lib/scrape-events.ts
View File

@ -59,12 +59,6 @@ export class ScrapeEvents {
try { try {
const previousLog = (await supabase.from("scrape_events").select().eq("id", logId).single()).data as any; const previousLog = (await supabase.from("scrape_events").select().eq("id", logId).single()).data as any;
if (!previousLog) {
Logger.warn("Previous log not found.");
return;
}
await supabase.from("scrape_events").update({ await supabase.from("scrape_events").update({
content: { content: {
...previousLog.content, ...previousLog.content,

2
apps/api/src/main/runWebScraper.ts
View File

@ -131,7 +131,7 @@ const saveJob = async (job: Job, result: any) => {
if (error) throw new Error(error.message); if (error) throw new Error(error.message);
try { try {
// await job.moveToCompleted(null, false, false); await job.moveToCompleted(null, false, false);
} catch (error) { } catch (error) {
// I think the job won't exist here anymore // I think the job won't exist here anymore
} }

2
apps/api/src/scraper/WebScraper/index.ts
View File

@ -94,7 +94,7 @@ export class WebScraperDataProvider {
const jobStatus = await job.getState(); const jobStatus = await job.getState();
if (jobStatus === "failed") { if (jobStatus === "failed") {
Logger.info( Logger.info(
"Job " + job.id + " has failed or has been cancelled by the user. Stopping the job..." "Job has failed or has been cancelled by the user. Stopping the job..."
); );
return [] as Document[]; return [] as Document[];
} }

9
apps/api/src/services/queue-worker.ts
View File

@ -22,11 +22,6 @@ const wsq = getWebScraperQueue();
async function processJob(job: Job, done) { async function processJob(job: Job, done) {
Logger.info(`🐂 Worker taking job ${job.id}`); Logger.info(`🐂 Worker taking job ${job.id}`);
const lockInterval = setInterval(() => {
Logger.info(`🐂 Renewing lock for ${job.id}`);
job.extendLock(60000);
}, 15000);
try { try {
job.progress({ job.progress({
current: 1, current: 1,
@ -67,8 +62,7 @@ async function processJob(job: Job, done) {
origin: job.data.origin, origin: job.data.origin,
}); });
Logger.info(`🐂 Job done ${job.id}`); Logger.info(`🐂 Job done ${job.id}`);
clearInterval(lockInterval); done(null, data);
done(null, null);
} catch (error) { } catch (error) {
Logger.error(`🐂 Job errored ${job.id} - ${error}`); Logger.error(`🐂 Job errored ${job.id} - ${error}`);
if (await getWebScraperQueue().isPaused(false)) { if (await getWebScraperQueue().isPaused(false)) {
@ -114,7 +108,6 @@ async function processJob(job: Job, done) {
pageOptions: job.data.pageOptions, pageOptions: job.data.pageOptions,
origin: job.data.origin, origin: job.data.origin,
}); });
clearInterval(lockInterval);
done(null, data); done(null, data);
} }
} }