AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 13:45:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update queue-worker.ts

Browse Source
This commit is contained in:
Nicolas 2024-06-24 16:51:32 -03:00 committed by GitHub
parent 3ebdf93342
commit 08c1fa799b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/services/queue-worker.ts
View File

@ -14,7 +14,7 @@ if(process.env.ENV === 'production') {
getWebScraperQueue().process( getWebScraperQueue().process(
Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)), Math.floor(Number(process.env.NUM_WORKERS_PER_QUEUE ?? 8)),
async function (job, done) { async function (job, done) {
console.log('getWebScraperQueue - start')
try { try {
job.progress({ job.progress({
current: 1, current: 1,
@ -23,13 +23,10 @@ getWebScraperQueue().process(
current_url: "", current_url: "",
}); });
const start = Date.now(); const start = Date.now();
console.log('getWebScraperQueue - startWebScraperPipeline')
const { success, message, docs } = await startWebScraperPipeline({ job }); const { success, message, docs } = await startWebScraperPipeline({ job });
console.log('getWebScraperQueue - startWebScraperPipeline - done')
const end = Date.now(); const end = Date.now();
const timeTakenInSeconds = (end - start) / 1000; const timeTakenInSeconds = (end - start) / 1000;
console.log('docs.length:', docs.length)
const data = { const data = {
success: success, success: success,
result: { result: {