AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-08 18:39:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(queue-worker): only do redirect handling logic on crawls, not batch scrape

Browse Source
This commit is contained in:
Gergő Móricz 2025-01-04 16:59:35 +01:00
parent d48ddb8820
commit b92a4eb79b
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/controllers/v1/batch-scrape.ts
View File

@ -61,7 +61,7 @@ export async function batchScrapeController(
}
logger.debug("Batch scrape " + id + " starting", {
urlsLength: urls,
urlsLength: urls.length,
appendToId: req.body.appendToId,
account: req.account,
});

3
apps/api/src/services/queue-worker.ts
View File

@ -710,7 +710,8 @@ async function processJob(job: Job & { id: string }, token: string) {
doc.metadata.url !== undefined &&
doc.metadata.sourceURL !== undefined &&
normalizeURL(doc.metadata.url, sc) !==
normalizeURL(doc.metadata.sourceURL, sc)
normalizeURL(doc.metadata.sourceURL, sc) &&
job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
) {
const crawler = crawlToCrawler(job.data.crawl_id, sc);
if (