diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 7af59b45..166deaf9 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -26,6 +26,7 @@ export async function startWebScraperPipeline({ url: job.data.url, mode: job.data.mode, crawlerOptions: job.data.crawlerOptions, + extractorOptions: job.data.extractorOptions, pageOptions: job.data.pageOptions, inProgress: (progress) => { Logger.debug(`🐂 Job in progress ${job.id}`); @@ -55,6 +56,7 @@ export async function runWebScraper({ mode, crawlerOptions, pageOptions, + extractorOptions, inProgress, onSuccess, onError, @@ -68,6 +70,7 @@ export async function runWebScraper({ jobId: bull_job_id, mode: mode, urls: [url], + extractorOptions, crawlerOptions: crawlerOptions, pageOptions: pageOptions, bullJobId: bull_job_id, @@ -77,6 +80,7 @@ export async function runWebScraper({ jobId: bull_job_id, mode: mode, urls: url.split(","), + extractorOptions, crawlerOptions: crawlerOptions, pageOptions: pageOptions, }); diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index fa746688..3b28b765 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -35,6 +35,7 @@ export interface RunWebScraperParams { mode: Mode; crawlerOptions: any; pageOptions?: any; + extractorOptions?: any; inProgress: (progress: any) => void; onSuccess: (result: any, mode: string) => void; onError: (error: Error) => void;