AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 19:36:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrape): pass extractorOptions

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-06 17:15:19 +02:00
parent 8e0aa69603
commit 8566ece700
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/main/runWebScraper.ts
View File

@ -26,6 +26,7 @@ export async function startWebScraperPipeline({
url: job.data.url, url: job.data.url,
mode: job.data.mode, mode: job.data.mode,
crawlerOptions: job.data.crawlerOptions, crawlerOptions: job.data.crawlerOptions,
extractorOptions: job.data.extractorOptions,
pageOptions: job.data.pageOptions, pageOptions: job.data.pageOptions,
inProgress: (progress) => { inProgress: (progress) => {
Logger.debug(`🐂 Job in progress ${job.id}`); Logger.debug(`🐂 Job in progress ${job.id}`);
@ -55,6 +56,7 @@ export async function runWebScraper({
mode, mode,
crawlerOptions, crawlerOptions,
pageOptions, pageOptions,
extractorOptions,
inProgress, inProgress,
onSuccess, onSuccess,
onError, onError,
@ -68,6 +70,7 @@ export async function runWebScraper({
jobId: bull_job_id, jobId: bull_job_id,
mode: mode, mode: mode,
urls: [url], urls: [url],
extractorOptions,
crawlerOptions: crawlerOptions, crawlerOptions: crawlerOptions,
pageOptions: pageOptions, pageOptions: pageOptions,
bullJobId: bull_job_id, bullJobId: bull_job_id,
@ -77,6 +80,7 @@ export async function runWebScraper({
jobId: bull_job_id, jobId: bull_job_id,
mode: mode, mode: mode,
urls: url.split(","), urls: url.split(","),
extractorOptions,
crawlerOptions: crawlerOptions, crawlerOptions: crawlerOptions,
pageOptions: pageOptions, pageOptions: pageOptions,
}); });

1
apps/api/src/types.ts
View File

@ -35,6 +35,7 @@ export interface RunWebScraperParams {
mode: Mode; mode: Mode;
crawlerOptions: any; crawlerOptions: any;
pageOptions?: any; pageOptions?: any;
extractorOptions?: any;
inProgress: (progress: any) => void; inProgress: (progress: any) => void;
onSuccess: (result: any, mode: string) => void; onSuccess: (result: any, mode: string) => void;
onError: (error: Error) => void; onError: (error: Error) => void;