AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 15:40:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scrape): pass extractorOptions

Browse Source
This commit is contained in:
Gergo Moricz 2024-08-06 17:15:19 +02:00
parent 8e0aa69603
commit 8566ece700
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/main/runWebScraper.ts
View File

@ -26,6 +26,7 @@ export async function startWebScraperPipeline({
url: job.data.url,
mode: job.data.mode,
crawlerOptions: job.data.crawlerOptions,
extractorOptions: job.data.extractorOptions,
pageOptions: job.data.pageOptions,
inProgress: (progress) => {
Logger.debug(`🐂 Job in progress ${job.id}`);
@ -55,6 +56,7 @@ export async function runWebScraper({
mode,
crawlerOptions,
pageOptions,
extractorOptions,
inProgress,
onSuccess,
onError,
@ -68,6 +70,7 @@ export async function runWebScraper({
jobId: bull_job_id,
mode: mode,
urls: [url],
extractorOptions,
crawlerOptions: crawlerOptions,
pageOptions: pageOptions,
bullJobId: bull_job_id,
@ -77,6 +80,7 @@ export async function runWebScraper({
jobId: bull_job_id,
mode: mode,
urls: url.split(","),
extractorOptions,
crawlerOptions: crawlerOptions,
pageOptions: pageOptions,
});

1
apps/api/src/types.ts
View File

@ -35,6 +35,7 @@ export interface RunWebScraperParams {
mode: Mode;
crawlerOptions: any;
pageOptions?: any;
extractorOptions?: any;
inProgress: (progress: any) => void;
onSuccess: (result: any, mode: string) => void;
onError: (error: Error) => void;