AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 07:08:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(js-sdk/batch/scrape): add ignoreInvalidURLs option

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-14 01:16:09 +01:00
parent 4b5014d7fe
commit 9cc6576571
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
apps/js-sdk/firecrawl/src/index.ts
View File

@ -183,6 +183,7 @@ export interface BatchScrapeResponse {
url?: string;
success: true;
error?: string;
invalidURLs?: string[];
}
/**
@ -576,9 +577,10 @@ export default class FirecrawlApp {
pollInterval: number = 2,
idempotencyKey?: string,
webhook?: CrawlParams["webhook"],
ignoreInvalidURLs?: boolean,
): Promise<BatchScrapeStatusResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
let jsonData: any = { urls, ...params };
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };
if (jsonData?.extract?.schema) {
let schema = jsonData.extract.schema;
@ -621,10 +623,12 @@ export default class FirecrawlApp {
async asyncBatchScrapeUrls(
urls: string[],
params?: ScrapeParams,
idempotencyKey?: string
idempotencyKey?: string,
webhook?: CrawlParams["webhook"],
ignoreInvalidURLs?: boolean,
): Promise<BatchScrapeResponse | ErrorResponse> {
const headers = this.prepareHeaders(idempotencyKey);
let jsonData: any = { urls, ...(params ?? {}) };
let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };
try {
const response: AxiosResponse = await this.postRequest(
this.apiUrl + `/v1/batch/scrape`,
@ -657,8 +661,10 @@ export default class FirecrawlApp {
urls: string[],
params?: ScrapeParams,
idempotencyKey?: string,
webhook?: CrawlParams["webhook"],
ignoreInvalidURLs?: boolean,
) {
const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
if (crawl.success && crawl.id) {
const id = crawl.id;