fix(v1): js-sdk fixed crawl type

This commit is contained in:
rafaelsideguide 2024-08-28 16:27:29 -03:00
parent 70bff7f8fb
commit c3158b0f98
4 changed files with 40 additions and 37 deletions

View File

@ -1,34 +1,39 @@
import FirecrawlApp, { ScrapeResponse } from './firecrawl/src/index' //'@mendable/firecrawl-js'; import FirecrawlApp, { CrawlStatusResponse, CrawlResponse } from '@mendable/firecrawl-js';
import { CrawlStatusResponse } from './firecrawl/src/index';
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"}); const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
// Scrape a website: const main = async () => {
const scrapeResult = await app.scrapeUrl('firecrawl.dev');
if (scrapeResult) { // Scrape a website:
console.log(scrapeResult.markdown) const scrapeResult = await app.scrapeUrl('firecrawl.dev');
}
// Crawl a website: if (scrapeResult) {
const crawlResult = await app.crawlUrl('mendable.ai', {crawlerOptions: {excludePaths: ['blog/*'], limit: 5}}, false); console.log(scrapeResult.markdown)
console.log(crawlResult)
const jobId: string = await crawlResult['jobId'];
console.log(jobId);
let job: CrawlStatusResponse;
while (true) {
job = await app.checkCrawlStatus(jobId) as CrawlStatusResponse;
if (job.status === 'completed') {
break;
} }
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
// Crawl a website:
// @ts-ignore
const crawlResult = await app.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5}, false) as CrawlResponse;
console.log(crawlResult)
const id = crawlResult.id;
console.log(id);
let checkStatus: CrawlStatusResponse;
while (true) {
checkStatus = await app.checkCrawlStatus(id);
if (checkStatus.status === 'completed') {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second
}
if (checkStatus.data) {
console.log(checkStatus.data[0].markdown);
}
const mapResult = await app.mapUrl('https://firecrawl.dev');
console.log(mapResult)
} }
if (job.data) { main()
console.log(job.data[0].markdown);
}
const mapResult = await app.mapUrl('https://firecrawl.dev');
console.log(mapResult)

View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "1.0.2", "version": "1.0.3",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "build/cjs/index.js", "main": "build/cjs/index.js",
"types": "types/index.d.ts", "types": "types/index.d.ts",

View File

@ -165,16 +165,14 @@ export interface ScrapeResponseV0 {
* Includes options for both scraping and mapping during a crawl. * Includes options for both scraping and mapping during a crawl.
*/ */
export interface CrawlParams { export interface CrawlParams {
includePaths?: string[];
excludePaths?: string[];
maxDepth?: number;
limit?: number;
allowBackwardLinks?: boolean;
allowExternalLinks?: boolean;
ignoreSitemap?: boolean;
scrapeOptions?: ScrapeParams; scrapeOptions?: ScrapeParams;
crawlerOptions?: {
includePaths?: string[]
excludePaths?: string[]
maxDepth?: number
limit?: number
allowBackwardLinks?: boolean
allowExternalLinks?: boolean
ignoreSitemap?: boolean
};
} }
/** /**

View File

@ -11,7 +11,7 @@
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"@mendable/firecrawl-js": "^0.0.36", "@mendable/firecrawl-js": "^1.0.3",
"axios": "^1.6.8", "axios": "^1.6.8",
"ts-node": "^10.9.2", "ts-node": "^10.9.2",
"typescript": "^5.4.5", "typescript": "^5.4.5",