AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-15 12:25:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat-SDK/added crawl id to ws

Browse Source
This commit is contained in:
rafaelmmiller 2024-12-18 18:36:04 -03:00
parent e899ecbe44
commit 19246f6289
4 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "1.9.4", "version": "1.9.5",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.ts", "types": "dist/index.d.ts",

15
apps/js-sdk/firecrawl/src/index.ts
View File

@ -934,9 +934,11 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
private ws: WebSocket; private ws: WebSocket;
public data: FirecrawlDocument<undefined>[]; public data: FirecrawlDocument<undefined>[];
public status: CrawlStatusResponse["status"]; public status: CrawlStatusResponse["status"];
public id: string;
constructor(id: string, app: FirecrawlApp) { constructor(id: string, app: FirecrawlApp) {
super(); super();
this.id = id;
this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey); this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
this.status = "scraping"; this.status = "scraping";
this.data = []; this.data = [];
@ -967,6 +969,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
detail: { detail: {
status: this.status, status: this.status,
data: this.data, data: this.data,
id: this.id,
}, },
})); }));
} else if (msg.type === "error") { } else if (msg.type === "error") {
@ -976,6 +979,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
status: this.status, status: this.status,
data: this.data, data: this.data,
error: msg.error, error: msg.error,
id: this.id,
}, },
})); }));
} else if (msg.type === "catchup") { } else if (msg.type === "catchup") {
@ -983,12 +987,18 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
this.data.push(...(msg.data.data ?? [])); this.data.push(...(msg.data.data ?? []));
for (const doc of this.data) { for (const doc of this.data) {
this.dispatchTypedEvent("document", new CustomEvent("document", { this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: doc, detail: {
...doc,
id: this.id,
},
})); }));
} }
} else if (msg.type === "document") { } else if (msg.type === "document") {
this.dispatchTypedEvent("document", new CustomEvent("document", { this.dispatchTypedEvent("document", new CustomEvent("document", {
detail: msg.data, detail: {
...msg.data,
id: this.id,
},
})); }));
} }
} }
@ -1015,6 +1025,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
status: this.status, status: this.status,
data: this.data, data: this.data,
error: "WebSocket error", error: "WebSocket error",
id: this.id,
}, },
})); }));
}).bind(this); }).bind(this);

2
apps/python-sdk/firecrawl/__init__.py
View File

@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp # noqa from .firecrawl import FirecrawlApp # noqa
__version__ = "1.6.4" __version__ = "1.6.5"
# Define the logger for the Firecrawl project # Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl") logger: logging.Logger = logging.getLogger("firecrawl")

8
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -704,15 +704,15 @@ class CrawlWatcher:
async def _handle_message(self, msg: Dict[str, Any]): async def _handle_message(self, msg: Dict[str, Any]):
if msg['type'] == 'done': if msg['type'] == 'done':
self.status = 'completed' self.status = 'completed'
self.dispatch_event('done', {'status': self.status, 'data': self.data}) self.dispatch_event('done', {'status': self.status, 'data': self.data, 'id': self.id})
elif msg['type'] == 'error': elif msg['type'] == 'error':
self.status = 'failed' self.status = 'failed'
self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error']}) self.dispatch_event('error', {'status': self.status, 'data': self.data, 'error': msg['error'], 'id': self.id})
elif msg['type'] == 'catchup': elif msg['type'] == 'catchup':
self.status = msg['data']['status'] self.status = msg['data']['status']
self.data.extend(msg['data'].get('data', [])) self.data.extend(msg['data'].get('data', []))
for doc in self.data: for doc in self.data:
self.dispatch_event('document', doc) self.dispatch_event('document', {'data': doc, 'id': self.id})
elif msg['type'] == 'document': elif msg['type'] == 'document':
self.data.append(msg['data']) self.data.append(msg['data'])
self.dispatch_event('document', msg['data']) self.dispatch_event('document', {'data': msg['data'], 'id': self.id})