AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 14:19:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: totalCount -> total, completedCount -> completed

Browse Source
This commit is contained in:
Nicolas 2024-08-28 14:46:53 -03:00
parent 2a56f89f50
commit 4edd9a34c7
10 changed files with 31 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -76,7 +76,7 @@ curl -X GET https://api.firecrawl.dev/v1/crawl/123-456-789 \
```json ```json
{ {
"status": "completed", "status": "completed",
"totalCount": 36, "total": 36,
"creditsUsed": 36, "creditsUsed": 36,
"expiresAt": "2024-00-00T00:00:00.000Z", "expiresAt": "2024-00-00T00:00:00.000Z",
"data": [ "data": [

3
apps/api/src/controllers/v1/crawl-status-ws.ts
View File

@ -102,7 +102,8 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
type: "catchup", type: "catchup",
data: { data: {
status, status,
totalCount: jobIDs.length, total: jobIDs.length,
completed: doneJobIDs.length,
creditsUsed: jobIDs.length, creditsUsed: jobIDs.length,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(), expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
data: data.map(x => legacyDocumentConverter(x)), data: data.map(x => legacyDocumentConverter(x)),

3
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -102,7 +102,8 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
res.status(200).json({ res.status(200).json({
status, status,
totalCount: jobIDs.length, completed: doneJobsLength,
total: jobIDs.length,
creditsUsed: jobIDs.length, creditsUsed: jobIDs.length,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(), expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
next: next:

3
apps/api/src/controllers/v1/types.ts
View File

@ -221,7 +221,8 @@ export type CrawlStatusResponse =
| ErrorResponse | ErrorResponse
| { | {
status: "scraping" | "completed" | "failed" | "cancelled"; status: "scraping" | "completed" | "failed" | "cancelled";
totalCount: number; completed: number;
total: number;
creditsUsed: number; creditsUsed: number;
expiresAt: string; expiresAt: string;
next?: string; next?: string;

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "0.0.36", "version": "1.0.0",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "build/cjs/index.js", "main": "build/cjs/index.js",
"types": "types/index.d.ts", "types": "types/index.d.ts",

16
apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
View File

@ -129,8 +129,8 @@ describe('FirecrawlApp E2E Tests', () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.crawlUrl('https://roastmywebsite.ai', {}, true, 30) as CrawlStatusResponse; const response = await app.crawlUrl('https://roastmywebsite.ai', {}, true, 30) as CrawlStatusResponse;
expect(response).not.toBeNull(); expect(response).not.toBeNull();
expect(response).toHaveProperty("totalCount"); expect(response).toHaveProperty("total");
expect(response.totalCount).toBeGreaterThan(0); expect(response.total).toBeGreaterThan(0);
expect(response).toHaveProperty("creditsUsed"); expect(response).toHaveProperty("creditsUsed");
expect(response.creditsUsed).toBeGreaterThan(0); expect(response.creditsUsed).toBeGreaterThan(0);
expect(response).toHaveProperty("expiresAt"); expect(response).toHaveProperty("expiresAt");
@ -175,8 +175,8 @@ describe('FirecrawlApp E2E Tests', () => {
} }
} as CrawlParams, true, 30) as CrawlStatusResponse; } as CrawlParams, true, 30) as CrawlStatusResponse;
expect(response).not.toBeNull(); expect(response).not.toBeNull();
expect(response).toHaveProperty("totalCount"); expect(response).toHaveProperty("total");
expect(response.totalCount).toBeGreaterThan(0); expect(response.total).toBeGreaterThan(0);
expect(response).toHaveProperty("creditsUsed"); expect(response).toHaveProperty("creditsUsed");
expect(response.creditsUsed).toBeGreaterThan(0); expect(response.creditsUsed).toBeGreaterThan(0);
expect(response).toHaveProperty("expiresAt"); expect(response).toHaveProperty("expiresAt");
@ -231,12 +231,12 @@ describe('FirecrawlApp E2E Tests', () => {
expect(statusResponse).not.toHaveProperty("partial_data"); // v0 expect(statusResponse).not.toHaveProperty("partial_data"); // v0
expect(statusResponse).not.toHaveProperty("current"); // v0 expect(statusResponse).not.toHaveProperty("current"); // v0
expect(statusResponse).toHaveProperty("data"); expect(statusResponse).toHaveProperty("data");
expect(statusResponse).toHaveProperty("totalCount"); expect(statusResponse).toHaveProperty("total");
expect(statusResponse).toHaveProperty("creditsUsed"); expect(statusResponse).toHaveProperty("creditsUsed");
expect(statusResponse).toHaveProperty("expiresAt"); expect(statusResponse).toHaveProperty("expiresAt");
expect(statusResponse).toHaveProperty("status"); expect(statusResponse).toHaveProperty("status");
expect(statusResponse).toHaveProperty("next"); expect(statusResponse).toHaveProperty("next");
expect(statusResponse.totalCount).toBeGreaterThan(0); expect(statusResponse.total).toBeGreaterThan(0);
expect(statusResponse.creditsUsed).toBeGreaterThan(0); expect(statusResponse.creditsUsed).toBeGreaterThan(0);
expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now()); expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
expect(statusResponse.status).toBe("scraping"); expect(statusResponse.status).toBe("scraping");
@ -246,8 +246,8 @@ describe('FirecrawlApp E2E Tests', () => {
} }
expect(statusResponse).not.toBeNull(); expect(statusResponse).not.toBeNull();
expect(statusResponse).toHaveProperty("totalCount"); expect(statusResponse).toHaveProperty("total");
expect(statusResponse.totalCount).toBeGreaterThan(0); expect(statusResponse.total).toBeGreaterThan(0);
expect(statusResponse).toHaveProperty("creditsUsed"); expect(statusResponse).toHaveProperty("creditsUsed");
expect(statusResponse.creditsUsed).toBeGreaterThan(0); expect(statusResponse.creditsUsed).toBeGreaterThan(0);
expect(statusResponse).toHaveProperty("expiresAt"); expect(statusResponse).toHaveProperty("expiresAt");

6
apps/js-sdk/firecrawl/src/index.ts
View File

@ -235,7 +235,8 @@ export interface CrawlResponseV0 {
*/ */
export interface CrawlStatusResponse { export interface CrawlStatusResponse {
success: boolean; success: boolean;
totalCount: number; total: number;
completed: number;
creditsUsed: number; creditsUsed: number;
expiresAt: Date; expiresAt: Date;
status: "scraping" | "completed" | "failed"; status: "scraping" | "completed" | "failed";
@ -530,7 +531,8 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
return ({ return ({
success: true, success: true,
status: response.data.status, status: response.data.status,
totalCount: response.data.totalCount, total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed, creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt), expiresAt: new Date(response.data.expiresAt),
next: response.data.next, next: response.data.next,

2
apps/python-sdk/firecrawl/__init__.py
View File

@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp from .firecrawl import FirecrawlApp
__version__ = "0.0.16" __version__ = "1.0.0"
# Define the logger for the Firecrawl project # Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl") logger: logging.Logger = logging.getLogger("firecrawl")

16
apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
View File

@ -147,8 +147,8 @@ def test_crawl_url_wait_for_completion_e2e():
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY) app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30) response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30)
assert response is not None assert response is not None
assert 'totalCount' in response assert 'total' in response
assert response['totalCount'] > 0 assert response['total'] > 0
assert 'creditsUsed' in response assert 'creditsUsed' in response
assert response['creditsUsed'] > 0 assert response['creditsUsed'] > 0
assert 'expiresAt' in response assert 'expiresAt' in response
@ -192,8 +192,8 @@ def test_crawl_url_with_options_and_wait_for_completion():
} }
}, True, 30) }, True, 30)
assert response is not None assert response is not None
assert 'totalCount' in response assert 'total' in response
assert response['totalCount'] > 0 assert response['total'] > 0
assert 'creditsUsed' in response assert 'creditsUsed' in response
assert response['creditsUsed'] > 0 assert response['creditsUsed'] > 0
assert 'expiresAt' in response assert 'expiresAt' in response
@ -247,12 +247,12 @@ def test_check_crawl_status_e2e():
assert 'partial_data' not in status_response assert 'partial_data' not in status_response
assert 'current' not in status_response assert 'current' not in status_response
assert 'data' in status_response assert 'data' in status_response
assert 'totalCount' in status_response assert 'total' in status_response
assert 'creditsUsed' in status_response assert 'creditsUsed' in status_response
assert 'expiresAt' in status_response assert 'expiresAt' in status_response
assert 'status' in status_response assert 'status' in status_response
assert 'next' in status_response assert 'next' in status_response
assert status_response['totalCount'] > 0 assert status_response['total'] > 0
assert status_response['creditsUsed'] > 0 assert status_response['creditsUsed'] > 0
assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now() assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
assert status_response['status'] == 'scraping' assert status_response['status'] == 'scraping'
@ -261,8 +261,8 @@ def test_check_crawl_status_e2e():
checks += 1 checks += 1
assert status_response is not None assert status_response is not None
assert 'totalCount' in status_response assert 'total' in status_response
assert status_response['totalCount'] > 0 assert status_response['total'] > 0
assert 'creditsUsed' in status_response assert 'creditsUsed' in status_response
assert status_response['creditsUsed'] > 0 assert status_response['creditsUsed'] > 0
assert 'expiresAt' in status_response assert 'expiresAt' in status_response

3
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -211,7 +211,8 @@ class FirecrawlApp:
return { return {
'success': True, 'success': True,
'status': data.get('status'), 'status': data.get('status'),
'totalCount': data.get('totalCount'), 'total': data.get('total'),
'completed': data.get('completed'),
'creditsUsed': data.get('creditsUsed'), 'creditsUsed': data.get('creditsUsed'),
'expiresAt': data.get('expiresAt'), 'expiresAt': data.get('expiresAt'),
'next': data.get('next'), 'next': data.get('next'),