AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-06-04 11:24:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed empty data with next causing infinite loop

Browse Source
This commit is contained in:
rafaelmmiller 2025-01-03 13:56:39 -03:00
parent bafcc008bc
commit 55dad5ea13
2 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/js-sdk/firecrawl/src/index.ts
View File

@ -565,6 +565,10 @@ export default class FirecrawlApp {
if ("data" in statusData) { if ("data" in statusData) {
let data = statusData.data; let data = statusData.data;
while (typeof statusData === 'object' && 'next' in statusData) { while (typeof statusData === 'object' && 'next' in statusData) {
if (data.length === 0) {
console.warn("Expected 'data' is missing.")
break
}
statusData = (await this.getRequest(statusData.next, headers)).data; statusData = (await this.getRequest(statusData.next, headers)).data;
data = data.concat(statusData.data); data = data.concat(statusData.data);
} }
@ -812,6 +816,10 @@ export default class FirecrawlApp {
if ("data" in statusData) { if ("data" in statusData) {
let data = statusData.data; let data = statusData.data;
while (typeof statusData === 'object' && 'next' in statusData) { while (typeof statusData === 'object' && 'next' in statusData) {
if (data.length === 0) {
console.warn("Expected 'data' is missing.")
break
}
statusData = (await this.getRequest(statusData.next, headers)).data; statusData = (await this.getRequest(statusData.next, headers)).data;
data = data.concat(statusData.data); data = data.concat(statusData.data);
} }
@ -995,6 +1003,10 @@ export default class FirecrawlApp {
if ("data" in statusData) { if ("data" in statusData) {
let data = statusData.data; let data = statusData.data;
while (typeof statusData === 'object' && 'next' in statusData) { while (typeof statusData === 'object' && 'next' in statusData) {
if (data.length === 0) {
console.warn("Expected 'data' is missing.")
break
}
statusResponse = await this.getRequest(statusData.next, headers); statusResponse = await this.getRequest(statusData.next, headers);
statusData = statusResponse.data; statusData = statusResponse.data;
data = data.concat(statusData.data); data = data.concat(statusData.data);

9
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -250,6 +250,9 @@ class FirecrawlApp:
if 'data' in status_data: if 'data' in status_data:
data = status_data['data'] data = status_data['data']
while 'next' in status_data: while 'next' in status_data:
if len(status_data['data']) == 0:
logger.warning("Expected 'data' is missing.")
break
next_url = status_data.get('next') next_url = status_data.get('next')
if not next_url: if not next_url:
logger.warning("Expected 'next' URL is missing.") logger.warning("Expected 'next' URL is missing.")
@ -467,6 +470,9 @@ class FirecrawlApp:
if 'data' in status_data: if 'data' in status_data:
data = status_data['data'] data = status_data['data']
while 'next' in status_data: while 'next' in status_data:
if len(status_data['data']) == 0:
logger.warning("Expected 'data' is missing.")
break
next_url = status_data.get('next') next_url = status_data.get('next')
if not next_url: if not next_url:
logger.warning("Expected 'next' URL is missing.") logger.warning("Expected 'next' URL is missing.")
@ -685,6 +691,9 @@ class FirecrawlApp:
if 'data' in status_data: if 'data' in status_data:
data = status_data['data'] data = status_data['data']
while 'next' in status_data: while 'next' in status_data:
if len(status_data['data']) == 0:
logger.warning("Expected 'data' is missing.")
break
status_response = self._get_request(status_data['next'], headers) status_response = self._get_request(status_data['next'], headers)
status_data = status_response.json() status_data = status_response.json()
data.extend(status_data.get('data', [])) data.extend(status_data.get('data', []))