From aa54fd16582e620025d5cd575c42e8884be9142d Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 2 Mar 2025 17:46:12 -0300 Subject: [PATCH] Nick: only new activities/sources in the callback --- apps/python-sdk/firecrawl/__init__.py | 2 +- apps/python-sdk/firecrawl/firecrawl.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py index 5dc83686..fce2416f 100644 --- a/apps/python-sdk/firecrawl/__init__.py +++ b/apps/python-sdk/firecrawl/__init__.py @@ -13,7 +13,7 @@ import os from .firecrawl import FirecrawlApp # noqa -__version__ = "1.13.1" +__version__ = "1.13.2" # Define the logger for the Firecrawl project logger: logging.Logger = logging.getLogger("firecrawl") diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index a4c69638..f50e5bc0 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -1128,16 +1128,23 @@ class FirecrawlApp: return response job_id = response['id'] + last_activity_count = 0 + last_source_count = 0 + while True: status = self.check_deep_research_status(job_id) if on_activity and 'activities' in status: - for activity in status['activities']: + new_activities = status['activities'][last_activity_count:] + for activity in new_activities: on_activity(activity) + last_activity_count = len(status['activities']) if on_source and 'sources' in status: - for source in status['sources']: + new_sources = status['sources'][last_source_count:] + for source in new_sources: on_source(source) + last_source_count = len(status['sources']) if status['status'] == 'completed': return status