Enhance error handling in E2E tests and introduce CrawlWatcher tests

- Updated error messages in E2E tests to provide clearer feedback for blocked URLs and invalid API keys.
- Added new test suite for CrawlWatcher to ensure proper instantiation and error handling when WebSocket is unavailable.
- Improved test conditions for URL scraping and crawling to reflect updated error responses.

apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts

@@ -58,7 +58,7 @@ describe("E2E Tests for v1 API Routes", () => {
 
       expect(response.statusCode).toBe(403);
       expect(response.body.error).toBe(
-        "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",
+        "Request failed with status code 403. Error: URL is blocked intentionally. Firecrawl currently does not support scraping this site due to policy restrictions. ",
       );
     });
 
@@ -757,7 +757,7 @@ describe("E2E Tests for v1 API Routes", () => {
 
       expect(response.statusCode).toBe(403);
       expect(response.body.error).toBe(
-        "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",
+        "Request failed with status code 403. Error: URL is blocked intentionally. Firecrawl currently does not support scraping this site due to policy restrictions. ",
       );
     });
 

apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.ts

@@ -0,0 +1,35 @@
+import { jest } from '@jest/globals';
+
+describe('CrawlWatcher', () => {
+  const mockApiUrl = 'https://api.firecrawl.dev';
+  const mockApiKey = 'test-api-key';
+
+  beforeEach(() => {
+    jest.resetModules();
+  });
+
+  test('should create a CrawlWatcher instance successfully when isows is available', async () => {
+    await jest.unstable_mockModule('isows', () => ({
+      WebSocket: jest.fn(),
+    }));
+
+    const { default: FirecrawlApp, CrawlWatcher } = await import('../index');
+    const app = new FirecrawlApp({ apiKey: mockApiKey, apiUrl: mockApiUrl });
+
+    const watcher = new CrawlWatcher('test-id', app);
+    expect(watcher).toBeInstanceOf(CrawlWatcher);
+  });
+
+  test('should throw when WebSocket is not available (isows import fails)', async () => {
+    await jest.unstable_mockModule('isows', () => {
+      throw new Error('Module not found');
+    });
+
+    const { default: FirecrawlApp, CrawlWatcher, FirecrawlError } = await import('../index');
+    const app = new FirecrawlApp({ apiKey: mockApiKey, apiUrl: mockApiUrl });
+
+    expect(() => {
+      new CrawlWatcher('test-id', app);
+    }).toThrow(FirecrawlError);
+  });
+});

apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts

@@ -32,7 +32,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
       });
       await expect(
         invalidApp.scrapeUrl("https://roastmywebsite.ai")
-      ).rejects.toThrow("Request failed with status code 401");
+      ).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
     }
   );
 
@@ -46,7 +46,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
       });
       const blocklistedUrl = "https://facebook.com/fake-test";
       await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow(
-        "Request failed with status code 403"
+        "Unexpected error occurred while trying to scrape URL. Status code: 403"
       );
     }
   );
@@ -169,7 +169,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
       });
       const blocklistedUrl = "https://twitter.com/fake-test";
       await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow(
-        "Request failed with status code 403"
+        "Unexpected error occurred while trying to scrape URL. Status code: 403"
       );
     }
   );
@@ -242,7 +242,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
       const maxChecks = 15;
       let checks = 0;
 
-      while (statusResponse.status === "active" && checks < maxChecks) {
+      while ((statusResponse.status === "active" || statusResponse.status === "scraping" ) && checks < maxChecks) {
         await new Promise((resolve) => setTimeout(resolve, 5000));
         expect(statusResponse.partial_data).not.toBeNull();
         // expect(statusResponse.current).toBeGreaterThanOrEqual(1);

apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -17,13 +17,13 @@ describe('FirecrawlApp E2E Tests', () => {
 
   test.concurrent('should throw error for invalid API key on scrape', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+    await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
   });
 
   test.concurrent('should throw error for blocklisted URL on scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const blocklistedUrl = "https://facebook.com/fake-test";
-    await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
+    await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 403");
   });
 
   test.concurrent('should return successful response with valid preview token', async () => {
@@ -61,7 +61,7 @@ describe('FirecrawlApp E2E Tests', () => {
       'https://roastmywebsite.ai', {
         formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
         headers: { "x-key": "test" },
-        includeTags: ['h1'],
+        // includeTags: ['h1'],
         excludeTags: ['h2'],
         onlyMainContent: true,
         timeout: 30000,
@@ -162,7 +162,7 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should throw error for blocklisted URL on crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const blocklistedUrl = "https://twitter.com/fake-test";
-    await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.");
+    await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403. Error: This website is no longer supported, please reach out to help@firecrawl.com for more info on how to activate it on your account. ");
   });
 
   test.concurrent('should return successful response for crawl and wait for completion', async () => {
@@ -212,7 +212,7 @@ describe('FirecrawlApp E2E Tests', () => {
       scrapeOptions: {
         formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
         headers: { "x-key": "test" },
-        includeTags: ['h1'],
+        // includeTags: ['h1'],
         excludeTags: ['h2'],
         onlyMainContent: true,
         waitFor: 1000
@@ -334,7 +334,7 @@ describe('FirecrawlApp E2E Tests', () => {
         expect(statusResponse.data[0].metadata).not.toHaveProperty("error");
       }
     }
-  }, 60000); // 60 seconds timeout
+  }, 120000); // 120 seconds timeout
 
   test.concurrent('should throw error for invalid API key on map', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });