AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 00:58:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: blocklist string

Browse Source
This commit is contained in:
Nicolas 2024-12-20 18:09:49 -03:00
parent 8e947344ad
commit d1f3e26f9e
17 changed files with 105 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
View File

@ -1,6 +1,7 @@
import request from "supertest"; import request from "supertest";
import dotenv from "dotenv"; import dotenv from "dotenv";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
dotenv.config(); dotenv.config();
@ -58,9 +59,7 @@ describe("E2E Tests for API Routes", () => {
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: blocklistedUrl }); .send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toContain( expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
);
}); });
// tested on rate limit test // tested on rate limit test
@ -480,9 +479,7 @@ describe("E2E Tests for API Routes", () => {
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: blocklistedUrl }); .send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toContain( expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
);
}); });
it.concurrent( it.concurrent(

13
apps/api/src/__tests__/e2e_noAuth/index.test.ts
View File

@ -1,5 +1,6 @@
import request from "supertest"; import request from "supertest";
import dotenv from "dotenv"; import dotenv from "dotenv";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
const fs = require("fs"); const fs = require("fs");
const path = require("path"); const path = require("path");
@ -61,9 +62,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: blocklistedUrl }); .send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toContain( expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
);
}); });
it("should return a successful response", async () => { it("should return a successful response", async () => {
@ -88,9 +87,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: blocklistedUrl }); .send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toContain( expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
);
}); });
it("should return a successful response", async () => { it("should return a successful response", async () => {
@ -119,9 +116,7 @@ describe("E2E Tests for API Routes with No Authentication", () => {
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: blocklistedUrl }); .send({ url: blocklistedUrl });
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toContain( expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
);
}); });
it("should return a successful response", async () => { it("should return a successful response", async () => {

9
apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts
View File

@ -4,6 +4,7 @@ import {
ScrapeRequestInput, ScrapeRequestInput,
ScrapeResponseRequestTest, ScrapeResponseRequestTest,
} from "../../controllers/v1/types"; } from "../../controllers/v1/types";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
configDotenv(); configDotenv();
const TEST_URL = "http://127.0.0.1:3002"; const TEST_URL = "http://127.0.0.1:3002";
@ -57,9 +58,7 @@ describe("E2E Tests for v1 API Routes", () => {
.send(scrapeRequest); .send(scrapeRequest);
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toBe( expect(response.body.error).toBe(BLOCKLISTED_URL_MESSAGE);
"URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",
);
}); });
it.concurrent( it.concurrent(
@ -756,9 +755,7 @@ describe("E2E Tests for v1 API Routes", () => {
.send(scrapeRequest); .send(scrapeRequest);
expect(response.statusCode).toBe(403); expect(response.statusCode).toBe(403);
expect(response.body.error).toBe( expect(response.body.error).toBe(BLOCKLISTED_URL_MESSAGE);
"URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",
);
}); });
it.concurrent( it.concurrent(

4
apps/api/src/controllers/v0/crawl.ts
View File

@ -29,6 +29,7 @@ import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority"; import { getJobPriority } from "../../lib/job-priority";
import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types"; import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types";
import { ZodError } from "zod"; import { ZodError } from "zod";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
export async function crawlController(req: Request, res: Response) { export async function crawlController(req: Request, res: Response) {
try { try {
@ -112,8 +113,7 @@ export async function crawlController(req: Request, res: Response) {
if (isUrlBlocked(url)) { if (isUrlBlocked(url)) {
return res.status(403).json({ return res.status(403).json({
error: error: BLOCKLISTED_URL_MESSAGE,
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
}); });
} }

4
apps/api/src/controllers/v0/crawlPreview.ts
View File

@ -15,6 +15,7 @@ import { addScrapeJob } from "../../../src/services/queue-jobs";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl"; import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import * as Sentry from "@sentry/node"; import * as Sentry from "@sentry/node";
import { fromLegacyScrapeOptions } from "../v1/types"; import { fromLegacyScrapeOptions } from "../v1/types";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
export async function crawlPreviewController(req: Request, res: Response) { export async function crawlPreviewController(req: Request, res: Response) {
try { try {
@ -42,8 +43,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
if (isUrlBlocked(url)) { if (isUrlBlocked(url)) {
return res.status(403).json({ return res.status(403).json({
error: error: BLOCKLISTED_URL_MESSAGE,
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
}); });
} }

20
apps/api/src/controllers/v0/scrape.ts
View File

@ -29,6 +29,7 @@ import { getJobPriority } from "../../lib/job-priority";
import { fromLegacyScrapeOptions } from "../v1/types"; import { fromLegacyScrapeOptions } from "../v1/types";
import { ZodError } from "zod"; import { ZodError } from "zod";
import { Document as V0Document } from "./../../lib/entities"; import { Document as V0Document } from "./../../lib/entities";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
export async function scrapeHelper( export async function scrapeHelper(
jobId: string, jobId: string,
@ -53,8 +54,7 @@ export async function scrapeHelper(
if (isUrlBlocked(url)) { if (isUrlBlocked(url)) {
return { return {
success: false, success: false,
error: error: BLOCKLISTED_URL_MESSAGE,
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
returnCode: 403, returnCode: 403,
}; };
} }
@ -265,13 +265,15 @@ export async function scrapeController(req: Request, res: Response) {
} }
if (creditsToBeBilled > 0) { if (creditsToBeBilled > 0) {
// billing for doc done on queue end, bill only for llm extraction // billing for doc done on queue end, bill only for llm extraction
billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch((error) => { billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch(
logger.error( (error) => {
`Failed to bill team ${team_id} for ${creditsToBeBilled} credits`, logger.error(
{ error } `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
); { error },
// Optionally, you could notify an admin or add to a retry queue here );
}); // Optionally, you could notify an admin or add to a retry queue here
},
);
} }
} }

9
apps/api/src/controllers/v1/__tests__/urlValidation.test.ts
View File

@ -1,4 +1,5 @@
import { url } from "../types"; import { url } from "../types";
import { BLOCKLISTED_URL_MESSAGE } from "../../../lib/strings";
describe("URL Schema Validation", () => { describe("URL Schema Validation", () => {
beforeEach(() => { beforeEach(() => {
@ -31,7 +32,7 @@ describe("URL Schema Validation", () => {
it("should reject blocked URLs", () => { it("should reject blocked URLs", () => {
expect(() => url.parse("https://facebook.com")).toThrow( expect(() => url.parse("https://facebook.com")).toThrow(
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.", BLOCKLISTED_URL_MESSAGE,
); );
}); });
@ -47,16 +48,16 @@ describe("URL Schema Validation", () => {
it("should handle URLs with subdomains that are blocked", () => { it("should handle URLs with subdomains that are blocked", () => {
expect(() => url.parse("https://sub.facebook.com")).toThrow( expect(() => url.parse("https://sub.facebook.com")).toThrow(
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.", BLOCKLISTED_URL_MESSAGE,
); );
}); });
it("should handle URLs with paths that are blocked", () => { it("should handle URLs with paths that are blocked", () => {
expect(() => url.parse("http://facebook.com/path")).toThrow( expect(() => url.parse("http://facebook.com/path")).toThrow(
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.", BLOCKLISTED_URL_MESSAGE,
); );
expect(() => url.parse("https://facebook.com/another/path")).toThrow( expect(() => url.parse("https://facebook.com/another/path")).toThrow(
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.", BLOCKLISTED_URL_MESSAGE,
); );
}); });

14
apps/api/src/controllers/v1/extract.ts
View File

@ -92,19 +92,18 @@ export async function extractController(
let mappedLinks = mapResults.mapResults as MapDocument[]; let mappedLinks = mapResults.mapResults as MapDocument[];
// Remove duplicates between mapResults.links and mappedLinks // Remove duplicates between mapResults.links and mappedLinks
const allUrls = [...mappedLinks.map(m => m.url), ...mapResults.links]; const allUrls = [...mappedLinks.map((m) => m.url), ...mapResults.links];
const uniqueUrls = removeDuplicateUrls(allUrls); const uniqueUrls = removeDuplicateUrls(allUrls);
// Only add URLs from mapResults.links that aren't already in mappedLinks // Only add URLs from mapResults.links that aren't already in mappedLinks
const existingUrls = new Set(mappedLinks.map(m => m.url)); const existingUrls = new Set(mappedLinks.map((m) => m.url));
const newUrls = uniqueUrls.filter(url => !existingUrls.has(url)); const newUrls = uniqueUrls.filter((url) => !existingUrls.has(url));
mappedLinks = [ mappedLinks = [
...mappedLinks, ...mappedLinks,
...newUrls.map(url => ({ url, title: "", description: "" })) ...newUrls.map((url) => ({ url, title: "", description: "" })),
]; ];
if (mappedLinks.length === 0) { if (mappedLinks.length === 0) {
mappedLinks = [{ url: baseUrl, title: "", description: "" }]; mappedLinks = [{ url: baseUrl, title: "", description: "" }];
} }
@ -117,7 +116,6 @@ export async function extractController(
`url: ${x.url}, title: ${x.title}, description: ${x.description}`, `url: ${x.url}, title: ${x.title}, description: ${x.description}`,
); );
if (req.body.prompt) { if (req.body.prompt) {
let searchQuery = let searchQuery =
req.body.prompt && allowExternalLinks req.body.prompt && allowExternalLinks

6
apps/api/src/controllers/v1/types.ts
View File

@ -11,6 +11,7 @@ import {
Document as V0Document, Document as V0Document,
} from "../../lib/entities"; } from "../../lib/entities";
import { InternalOptions } from "../../scraper/scrapeURL"; import { InternalOptions } from "../../scraper/scrapeURL";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
export type Format = export type Format =
| "markdown" | "markdown"
@ -44,10 +45,7 @@ export const url = z.preprocess(
return false; return false;
} }
}, "Invalid URL") }, "Invalid URL")
.refine( .refine((x) => !isUrlBlocked(x as string), BLOCKLISTED_URL_MESSAGE),
(x) => !isUrlBlocked(x as string),
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
),
); );
const strictMessage = const strictMessage =

2
apps/api/src/lib/strings.ts Normal file
View File

@ -0,0 +1,2 @@
export const BLOCKLISTED_URL_MESSAGE =
"This website is no longer supported, please reach out to help@firecrawl.com for more info on how to activate it on your account.";

4
apps/api/src/main/runWebScraper.ts
View File

@ -71,7 +71,7 @@ export async function runWebScraper({
module: "runWebscraper", module: "runWebscraper",
scrapeId: bull_job_id, scrapeId: bull_job_id,
jobId: bull_job_id, jobId: bull_job_id,
}) });
const tries = is_crawl ? 3 : 1; const tries = is_crawl ? 3 : 1;
let response: ScrapeUrlResponse | undefined = undefined; let response: ScrapeUrlResponse | undefined = undefined;
@ -176,7 +176,7 @@ export async function runWebScraper({
billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => { billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {
logger.error( logger.error(
`Failed to bill team ${team_id} for ${creditsToBeBilled} credits`, `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
{ error } { error },
); );
// Optionally, you could notify an admin or add to a retry queue here // Optionally, you could notify an admin or add to a retry queue here
}); });

5
apps/api/src/routes/v1.ts
View File

@ -32,6 +32,7 @@ import { extractController } from "../controllers/v1/extract";
// import { livenessController } from "../controllers/v1/liveness"; // import { livenessController } from "../controllers/v1/liveness";
// import { readinessController } from "../controllers/v1/readiness"; // import { readinessController } from "../controllers/v1/readiness";
import { creditUsageController } from "../controllers/v1/credit-usage"; import { creditUsageController } from "../controllers/v1/credit-usage";
import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";
function checkCreditsMiddleware( function checkCreditsMiddleware(
minimum?: number, minimum?: number,
@ -123,8 +124,7 @@ function blocklistMiddleware(req: Request, res: Response, next: NextFunction) {
if (!res.headersSent) { if (!res.headersSent) {
return res.status(403).json({ return res.status(403).json({
success: false, success: false,
error: error: BLOCKLISTED_URL_MESSAGE,
"URL is blocked intentionally. Firecrawl currently does not support scraping this site due to policy restrictions.",
}); });
} }
} }
@ -231,4 +231,3 @@ v1Router.get(
authMiddleware(RateLimiterMode.CrawlStatus), authMiddleware(RateLimiterMode.CrawlStatus),
wrap(creditUsageController), wrap(creditUsageController),
); );

5
apps/api/src/services/billing/credit_billing.ts
View File

@ -44,7 +44,10 @@ export async function supaBillTeam(
if (team_id === "preview") { if (team_id === "preview") {
return { success: true, message: "Preview team, no credits used" }; return { success: true, message: "Preview team, no credits used" };
} }
_logger.info(`Billing team ${team_id} for ${credits} credits`, { team_id, credits }); _logger.info(`Billing team ${team_id} for ${credits} credits`, {
team_id,
credits,
});
const { data, error } = await supabase_service.rpc("bill_team", { const { data, error } = await supabase_service.rpc("bill_team", {
_team_id: team_id, _team_id: team_id,

6
apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
View File

@ -159,12 +159,6 @@ describe('FirecrawlApp E2E Tests', () => {
await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401"); await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
}); });
test.concurrent('should throw error for blocklisted URL on crawl', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const blocklistedUrl = "https://twitter.com/fake-test";
await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.");
});
test.concurrent('should return successful response for crawl and wait for completion', async () => { test.concurrent('should return successful response for crawl and wait for completion', async () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse; const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;

24
apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
View File

@ -29,12 +29,12 @@ def test_scrape_url_invalid_api_key():
invalid_app.scrape_url('https://firecrawl.dev') invalid_app.scrape_url('https://firecrawl.dev')
assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value) assert "Unexpected error during scrape URL: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
def test_blocklisted_url(): # def test_blocklisted_url():
blocklisted_url = "https://facebook.com/fake-test" # blocklisted_url = "https://facebook.com/fake-test"
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0') # app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
with pytest.raises(Exception) as excinfo: # with pytest.raises(Exception) as excinfo:
app.scrape_url(blocklisted_url) # app.scrape_url(blocklisted_url)
assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value) # assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
def test_successful_response_with_valid_preview_token(): def test_successful_response_with_valid_preview_token():
app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0') app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')
@ -90,12 +90,12 @@ def test_crawl_url_invalid_api_key():
invalid_app.crawl_url('https://firecrawl.dev') invalid_app.crawl_url('https://firecrawl.dev')
assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value) assert "Unexpected error during start crawl job: Status code 401. Unauthorized: Invalid token" in str(excinfo.value)
def test_should_return_error_for_blocklisted_url(): # def test_should_return_error_for_blocklisted_url():
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0') # app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')
blocklisted_url = "https://twitter.com/fake-test" # blocklisted_url = "https://twitter.com/fake-test"
with pytest.raises(Exception) as excinfo: # with pytest.raises(Exception) as excinfo:
app.crawl_url(blocklisted_url) # app.crawl_url(blocklisted_url)
assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value) # assert "Unexpected error during start crawl job: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
def test_crawl_url_wait_for_completion_e2e(): def test_crawl_url_wait_for_completion_e2e():
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0') app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY, version='v0')

36
apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
View File

@ -30,12 +30,12 @@ def test_scrape_url_invalid_api_key():
invalid_app.scrape_url('https://firecrawl.dev') invalid_app.scrape_url('https://firecrawl.dev')
assert "Unauthorized: Invalid token" in str(excinfo.value) assert "Unauthorized: Invalid token" in str(excinfo.value)
def test_blocklisted_url(): # def test_blocklisted_url():
blocklisted_url = "https://facebook.com/fake-test" # blocklisted_url = "https://facebook.com/fake-test"
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY) # app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
with pytest.raises(Exception) as excinfo: # with pytest.raises(Exception) as excinfo:
app.scrape_url(blocklisted_url) # app.scrape_url(blocklisted_url)
assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
def test_successful_response_with_valid_preview_token(): def test_successful_response_with_valid_preview_token():
app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token") app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
@ -136,12 +136,12 @@ def test_crawl_url_invalid_api_key():
invalid_app.crawl_url('https://firecrawl.dev') invalid_app.crawl_url('https://firecrawl.dev')
assert "Unauthorized: Invalid token" in str(excinfo.value) assert "Unauthorized: Invalid token" in str(excinfo.value)
def test_should_return_error_for_blocklisted_url(): # def test_should_return_error_for_blocklisted_url():
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY) # app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
blocklisted_url = "https://twitter.com/fake-test" # blocklisted_url = "https://twitter.com/fake-test"
with pytest.raises(Exception) as excinfo: # with pytest.raises(Exception) as excinfo:
app.crawl_url(blocklisted_url) # app.crawl_url(blocklisted_url)
assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
def test_crawl_url_wait_for_completion_e2e(): def test_crawl_url_wait_for_completion_e2e():
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY) app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
@ -296,12 +296,12 @@ def test_invalid_api_key_on_map():
invalid_app.map_url('https://roastmywebsite.ai') invalid_app.map_url('https://roastmywebsite.ai')
assert "Unauthorized: Invalid token" in str(excinfo.value) assert "Unauthorized: Invalid token" in str(excinfo.value)
def test_blocklisted_url_on_map(): # def test_blocklisted_url_on_map():
app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL) # app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)
blocklisted_url = "https://facebook.com/fake-test" # blocklisted_url = "https://facebook.com/fake-test"
with pytest.raises(Exception) as excinfo: # with pytest.raises(Exception) as excinfo:
app.map_url(blocklisted_url) # app.map_url(blocklisted_url)
assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
def test_successful_response_with_valid_preview_token_on_map(): def test_successful_response_with_valid_preview_token_on_map():
app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL) app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)

53
apps/rust-sdk/tests/e2e_with_auth.rs
View File

@ -5,20 +5,20 @@ use firecrawl::FirecrawlApp;
use serde_json::json; use serde_json::json;
use std::env; use std::env;
#[tokio::test] // #[tokio::test]
async fn test_blocklisted_url() { // async fn test_blocklisted_url() {
dotenv().ok(); // dotenv().ok();
let api_url = env::var("API_URL").unwrap(); // let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").ok(); // let api_key = env::var("TEST_API_KEY").ok();
let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap(); // let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
let blocklisted_url = "https://facebook.com/fake-test"; // let blocklisted_url = "https://facebook.com/fake-test";
let result = app.scrape_url(blocklisted_url, None).await; // let result = app.scrape_url(blocklisted_url, None).await;
assert_matches!( // assert_matches!(
result, // result,
Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions") // Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions")
); // );
} // }
#[tokio::test] #[tokio::test]
async fn test_successful_response_with_valid_preview_token() { async fn test_successful_response_with_valid_preview_token() {
@ -103,20 +103,21 @@ async fn test_successful_response_for_valid_scrape_with_pdf_file_without_explici
.contains("We present spectrophotometric observations of the Broad Line Radio Galaxy")); .contains("We present spectrophotometric observations of the Broad Line Radio Galaxy"));
} }
#[tokio::test]
async fn test_should_return_error_for_blocklisted_url() {
dotenv().ok();
let api_url = env::var("API_URL").unwrap();
let api_key = env::var("TEST_API_KEY").ok();
let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
let blocklisted_url = "https://twitter.com/fake-test";
let result = app.crawl_url(blocklisted_url, None).await;
assert_matches!( // #[tokio::test]
result, // async fn test_should_return_error_for_blocklisted_url() {
Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions.") // dotenv().ok();
); // let api_url = env::var("API_URL").unwrap();
} // let api_key = env::var("TEST_API_KEY").ok();
// let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
// let blocklisted_url = "https://twitter.com/fake-test";
// let result = app.crawl_url(blocklisted_url, None).await;
// assert_matches!(
// result,
// Err(e) if e.to_string().contains("Firecrawl currently does not support social media scraping due to policy restrictions.")
// );
// }
#[tokio::test] #[tokio::test]
async fn test_llm_extraction() { async fn test_llm_extraction() {