AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 21:09:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix/p token (#1305)

Browse Source 
* preview-token

* Nick:

---------

Co-authored-by: rafaelmmiller <150964962+rafaelsideguide@users.noreply.github.com>
This commit is contained in:
Nicolas 2025-03-06 19:03:33 -03:00 committed by GitHub
parent 39b1390289
commit 60346ecfa3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 19 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
View File

@ -62,15 +62,6 @@ describe("E2E Tests for API Routes", () => {
expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE); expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
}); });
// tested on rate limit test
// it.concurrent("should return a successful response with a valid preview token", async () => {
// const response = await request(TEST_URL)
// .post("/v0/scrape")
// .set("Authorization", `Bearer this_is_just_a_preview_token`)
// .set("Content-Type", "application/json")
// .send({ url: "https://roastmywebsite.ai" });
// expect(response.statusCode).toBe(200);
// }, 30000); // 30 seconds timeout
it.concurrent( it.concurrent(
"should return a successful response with a valid API key", "should return a successful response with a valid API key",
@ -1087,19 +1078,6 @@ describe("E2E Tests for API Routes", () => {
}, },
3000, 3000,
); );
// it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => {
// const response = await request(TEST_URL)
// .post("/v0/crawlWebsitePreview")
// .set("Authorization", `Bearer this_is_just_a_preview_token`)
// .set("Content-Type", "application/json")
// .send({ url: "https://firecrawl.dev" });
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty("jobId");
// expect(response.body.jobId).toMatch(
// /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
// );
// });
}); });
describe("POST /v0/search", () => { describe("POST /v0/search", () => {
@ -1724,7 +1702,7 @@ describe("E2E Tests for API Routes", () => {
for (let i = 0; i < 5; i++) { for (let i = 0; i < 5; i++) {
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer this_is_just_a_preview_token`) .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com" }); .send({ url: "https://www.scrapethissite.com" });
@ -1732,7 +1710,7 @@ describe("E2E Tests for API Routes", () => {
} }
const response = await request(TEST_URL) const response = await request(TEST_URL)
.post("/v0/scrape") .post("/v0/scrape")
.set("Authorization", `Bearer this_is_just_a_preview_token`) .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)
.set("Content-Type", "application/json") .set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com" }); .send({ url: "https://www.scrapethissite.com" });

14
apps/api/src/controllers/auth.ts
View File

@ -100,7 +100,8 @@ export async function getACUC(
? "auth_credit_usage_chunk_extract" ? "auth_credit_usage_chunk_extract"
: "auth_credit_usage_chunk_test_22_credit_pack_n_extract"; : "auth_credit_usage_chunk_test_22_credit_pack_n_extract";
while (retries < maxRetries) { while (retries < maxRetries) {
const client = Math.random() > 0.5 ? supabase_rr_service : supabase_service; const client =
Math.random() > 0.5 ? supabase_rr_service : supabase_service;
({ data, error } = await client.rpc( ({ data, error } = await client.rpc(
rpcName, rpcName,
{ input_key: api_key }, { input_key: api_key },
@ -201,8 +202,11 @@ export async function supaAuthenticateUser(
let chunk: AuthCreditUsageChunk | null = null; let chunk: AuthCreditUsageChunk | null = null;
let plan: PlanType = "free"; let plan: PlanType = "free";
if (token == "this_is_just_a_preview_token") { if (token == "this_is_just_a_preview_token") {
throw new Error("Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up."); throw new Error(
"Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",
);
}
if (token == process.env.PREVIEW_TOKEN) {
if (mode == RateLimiterMode.CrawlStatus) { if (mode == RateLimiterMode.CrawlStatus) {
rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);
} else if (mode == RateLimiterMode.ExtractStatus) { } else if (mode == RateLimiterMode.ExtractStatus) {
@ -297,7 +301,7 @@ export async function supaAuthenticateUser(
} }
const team_endpoint_token = const team_endpoint_token =
token === "this_is_just_a_preview_token" ? iptoken : teamId; token === process.env.PREVIEW_TOKEN ? iptoken : teamId;
try { try {
await rateLimiter.consume(team_endpoint_token); await rateLimiter.consume(team_endpoint_token);
@ -327,7 +331,7 @@ export async function supaAuthenticateUser(
} }
if ( if (
token === "this_is_just_a_preview_token" && token === process.env.PREVIEW_TOKEN &&
(mode === RateLimiterMode.Scrape || (mode === RateLimiterMode.Scrape ||
mode === RateLimiterMode.Preview || mode === RateLimiterMode.Preview ||
mode === RateLimiterMode.Map || mode === RateLimiterMode.Map ||

2
apps/api/src/controllers/v0/crawlPreview.ts
View File

@ -24,7 +24,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
const incomingIP = (req.headers["x-forwarded-for"] || const incomingIP = (req.headers["x-forwarded-for"] ||
req.socket.remoteAddress) as string; req.socket.remoteAddress) as string;
const iptoken = incomingIP + "this_is_just_a_preview_token"; const iptoken = incomingIP + process.env.PREVIEW_TOKEN;
const team_id = `preview_${iptoken}`; const team_id = `preview_${iptoken}`;
if (!auth.success) { if (!auth.success) {

2
apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -55,7 +55,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
"should return successful response with valid preview token", "should return successful response with valid preview token",
async () => { async () => {
const app = new FirecrawlApp<"v0">({ const app = new FirecrawlApp<"v0">({
apiKey: "this_is_just_a_preview_token", apiKey: process.env.PREVIEW_TOKEN,
apiUrl: API_URL, apiUrl: API_URL,
version: "v0", version: "v0",
}); });

4
apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
View File

@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => {
}); });
test.concurrent('should return successful response with valid preview token', async () => { test.concurrent('should return successful response with valid preview token', async () => {
const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });
const response = await app.scrapeUrl('https://roastmywebsite.ai'); const response = await app.scrapeUrl('https://roastmywebsite.ai');
if (!response.success) { if (!response.success) {
throw new Error(response.error); throw new Error(response.error);
@ -365,7 +365,7 @@ describe('FirecrawlApp E2E Tests', () => {
}); });
test.concurrent('should return successful response with valid preview token', async () => { test.concurrent('should return successful response with valid preview token', async () => {
const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL }); const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });
const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse; const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
expect(response).not.toBeNull(); expect(response).not.toBeNull();
expect(response.links?.length).toBeGreaterThan(0); expect(response.links?.length).toBeGreaterThan(0);

2
apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
View File

@ -37,7 +37,7 @@ def test_scrape_url_invalid_api_key():
# assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value) # assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
def test_successful_response_with_valid_preview_token(): def test_successful_response_with_valid_preview_token():
app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0') app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'), version='v0')
response = app.scrape_url('https://roastmywebsite.ai') response = app.scrape_url('https://roastmywebsite.ai')
assert response is not None assert response is not None
assert 'content' in response assert 'content' in response

4
apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
View File

@ -49,7 +49,7 @@ def test_scrape_url_invalid_api_key():
# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
def test_successful_response_with_valid_preview_token(): def test_successful_response_with_valid_preview_token():
app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token") app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'))
response = app.scrape_url('https://roastmywebsite.ai') response = app.scrape_url('https://roastmywebsite.ai')
assert response is not None assert response is not None
assert "_Roast_" in response['markdown'] assert "_Roast_" in response['markdown']
@ -327,7 +327,7 @@ def test_invalid_api_key_on_map():
# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value) # assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
def test_successful_response_with_valid_preview_token_on_map(): def test_successful_response_with_valid_preview_token_on_map():
app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL) app = FirecrawlApp(api_key=os.getenv('PREVIEW_TOKEN'), api_url=API_URL)
response = app.map_url('https://roastmywebsite.ai') response = app.map_url('https://roastmywebsite.ai')
assert response is not None assert response is not None
assert len(response) > 0 assert len(response) > 0

2
apps/rust-sdk/tests/e2e_with_auth.rs
View File

@ -26,7 +26,7 @@ async fn test_successful_response_with_valid_preview_token() {
let api_url = env::var("API_URL").unwrap(); let api_url = env::var("API_URL").unwrap();
let app = FirecrawlApp::new_selfhosted( let app = FirecrawlApp::new_selfhosted(
api_url, api_url,
Some("this_is_just_a_preview_token"), Some(env::var("PREVIEW_TOKEN").unwrap()),
) )
.unwrap(); .unwrap();
let result = app let result = app