From 503e83e83edcf578ab7acec11275ad5695bc3c41 Mon Sep 17 00:00:00 2001 From: Sebastjan Prachovskij Date: Tue, 3 Sep 2024 18:26:11 +0300 Subject: [PATCH 01/27] Add SearchApi to search Add support for engines, improve status code error Remove changes in package, add engine to env params Improve description in env example Remove unnecessary empty line Improve text --- apps/api/.env.example | 11 ++++-- apps/api/.env.local | 2 +- apps/api/src/search/index.ts | 12 ++++++- apps/api/src/search/searchapi.ts | 60 ++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 apps/api/src/search/searchapi.ts diff --git a/apps/api/.env.example b/apps/api/.env.example index f3c1dc1b..6ba49daa 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -1,5 +1,5 @@ # ===== Required ENVS ====== -NUM_WORKERS_PER_QUEUE=8 +NUM_WORKERS_PER_QUEUE=8 PORT=3002 HOST=0.0.0.0 REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379 @@ -11,9 +11,14 @@ USE_DB_AUTHENTICATION=true # ===== Optional ENVS ====== +# SearchApi key. Head to https://searchapi.com/ to get your API key +SEARCHAPI_API_KEY= +# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more engines +SEARCHAPI_ENGINE= + # Supabase Setup (used to support DB authentication, advanced logging, etc.) -SUPABASE_ANON_TOKEN= -SUPABASE_URL= +SUPABASE_ANON_TOKEN= +SUPABASE_URL= SUPABASE_SERVICE_TOKEN= # Other Optionals diff --git a/apps/api/.env.local b/apps/api/.env.local index 17f85935..9fa41498 100644 --- a/apps/api/.env.local +++ b/apps/api/.env.local @@ -12,4 +12,4 @@ ANTHROPIC_API_KEY= BULL_AUTH_KEY= LOGTAIL_KEY= PLAYWRIGHT_MICROSERVICE_URL= - +SEARCHAPI_API_KEY= diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts index f4c5b6d0..3bcb85d2 100644 --- a/apps/api/src/search/index.ts +++ b/apps/api/src/search/index.ts @@ -2,6 +2,7 @@ import { Logger } from "../../src/lib/logger"; import { SearchResult } from "../../src/lib/entities"; import { googleSearch } from "./googlesearch"; import { fireEngineMap } from "./fireEngine"; +import { searchapi_search } from "./searchapi"; import { serper_search } from "./serper"; export async function search({ @@ -30,7 +31,16 @@ export async function search({ timeout?: number; }): Promise { try { - + if (process.env.SEARCHAPI_API_KEY) { + return await searchapi_search(query, { + num_results, + tbs, + filter, + lang, + country, + location + }); + } if (process.env.SERPER_API_KEY) { return await serper_search(query, { num_results, diff --git a/apps/api/src/search/searchapi.ts b/apps/api/src/search/searchapi.ts new file mode 100644 index 00000000..24778a77 --- /dev/null +++ b/apps/api/src/search/searchapi.ts @@ -0,0 +1,60 @@ +import axios from "axios"; +import dotenv from "dotenv"; +import { SearchResult } from "../../src/lib/entities"; + +dotenv.config(); + +interface SearchOptions { + tbs?: string; + filter?: string; + lang?: string; + country?: string; + location?: string; + num_results: number; + page?: number; +} + +export async function searchapi_search(q: string, options: SearchOptions): Promise { + const params = { + q: q, + hl: options.lang, + gl: options.country, + location: options.location, + num: options.num_results, + page: options.page ?? 1, + engine: process.env.SEARCHAPI_ENGINE || "google", + }; + + const url = `https://www.searchapi.io/api/v1/search`; + + try { + const response = await axios.get(url, { + headers: { + "Authorization": `Bearer ${process.env.SEARCHAPI_API_KEY}`, + "Content-Type": "application/json", + "X-SearchApi-Source": "Firecrawl", + }, + params: params, + }); + + + if (response.status === 401) { + throw new Error("Unauthorized. Please check your API key."); + } + + const data = response.data; + + if (data && Array.isArray(data.organic_results)) { + return data.organic_results.map((a: any) => ({ + url: a.link, + title: a.title, + description: a.snippet, + })); + } else { + return []; + } + } catch (error) { + console.error(`There was an error searching for content: ${error.message}`); + return []; + } +} From 1c02187054ad1847dcee77728255018ec743f6d5 Mon Sep 17 00:00:00 2001 From: Stijn Smits <167638923+s-smits@users.noreply.github.com> Date: Sun, 6 Oct 2024 13:25:23 +0200 Subject: [PATCH 02/27] Update website_qa_with_gemini_caching.ipynb --- .../website_qa_with_gemini_caching.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/website_qa_with_gemini_caching/website_qa_with_gemini_caching.ipynb b/examples/website_qa_with_gemini_caching/website_qa_with_gemini_caching.ipynb index 0876affa..9a2244a1 100644 --- a/examples/website_qa_with_gemini_caching/website_qa_with_gemini_caching.ipynb +++ b/examples/website_qa_with_gemini_caching/website_qa_with_gemini_caching.ipynb @@ -98,7 +98,7 @@ "source": [ "# Create a cache with a 5 minute TTL\n", "cache = caching.CachedContent.create(\n", - " model=\"models/gemini-1.5-pro-001\",\n", + " model=\"models/gemini-1.5-pro-002\",\n", " display_name=\"website crawl testing again\", # used to identify the cache\n", " system_instruction=\"You are an expert at this website, and your job is to answer user's query based on the website you have access to.\",\n", " contents=[text_file],\n", From 460f5581fef1d800e2a96fbf10f30b242921ac60 Mon Sep 17 00:00:00 2001 From: Stijn Smits <167638923+s-smits@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:17:47 +0200 Subject: [PATCH 03/27] Add files via upload --- ...website_qa_with_gemini_flash_caching.ipynb | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 examples/website_qa_with_gemini_caching/website_qa_with_gemini_flash_caching.ipynb diff --git a/examples/website_qa_with_gemini_caching/website_qa_with_gemini_flash_caching.ipynb b/examples/website_qa_with_gemini_caching/website_qa_with_gemini_flash_caching.ipynb new file mode 100644 index 00000000..19d72c9d --- /dev/null +++ b/examples/website_qa_with_gemini_caching/website_qa_with_gemini_flash_caching.ipynb @@ -0,0 +1,166 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ericciarla/projects/python_projects/agents_testing/.conda/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import os\n", + "import datetime\n", + "import time\n", + "import google.generativeai as genai\n", + "from google.generativeai import caching\n", + "from dotenv import load_dotenv\n", + "from firecrawl import FirecrawlApp\n", + "import json\n", + "\n", + "# Load environment variables\n", + "load_dotenv()\n", + "\n", + "# Retrieve API keys from environment variables\n", + "google_api_key = os.getenv(\"GOOGLE_API_KEY\")\n", + "firecrawl_api_key = os.getenv(\"FIRECRAWL_API_KEY\")\n", + "\n", + "# Configure the Google Generative AI module with the API key\n", + "genai.configure(api_key=google_api_key)\n", + "\n", + "# Initialize the FirecrawlApp with your API key\n", + "app = FirecrawlApp(api_key=firecrawl_api_key)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No data returned from crawl.\n" + ] + } + ], + "source": [ + "# Crawl a website\n", + "crawl_url = 'https://dify.ai/'\n", + "params = {\n", + " \n", + " 'crawlOptions': {\n", + " 'limit': 100\n", + " }\n", + "}\n", + "crawl_result = app.crawl_url(crawl_url, params=params)\n", + "\n", + "if crawl_result is not None:\n", + " # Convert crawl results to JSON format, excluding 'content' field from each entry\n", + " cleaned_crawl_result = [{k: v for k, v in entry.items() if k != 'content'} for entry in crawl_result]\n", + "\n", + " # Save the modified results as a text file containing JSON data\n", + " with open('crawl_result.txt', 'w') as file:\n", + " file.write(json.dumps(cleaned_crawl_result, indent=4))\n", + "else:\n", + " print(\"No data returned from crawl.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# Upload the video using the Files API\n", + "text_file = genai.upload_file(path=\"crawl_result.txt\")\n", + "\n", + "# Wait for the file to finish processing\n", + "while text_file.state.name == \"PROCESSING\":\n", + " print('Waiting for file to be processed.')\n", + " time.sleep(2)\n", + " text_file = genai.get_file(text_file.name)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a cache with a 5 minute TTL\n", + "cache = caching.CachedContent.create(\n", + " model=\"models/gemini-1.5-flash-002\",\n", + " display_name=\"website crawl testing again\", # used to identify the cache\n", + " system_instruction=\"You are an expert at this website, and your job is to answer user's query based on the website you have access to.\",\n", + " contents=[text_file],\n", + " ttl=datetime.timedelta(minutes=15),\n", + ")\n", + "# Construct a GenerativeModel which uses the created cache.\n", + "model = genai.GenerativeModel.from_cached_content(cached_content=cache)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dify.AI utilizes the **Firecrawl** service for website scraping. This service can crawl and convert any website into clean markdown or structured data that's ready for use in building RAG applications. \n", + "\n", + "Here's how Firecrawl helps:\n", + "\n", + "* **Crawling and Conversion:** Firecrawl crawls the website and converts the content into a format that is easily understood by LLMs, such as markdown or structured data.\n", + "* **Clean Output:** Firecrawl ensures the data is clean and free of errors, making it easier to use in Dify's RAG engine.\n", + "* **Parallel Crawling:** Firecrawl efficiently crawls web pages in parallel, delivering results quickly.\n", + "\n", + "You can find Firecrawl on their website: [https://www.firecrawl.dev/](https://www.firecrawl.dev/)\n", + "\n", + "Firecrawl offers both a cloud service and an open-source software (OSS) edition. \n", + "\n" + ] + } + ], + "source": [ + "# Query the model\n", + "response = model.generate_content([\"What powers website scraping with Dify?\"])\n", + "response_dict = response.to_dict()\n", + "response_text = response_dict['candidates'][0]['content']['parts'][0]['text']\n", + "print(response_text)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From bbfdda8867614812b3dea399647117317357b783 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 22 Oct 2024 19:47:23 -0300 Subject: [PATCH 04/27] Nick: init --- apps/api/src/controllers/auth.ts | 20 ++- apps/api/src/services/billing/auto_charge.ts | 148 ++++++++++++++++ .../src/services/billing/credit_billing.ts | 134 +++++++++++--- .../api/src/services/billing/issue_credits.ts | 20 +++ apps/api/src/services/billing/stripe.ts | 51 ++++++ .../notification/email_notification.ts | 166 ++++++++++-------- .../notification/notification_string.ts | 4 + apps/api/src/types.ts | 2 + 8 files changed, 435 insertions(+), 110 deletions(-) create mode 100644 apps/api/src/services/billing/auto_charge.ts create mode 100644 apps/api/src/services/billing/issue_credits.ts create mode 100644 apps/api/src/services/billing/stripe.ts diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 5546bc17..bf41b96a 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -75,15 +75,19 @@ export async function setCachedACUC( export async function getACUC( api_key: string, - cacheOnly = false + cacheOnly = false, + useCache = true ): Promise { const cacheKeyACUC = `acuc_${api_key}`; - const cachedACUC = await getValue(cacheKeyACUC); + if (useCache) { + const cachedACUC = await getValue(cacheKeyACUC); + if (cachedACUC !== null) { + return JSON.parse(cachedACUC); + } + } - if (cachedACUC !== null) { - return JSON.parse(cachedACUC); - } else if (!cacheOnly) { + if (!cacheOnly) { let data; let error; let retries = 0; @@ -91,7 +95,7 @@ export async function getACUC( while (retries < maxRetries) { ({ data, error } = await supabase_service.rpc( - "auth_credit_usage_chunk_test_3", + "auth_credit_usage_chunk_test_17_credit_pack", { input_key: api_key } )); @@ -118,9 +122,11 @@ export async function getACUC( data.length === 0 ? null : data[0].team_id === null ? null : data[0]; // NOTE: Should we cache null chunks? - mogery - if (chunk !== null) { + if (chunk !== null && useCache) { setCachedACUC(api_key, chunk); } + // Log the chunk for now + console.log(chunk); return chunk; } else { diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts new file mode 100644 index 00000000..6ab6914c --- /dev/null +++ b/apps/api/src/services/billing/auto_charge.ts @@ -0,0 +1,148 @@ +// Import necessary dependencies and types +import { AuthCreditUsageChunk } from "../../controllers/v1/types"; +import { getACUC, setCachedACUC } from "../../controllers/auth"; +import { redlock } from "../redlock"; +import { supabase_service } from "../supabase"; +import { createPaymentIntent } from "./stripe"; +import { issueCredits } from "./issue_credits"; +import { sendNotification } from "../notification/email_notification"; +import { NotificationType } from "../../types"; +import { deleteKey } from "../redis"; +import { sendSlackWebhook } from "../alerts/slack"; +import { Logger } from "../../lib/logger"; + +// Define the number of credits to be added during auto-recharge +const AUTO_RECHARGE_CREDITS = 1000; + +/** + * Attempt to automatically charge a user's account when their credit balance falls below a threshold + * @param chunk The user's current usage data + * @param autoRechargeThreshold The credit threshold that triggers auto-recharge + */ +export async function autoCharge( + chunk: AuthCreditUsageChunk, + autoRechargeThreshold: number +): Promise<{ success: boolean; message: string; remainingCredits: number; chunk: AuthCreditUsageChunk }> { + const resource = `auto-recharge:${chunk.team_id}`; + try { + // Use a distributed lock to prevent concurrent auto-charge attempts + return await redlock.using([resource], 5000, async (signal) : Promise<{ success: boolean; message: string; remainingCredits: number; chunk: AuthCreditUsageChunk }> => { + // Recheck the condition inside the lock to prevent race conditions + const updatedChunk = await getACUC(chunk.api_key, false, false); + if ( + updatedChunk && + updatedChunk.remaining_credits < autoRechargeThreshold + ) { + if (chunk.sub_user_id) { + // Fetch the customer's Stripe information + const { data: customer, error: customersError } = + await supabase_service + .from("customers") + .select("id, stripe_customer_id") + .eq("id", chunk.sub_user_id) + .single(); + + if (customersError) { + Logger.error(`Error fetching customer data: ${customersError}`); + return { + success: false, + message: "Error fetching customer data", + remainingCredits: chunk.remaining_credits, + chunk, + }; + } + + if (customer && customer.stripe_customer_id) { + let issueCreditsSuccess = false; + // Attempt to create a payment intent + const paymentStatus = await createPaymentIntent( + chunk.team_id, + customer.stripe_customer_id + ); + + // If payment is successful or requires further action, issue credits + if ( + paymentStatus.return_status === "succeeded" || + paymentStatus.return_status === "requires_action" + ) { + issueCreditsSuccess = await issueCredits( + chunk.team_id, + AUTO_RECHARGE_CREDITS + ); + } + + // Record the auto-recharge transaction + await supabase_service.from("auto_recharge_transactions").insert({ + team_id: chunk.team_id, + initial_payment_status: paymentStatus.return_status, + credits_issued: issueCreditsSuccess ? AUTO_RECHARGE_CREDITS : 0, + stripe_charge_id: paymentStatus.charge_id, + }); + + // Send a notification if credits were successfully issued + if (issueCreditsSuccess) { + await sendNotification( + chunk.team_id, + NotificationType.AUTO_RECHARGE_SUCCESS, + chunk.sub_current_period_start, + chunk.sub_current_period_end, + chunk, + true + ); + } + + // Reset ACUC cache to reflect the new credit balance + const cacheKeyACUC = `acuc_${chunk.api_key}`; + await deleteKey(cacheKeyACUC); + if (process.env.SLACK_ADMIN_WEBHOOK_URL ) { + sendSlackWebhook( + `Auto-recharge successful: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}. User was notified via email.`, + false, + process.env.SLACK_ADMIN_WEBHOOK_URL + ).catch((error) => { + Logger.debug(`Error sending slack notification: ${error}`); + }); + } + return { + success: true, + message: "Auto-recharge successful", + remainingCredits: chunk.remaining_credits + AUTO_RECHARGE_CREDITS, + chunk: {...chunk, remaining_credits: chunk.remaining_credits + AUTO_RECHARGE_CREDITS}, + }; + } else { + Logger.error("No Stripe customer ID found for user"); + return { + success: false, + message: "No Stripe customer ID found for user", + remainingCredits: chunk.remaining_credits, + chunk, + }; + } + } else { + Logger.error("No sub_user_id found in chunk"); + return { + success: false, + message: "No sub_user_id found in chunk", + remainingCredits: chunk.remaining_credits, + chunk, + }; + } + } + return { + success: false, + message: "No need to auto-recharge", + remainingCredits: chunk.remaining_credits, + chunk, + }; + + }); + } catch (error) { + Logger.error(`Failed to acquire lock for auto-recharge: ${error}`); + return { + success: false, + message: "Failed to acquire lock for auto-recharge", + remainingCredits: chunk.remaining_credits, + chunk, + }; + } +} diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index fc73ca7c..3c43f5a0 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -6,24 +6,40 @@ import { Logger } from "../../lib/logger"; import * as Sentry from "@sentry/node"; import { AuthCreditUsageChunk } from "../../controllers/v1/types"; import { getACUC, setCachedACUC } from "../../controllers/auth"; +import { issueCredits } from "./issue_credits"; +import { redlock } from "../redlock"; +import { autoCharge } from "./auto_charge"; +import { getValue, setValue } from "../redis"; const FREE_CREDITS = 500; /** * If you do not know the subscription_id in the current context, pass subscription_id as undefined. */ -export async function billTeam(team_id: string, subscription_id: string | null | undefined, credits: number) { +export async function billTeam( + team_id: string, + subscription_id: string | null | undefined, + credits: number +) { return withAuth(supaBillTeam)(team_id, subscription_id, credits); } -export async function supaBillTeam(team_id: string, subscription_id: string, credits: number) { +export async function supaBillTeam( + team_id: string, + subscription_id: string, + credits: number +) { if (team_id === "preview") { return { success: true, message: "Preview team, no credits used" }; } Logger.info(`Billing team ${team_id} for ${credits} credits`); - const { data, error } = - await supabase_service.rpc("bill_team", { _team_id: team_id, sub_id: subscription_id ?? null, fetch_subscription: subscription_id === undefined, credits }); - + const { data, error } = await supabase_service.rpc("bill_team", { + _team_id: team_id, + sub_id: subscription_id ?? null, + fetch_subscription: subscription_id === undefined, + credits, + }); + if (error) { Sentry.captureException(error); Logger.error("Failed to bill team: " + JSON.stringify(error)); @@ -31,48 +47,109 @@ export async function supaBillTeam(team_id: string, subscription_id: string, cre } (async () => { - for (const apiKey of (data ?? []).map(x => x.api_key)) { - await setCachedACUC(apiKey, acuc => (acuc ? { - ...acuc, - credits_used: acuc.credits_used + credits, - adjusted_credits_used: acuc.adjusted_credits_used + credits, - remaining_credits: acuc.remaining_credits - credits, - } : null)); + for (const apiKey of (data ?? []).map((x) => x.api_key)) { + await setCachedACUC(apiKey, (acuc) => + acuc + ? { + ...acuc, + credits_used: acuc.credits_used + credits, + adjusted_credits_used: acuc.adjusted_credits_used + credits, + remaining_credits: acuc.remaining_credits - credits, + } + : null + ); } })(); } -export async function checkTeamCredits(chunk: AuthCreditUsageChunk, team_id: string, credits: number) { - return withAuth(supaCheckTeamCredits)(chunk, team_id, credits); +export async function checkTeamCredits( + chunk: AuthCreditUsageChunk, + team_id: string, + credits: number +): Promise<{ success: boolean; message: string; remainingCredits: number; chunk: AuthCreditUsageChunk }> { + const result = await withAuth(supaCheckTeamCredits)(chunk, team_id, credits); + return { + success: result.success, + message: result.message, + remainingCredits: result.remainingCredits, + chunk: chunk // Ensure chunk is always returned + }; } // if team has enough credits for the operation, return true, else return false -export async function supaCheckTeamCredits(chunk: AuthCreditUsageChunk, team_id: string, credits: number) { +export async function supaCheckTeamCredits( + chunk: AuthCreditUsageChunk, + team_id: string, + credits: number +) { // WARNING: chunk will be null if team_id is preview -- do not perform operations on it under ANY circumstances - mogery if (team_id === "preview") { - return { success: true, message: "Preview team, no credits used", remainingCredits: Infinity }; + return { + success: true, + message: "Preview team, no credits used", + remainingCredits: Infinity, + }; } const creditsWillBeUsed = chunk.adjusted_credits_used + credits; // In case chunk.price_credits is undefined, set it to a large number to avoid mistakes - const totalPriceCredits = chunk.price_credits ?? 100000000; + const totalPriceCredits = chunk.total_credits_sum ?? 100000000; // Removal of + credits const creditUsagePercentage = chunk.adjusted_credits_used / totalPriceCredits; + let isAutoRechargeEnabled = false, autoRechargeThreshold = 1000; + const cacheKey = `team_auto_recharge_${team_id}`; + let cachedData = await getValue(cacheKey); + if (cachedData) { + const parsedData = JSON.parse(cachedData); + isAutoRechargeEnabled = parsedData.auto_recharge; + autoRechargeThreshold = parsedData.auto_recharge_threshold; + } else { + const { data, error } = await supabase_service + .from("teams") + .select("auto_recharge, auto_recharge_threshold") + .eq("id", team_id) + .single(); + + if (data) { + isAutoRechargeEnabled = data.auto_recharge; + autoRechargeThreshold = data.auto_recharge_threshold; + await setValue(cacheKey, JSON.stringify(data), 300); // Cache for 5 minutes (300 seconds) + } + } + + if (isAutoRechargeEnabled && chunk.remaining_credits < autoRechargeThreshold) { + const autoChargeResult = await autoCharge(chunk, autoRechargeThreshold); + if (autoChargeResult.success) { + return { + success: true, + message: autoChargeResult.message, + remainingCredits: autoChargeResult.remainingCredits, + chunk: autoChargeResult.chunk, + }; + } + } + // Compare the adjusted total credits used with the credits allowed by the plan if (creditsWillBeUsed > totalPriceCredits) { // Only notify if their actual credits (not what they will use) used is greater than the total price credits - if(chunk.adjusted_credits_used > totalPriceCredits) { + if (chunk.adjusted_credits_used > totalPriceCredits) { sendNotification( team_id, - NotificationType.LIMIT_REACHED, - chunk.sub_current_period_start, - chunk.sub_current_period_end, - chunk - ); - } - return { success: false, message: "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing.", remainingCredits: chunk.remaining_credits, chunk }; + NotificationType.LIMIT_REACHED, + chunk.sub_current_period_start, + chunk.sub_current_period_end, + chunk + ); + } + return { + success: false, + message: + "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing.", + remainingCredits: chunk.remaining_credits, + chunk, + }; } else if (creditUsagePercentage >= 0.8 && creditUsagePercentage < 1) { // Send email notification for approaching credit limit sendNotification( @@ -84,7 +161,12 @@ export async function supaCheckTeamCredits(chunk: AuthCreditUsageChunk, team_id: ); } - return { success: true, message: "Sufficient credits available", remainingCredits: chunk.remaining_credits, chunk }; + return { + success: true, + message: "Sufficient credits available", + remainingCredits: chunk.remaining_credits, + chunk, + }; } // Count the total credits used by a team within the current billing period and return the remaining credits. diff --git a/apps/api/src/services/billing/issue_credits.ts b/apps/api/src/services/billing/issue_credits.ts new file mode 100644 index 00000000..6b34b2ed --- /dev/null +++ b/apps/api/src/services/billing/issue_credits.ts @@ -0,0 +1,20 @@ +import { Logger } from "../../lib/logger"; +import { supabase_service } from "../supabase"; + +export async function issueCredits(team_id: string, credits: number) { + // Add an entry to supabase coupons + const { data, error } = await supabase_service.from("coupons").insert({ + team_id: team_id, + credits: credits, + status: "active", + // indicates that this coupon was issued from auto recharge + from_auto_recharge: true, + }); + + if (error) { + Logger.error(`Error adding coupon: ${error}`); + return false; + } + + return true; +} diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.ts new file mode 100644 index 00000000..f1400804 --- /dev/null +++ b/apps/api/src/services/billing/stripe.ts @@ -0,0 +1,51 @@ +import { Logger } from "../../lib/logger"; +import Stripe from "stripe"; + +const stripe = new Stripe(process.env.STRIPE_SECRET_KEY ?? ""); + +async function getCustomerDefaultPaymentMethod(customerId: string) { + const paymentMethods = await stripe.customers.listPaymentMethods(customerId, { + limit: 3, + }); + return paymentMethods.data[0]?.id; +} + +type ReturnStatus = "succeeded" | "requires_action" | "failed"; +export async function createPaymentIntent( + team_id: string, + customer_id: string +): Promise<{ return_status: ReturnStatus; charge_id: string }> { + try { + const paymentIntent = await stripe.paymentIntents.create({ + amount: 1100, + currency: "usd", + customer: customer_id, + description: "Firecrawl: Auto re-charge of 1000 credits", + payment_method_types: ["card"], + payment_method: await getCustomerDefaultPaymentMethod(customer_id), + off_session: true, + confirm: true, + }); + + if (paymentIntent.status === "succeeded") { + Logger.info(`Payment succeeded for team: ${team_id}`); + return { return_status: "succeeded", charge_id: paymentIntent.id }; + } else if ( + paymentIntent.status === "requires_action" || + paymentIntent.status === "processing" || + paymentIntent.status === "requires_capture" + ) { + Logger.warn(`Payment requires further action for team: ${team_id}`); + return { return_status: "requires_action", charge_id: paymentIntent.id }; + } else { + Logger.error(`Payment failed for team: ${team_id}`); + return { return_status: "failed", charge_id: paymentIntent.id }; + } + } catch (error) { + Logger.error( + `Failed to create or confirm PaymentIntent for team: ${team_id}` + ); + console.error(error); + return { return_status: "failed", charge_id: "" }; + } +} diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts index cf02892e..1eceb06b 100644 --- a/apps/api/src/services/notification/email_notification.ts +++ b/apps/api/src/services/notification/email_notification.ts @@ -24,6 +24,14 @@ const emailTemplates: Record< subject: "Rate Limit Reached - Firecrawl", html: "Hey there,

You've hit one of the Firecrawl endpoint's rate limit! Take a breather and try again in a few moments. If you need higher rate limits, consider upgrading your plan. Check out our pricing page for more info.

If you have any questions, feel free to reach out to us at hello@firecrawl.com


Thanks,
Firecrawl Team

Ps. this email is only sent once every 7 days if you reach a rate limit.", }, + [NotificationType.AUTO_RECHARGE_SUCCESS]: { + subject: "Auto recharge successful - Firecrawl", + html: "Hey there,

Your account was successfully recharged with 1000 credits because your remaining credits were below the threshold.


Thanks,
Firecrawl Team
", + }, + [NotificationType.AUTO_RECHARGE_FAILED]: { + subject: "Auto recharge failed - Firecrawl", + html: "Hey there,

Your auto recharge failed. Please try again manually. If the issue persists, please reach out to us at hello@firecrawl.com


Thanks,
Firecrawl Team
", + }, }; export async function sendNotification( @@ -31,18 +39,20 @@ export async function sendNotification( notificationType: NotificationType, startDateString: string, endDateString: string, - chunk: AuthCreditUsageChunk + chunk: AuthCreditUsageChunk, + bypassRecentChecks: boolean = false ) { return withAuth(sendNotificationInternal)( team_id, notificationType, startDateString, endDateString, - chunk + chunk, + bypassRecentChecks ); } -async function sendEmailNotification( +export async function sendEmailNotification( email: string, notificationType: NotificationType, ) { @@ -72,90 +82,92 @@ export async function sendNotificationInternal( notificationType: NotificationType, startDateString: string, endDateString: string, - chunk: AuthCreditUsageChunk + chunk: AuthCreditUsageChunk, + bypassRecentChecks: boolean = false ): Promise<{ success: boolean }> { if (team_id === "preview") { return { success: true }; } - const fifteenDaysAgo = new Date(); - fifteenDaysAgo.setDate(fifteenDaysAgo.getDate() - 15); + if (!bypassRecentChecks) { + const fifteenDaysAgo = new Date(); + fifteenDaysAgo.setDate(fifteenDaysAgo.getDate() - 15); - const { data, error } = await supabase_service - .from("user_notifications") - .select("*") - .eq("team_id", team_id) - .eq("notification_type", notificationType) - .gte("sent_date", fifteenDaysAgo.toISOString()); - - if (error) { - Logger.debug(`Error fetching notifications: ${error}`); - return { success: false }; - } - - if (data.length !== 0) { - // Logger.debug(`Notification already sent for team_id: ${team_id} and notificationType: ${notificationType} in the last 15 days`); - return { success: false }; - } - - const { data: recentData, error: recentError } = await supabase_service - .from("user_notifications") - .select("*") - .eq("team_id", team_id) - .eq("notification_type", notificationType) - .gte("sent_date", startDateString) - .lte("sent_date", endDateString); - - if (recentError) { - Logger.debug(`Error fetching recent notifications: ${recentError}`); - return { success: false }; - } - - if (recentData.length !== 0) { - // Logger.debug(`Notification already sent for team_id: ${team_id} and notificationType: ${notificationType} within the specified date range`); - return { success: false }; - } else { - console.log(`Sending notification for team_id: ${team_id} and notificationType: ${notificationType}`); - // get the emails from the user with the team_id - const { data: emails, error: emailsError } = await supabase_service - .from("users") - .select("email") - .eq("team_id", team_id); - - if (emailsError) { - Logger.debug(`Error fetching emails: ${emailsError}`); - return { success: false }; - } - - for (const email of emails) { - await sendEmailNotification(email.email, notificationType); - } - - const { error: insertError } = await supabase_service + const { data, error } = await supabase_service .from("user_notifications") - .insert([ - { - team_id: team_id, - notification_type: notificationType, - sent_date: new Date().toISOString(), - }, - ]); + .select("*") + .eq("team_id", team_id) + .eq("notification_type", notificationType) + .gte("sent_date", fifteenDaysAgo.toISOString()); - if (process.env.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) { - sendSlackWebhook( - `${getNotificationString(notificationType)}: Team ${team_id}, with email ${emails[0].email}. Number of credits used: ${chunk.adjusted_credits_used} | Number of credits in the plan: ${chunk.price_credits}`, - false, - process.env.SLACK_ADMIN_WEBHOOK_URL - ).catch((error) => { - Logger.debug(`Error sending slack notification: ${error}`); - }); - } - - if (insertError) { - Logger.debug(`Error inserting notification record: ${insertError}`); + if (error) { + Logger.debug(`Error fetching notifications: ${error}`); return { success: false }; } - return { success: true }; + if (data.length !== 0) { + return { success: false }; + } + + const { data: recentData, error: recentError } = await supabase_service + .from("user_notifications") + .select("*") + .eq("team_id", team_id) + .eq("notification_type", notificationType) + .gte("sent_date", startDateString) + .lte("sent_date", endDateString); + + if (recentError) { + Logger.debug(`Error fetching recent notifications: ${recentError}`); + return { success: false }; + } + + if (recentData.length !== 0) { + return { success: false }; + } + } + + console.log(`Sending notification for team_id: ${team_id} and notificationType: ${notificationType}`); + // get the emails from the user with the team_id + const { data: emails, error: emailsError } = await supabase_service + .from("users") + .select("email") + .eq("team_id", team_id); + + if (emailsError) { + Logger.debug(`Error fetching emails: ${emailsError}`); + return { success: false }; + } + + for (const email of emails) { + await sendEmailNotification(email.email, notificationType); + } + + const { error: insertError } = await supabase_service + .from("user_notifications") + .insert([ + { + team_id: team_id, + notification_type: notificationType, + sent_date: new Date().toISOString(), + }, + ]); + + if (process.env.SLACK_ADMIN_WEBHOOK_URL && emails.length > 0) { + sendSlackWebhook( + `${getNotificationString(notificationType)}: Team ${team_id}, with email ${emails[0].email}. Number of credits used: ${chunk.adjusted_credits_used} | Number of credits in the plan: ${chunk.price_credits}`, + false, + process.env.SLACK_ADMIN_WEBHOOK_URL + ).catch((error) => { + Logger.debug(`Error sending slack notification: ${error}`); + }); + } + + if (insertError) { + Logger.debug(`Error inserting notification record: ${insertError}`); + return { success: false }; + } + + return { success: true }; } diff --git a/apps/api/src/services/notification/notification_string.ts b/apps/api/src/services/notification/notification_string.ts index 8369a0ca..72bc60c4 100644 --- a/apps/api/src/services/notification/notification_string.ts +++ b/apps/api/src/services/notification/notification_string.ts @@ -11,6 +11,10 @@ export function getNotificationString( return "Limit reached (100%)"; case NotificationType.RATE_LIMIT_REACHED: return "Rate limit reached"; + case NotificationType.AUTO_RECHARGE_SUCCESS: + return "Auto-recharge successful"; + case NotificationType.AUTO_RECHARGE_FAILED: + return "Auto-recharge failed"; default: return "Unknown notification type"; } diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index a03176da..b43aa02c 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -130,6 +130,8 @@ export enum NotificationType { APPROACHING_LIMIT = "approachingLimit", LIMIT_REACHED = "limitReached", RATE_LIMIT_REACHED = "rateLimitReached", + AUTO_RECHARGE_SUCCESS = "autoRechargeSuccess", + AUTO_RECHARGE_FAILED = "autoRechargeFailed", } export type ScrapeLog = { From d965f2ce7d9a3519da2e659ccc9ab78956b35030 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 24 Oct 2024 23:13:30 -0300 Subject: [PATCH 05/27] Nick: fixes --- apps/api/src/controllers/auth.ts | 6 +++--- apps/api/src/controllers/v1/types.ts | 2 ++ apps/api/src/services/notification/email_notification.ts | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index c9705f0f..93327e66 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -95,7 +95,7 @@ export async function getACUC( while (retries < maxRetries) { ({ data, error } = await supabase_service.rpc( - "auth_credit_usage_chunk_test_17_credit_pack", + "auth_credit_usage_chunk_test_21_credit_pack", { input_key: api_key } )); @@ -125,8 +125,8 @@ export async function getACUC( if (chunk !== null && useCache) { setCachedACUC(api_key, chunk); } - // Log the chunk for now - console.log(chunk); + + // console.log(chunk); return chunk; } else { diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 9705b855..22ac6294 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -362,6 +362,8 @@ export type AuthCreditUsageChunk = { coupons: any[]; adjusted_credits_used: number; // credits this period minus coupons used remaining_credits: number; + sub_user_id: string | null; + total_credits_sum: number; }; export interface RequestWithMaybeACUC< diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts index 1eceb06b..a94d34c4 100644 --- a/apps/api/src/services/notification/email_notification.ts +++ b/apps/api/src/services/notification/email_notification.ts @@ -109,6 +109,8 @@ export async function sendNotificationInternal( return { success: false }; } + // TODO: observation: Free credits people are not receiving notifications + const { data: recentData, error: recentError } = await supabase_service .from("user_notifications") .select("*") @@ -118,7 +120,7 @@ export async function sendNotificationInternal( .lte("sent_date", endDateString); if (recentError) { - Logger.debug(`Error fetching recent notifications: ${recentError}`); + Logger.debug(`Error fetching recent notifications: ${recentError.message}`); return { success: false }; } From 73e6db45debdefbe606c40675cd28620de7e9500 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 24 Oct 2024 23:14:41 -0300 Subject: [PATCH 06/27] Update email_notification.ts --- apps/api/src/services/notification/email_notification.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/services/notification/email_notification.ts b/apps/api/src/services/notification/email_notification.ts index a94d34c4..001f164a 100644 --- a/apps/api/src/services/notification/email_notification.ts +++ b/apps/api/src/services/notification/email_notification.ts @@ -26,7 +26,7 @@ const emailTemplates: Record< }, [NotificationType.AUTO_RECHARGE_SUCCESS]: { subject: "Auto recharge successful - Firecrawl", - html: "Hey there,

Your account was successfully recharged with 1000 credits because your remaining credits were below the threshold.


Thanks,
Firecrawl Team
", + html: "Hey there,

Your account was successfully recharged with 1000 credits because your remaining credits were below the threshold. Consider upgrading your plan at firecrawl.dev/pricing to avoid hitting the limit.


Thanks,
Firecrawl Team
", }, [NotificationType.AUTO_RECHARGE_FAILED]: { subject: "Auto recharge failed - Firecrawl", From 95c4652fd4002ccee9a4f0bce3e39192ce0ec966 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 25 Oct 2024 16:05:23 -0300 Subject: [PATCH 07/27] Nick: 10min cooldown on auto charge --- apps/api/src/services/billing/auto_charge.ts | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts index 6ab6914c..61e2bdb8 100644 --- a/apps/api/src/services/billing/auto_charge.ts +++ b/apps/api/src/services/billing/auto_charge.ts @@ -7,12 +7,13 @@ import { createPaymentIntent } from "./stripe"; import { issueCredits } from "./issue_credits"; import { sendNotification } from "../notification/email_notification"; import { NotificationType } from "../../types"; -import { deleteKey } from "../redis"; +import { deleteKey, getValue, setValue } from "../redis"; import { sendSlackWebhook } from "../alerts/slack"; import { Logger } from "../../lib/logger"; // Define the number of credits to be added during auto-recharge const AUTO_RECHARGE_CREDITS = 1000; +const AUTO_RECHARGE_COOLDOWN = 600; // 10 minutes in seconds /** * Attempt to automatically charge a user's account when their credit balance falls below a threshold @@ -24,7 +25,22 @@ export async function autoCharge( autoRechargeThreshold: number ): Promise<{ success: boolean; message: string; remainingCredits: number; chunk: AuthCreditUsageChunk }> { const resource = `auto-recharge:${chunk.team_id}`; + const cooldownKey = `auto-recharge-cooldown:${chunk.team_id}`; + try { + // Check if the team is in the cooldown period + // Another check to prevent race conditions, double charging - cool down of 10 minutes + const cooldownValue = await getValue(cooldownKey); + if (cooldownValue) { + Logger.info(`Auto-recharge for team ${chunk.team_id} is in cooldown period`); + return { + success: false, + message: "Auto-recharge is in cooldown period", + remainingCredits: chunk.remaining_credits, + chunk, + }; + } + // Use a distributed lock to prevent concurrent auto-charge attempts return await redlock.using([resource], 5000, async (signal) : Promise<{ success: boolean; message: string; remainingCredits: number; chunk: AuthCreditUsageChunk }> => { // Recheck the condition inside the lock to prevent race conditions @@ -89,6 +105,9 @@ export async function autoCharge( chunk, true ); + + // Set cooldown period + await setValue(cooldownKey, 'true', AUTO_RECHARGE_COOLDOWN); } // Reset ACUC cache to reflect the new credit balance From 97b8d6c333400e98792eafd83154ea96a7b916f5 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 25 Oct 2024 16:05:39 -0300 Subject: [PATCH 08/27] Update auto_charge.ts --- apps/api/src/services/billing/auto_charge.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts index 61e2bdb8..9669972a 100644 --- a/apps/api/src/services/billing/auto_charge.ts +++ b/apps/api/src/services/billing/auto_charge.ts @@ -13,7 +13,7 @@ import { Logger } from "../../lib/logger"; // Define the number of credits to be added during auto-recharge const AUTO_RECHARGE_CREDITS = 1000; -const AUTO_RECHARGE_COOLDOWN = 600; // 10 minutes in seconds +const AUTO_RECHARGE_COOLDOWN = 300; // 5 minutes in seconds /** * Attempt to automatically charge a user's account when their credit balance falls below a threshold @@ -29,7 +29,7 @@ export async function autoCharge( try { // Check if the team is in the cooldown period - // Another check to prevent race conditions, double charging - cool down of 10 minutes + // Another check to prevent race conditions, double charging - cool down of 5 minutes const cooldownValue = await getValue(cooldownKey); if (cooldownValue) { Logger.info(`Auto-recharge for team ${chunk.team_id} is in cooldown period`); From 801f0f773e82abf731f20ab3ad968f461b542eb2 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 26 Oct 2024 03:59:15 -0300 Subject: [PATCH 09/27] Nick: fix auto charge failing when payment is through Link --- apps/api/src/services/billing/auto_charge.ts | 13 +++++++++++-- apps/api/src/services/billing/stripe.ts | 11 ++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.ts index 9669972a..5f318321 100644 --- a/apps/api/src/services/billing/auto_charge.ts +++ b/apps/api/src/services/billing/auto_charge.ts @@ -113,15 +113,24 @@ export async function autoCharge( // Reset ACUC cache to reflect the new credit balance const cacheKeyACUC = `acuc_${chunk.api_key}`; await deleteKey(cacheKeyACUC); - if (process.env.SLACK_ADMIN_WEBHOOK_URL ) { + + if (process.env.SLACK_ADMIN_WEBHOOK_URL) { + const webhookCooldownKey = `webhook_cooldown_${chunk.team_id}`; + const isInCooldown = await getValue(webhookCooldownKey); + + if (!isInCooldown) { sendSlackWebhook( - `Auto-recharge successful: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}. User was notified via email.`, + `Auto-recharge: Team ${chunk.team_id}. ${AUTO_RECHARGE_CREDITS} credits added. Payment status: ${paymentStatus.return_status}.`, false, process.env.SLACK_ADMIN_WEBHOOK_URL ).catch((error) => { Logger.debug(`Error sending slack notification: ${error}`); }); + + // Set cooldown for 1 hour + await setValue(webhookCooldownKey, 'true', 60 * 60); } + } return { success: true, message: "Auto-recharge successful", diff --git a/apps/api/src/services/billing/stripe.ts b/apps/api/src/services/billing/stripe.ts index f1400804..e459d3e3 100644 --- a/apps/api/src/services/billing/stripe.ts +++ b/apps/api/src/services/billing/stripe.ts @@ -7,7 +7,7 @@ async function getCustomerDefaultPaymentMethod(customerId: string) { const paymentMethods = await stripe.customers.listPaymentMethods(customerId, { limit: 3, }); - return paymentMethods.data[0]?.id; + return paymentMethods.data[0] ?? null; } type ReturnStatus = "succeeded" | "requires_action" | "failed"; @@ -16,13 +16,18 @@ export async function createPaymentIntent( customer_id: string ): Promise<{ return_status: ReturnStatus; charge_id: string }> { try { + const defaultPaymentMethod = await getCustomerDefaultPaymentMethod(customer_id); + if (!defaultPaymentMethod) { + Logger.error(`No default payment method found for customer: ${customer_id}`); + return { return_status: "failed", charge_id: "" }; + } const paymentIntent = await stripe.paymentIntents.create({ amount: 1100, currency: "usd", customer: customer_id, description: "Firecrawl: Auto re-charge of 1000 credits", - payment_method_types: ["card"], - payment_method: await getCustomerDefaultPaymentMethod(customer_id), + payment_method_types: [defaultPaymentMethod?.type ?? "card"], + payment_method: defaultPaymentMethod?.id, off_session: true, confirm: true, }); From 9593ab80e11b08fba076235a4e90bdd6cb0b9487 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 26 Oct 2024 16:03:07 -0300 Subject: [PATCH 10/27] Update README.md --- apps/js-sdk/firecrawl/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md index a90907ba..e404a317 100644 --- a/apps/js-sdk/firecrawl/README.md +++ b/apps/js-sdk/firecrawl/README.md @@ -147,7 +147,7 @@ watch.addEventListener("done", state => { ### Batch scraping multiple URLs -To batch scrape multiple URLs with error handling, use the `batchScrapeUrls` method. It takes the starting URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the output formats. +To batch scrape multiple URLs with error handling, use the `batchScrapeUrls` method. It takes the starting URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the batch scrape job, such as the output formats. ```js const batchScrapeResponse = await app.batchScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], { @@ -158,10 +158,10 @@ const batchScrapeResponse = await app.batchScrapeUrls(['https://firecrawl.dev', #### Asynchronous batch scrape -To initiate an asynchronous batch scrape, utilize the `asyncBulkScrapeUrls` method. This method requires the starting URLs and optional parameters as inputs. The params argument enables you to define various settings for the scrape, such as the output formats. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the batch scrape. +To initiate an asynchronous batch scrape, utilize the `asyncBatchScrapeUrls` method. This method requires the starting URLs and optional parameters as inputs. The params argument enables you to define various settings for the scrape, such as the output formats. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the batch scrape. ```js -const asyncBulkScrapeResult = await app.asyncBulkScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] }); +const asyncBatchScrapeResult = await app.asyncBatchScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] }); ``` #### Batch scrape with WebSockets From 8a4f4cb9d98884bc70f4cf188a2c4dc87f656462 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 26 Oct 2024 16:10:51 -0300 Subject: [PATCH 11/27] Update README.md --- apps/python-sdk/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md index 412c3e05..abae05de 100644 --- a/apps/python-sdk/README.md +++ b/apps/python-sdk/README.md @@ -170,11 +170,11 @@ print(batch_scrape_result) ### Checking batch scrape status -To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_job` method. It takes the job ID as a parameter and returns the current status of the batch scrape job. +To check the status of an asynchronous batch scrape job, use the `check_batch_scrape_status` method. It takes the job ID as a parameter and returns the current status of the batch scrape job. ```python id = batch_scrape_result['id'] -status = app.check_batch_scrape_job(id) +status = app.check_batch_scrape_status(id) ``` ### Batch scrape with WebSockets From 68b2e1b20966733494c09db0951d8b5d27d6c298 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 27 Oct 2024 23:14:25 -0300 Subject: [PATCH 12/27] Update log_job.ts --- apps/api/src/services/logging/log_job.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index 4d8ee014..c2aedd13 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -70,7 +70,9 @@ export async function logJob(job: FirecrawlJob) { retry: job.retry, }, }; - posthog.capture(phLog); + if(job.mode !== "single_urls") { + posthog.capture(phLog); + } } if (error) { Logger.error(`Error logging job: ${error.message}`); From 877d5e4383bde79f5aab4b2bbaa804c2497a0bdd Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 27 Oct 2024 23:17:20 -0300 Subject: [PATCH 13/27] Update types.ts --- apps/api/src/controllers/v1/types.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 22ac6294..8c60c0fb 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -109,6 +109,16 @@ export const scrapeOptions = z.object({ extract: extractOptions.optional(), parsePDF: z.boolean().default(true), actions: actionsSchema.optional(), + // New + location: z.object({ + country: z.string().optional().refine( + (val) => !val || Object.keys(countries).includes(val.toUpperCase()), + { + message: "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.", + } + ).transform(val => val ? val.toUpperCase() : 'US') + }).optional(), + // Deprecated geolocation: z.object({ country: z.string().optional().refine( (val) => !val || Object.keys(countries).includes(val.toUpperCase()), @@ -445,7 +455,7 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions { fullPageScreenshot: x.formats.includes("screenshot@fullPage"), parsePDF: x.parsePDF, actions: x.actions as Action[], // no strict null checking grrrr - mogery - geolocation: x.geolocation, + geolocation: x.location ?? x.geolocation, skipTlsVerification: x.skipTlsVerification }; } From b48eed5716d7bad5f19702bce839b2998fe8aaf6 Mon Sep 17 00:00:00 2001 From: Twilight <46562212+twlite@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:40:35 +0545 Subject: [PATCH 14/27] chore(README.md): use `satisfies` instead of `as` for ts example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dd8a740a..e7c48fa5 100644 --- a/README.md +++ b/README.md @@ -483,7 +483,7 @@ const crawlResponse = await app.crawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html'], } -} as CrawlParams, true, 30) as CrawlStatusResponse; +} satisfies CrawlParams, true, 30) satisfies CrawlStatusResponse; if (crawlResponse) { console.log(crawlResponse) From e3e8375c7de0c64df66a56f0b3f1d9ddc4fd2c9c Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Mon, 28 Oct 2024 11:13:33 -0400 Subject: [PATCH 15/27] Add AgentOps Monitoring --- examples/claude_web_crawler/claude_web_crawler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/claude_web_crawler/claude_web_crawler.py b/examples/claude_web_crawler/claude_web_crawler.py index 55168f30..6ca29f14 100644 --- a/examples/claude_web_crawler/claude_web_crawler.py +++ b/examples/claude_web_crawler/claude_web_crawler.py @@ -3,6 +3,7 @@ from firecrawl import FirecrawlApp import json from dotenv import load_dotenv import anthropic +import agentops # ANSI color codes class Colors: @@ -161,4 +162,5 @@ def main(): print(f"{Colors.RED}No relevant pages identified. Consider refining the search parameters or trying a different website.{Colors.RESET}") if __name__ == "__main__": + agentops.init(os.getenv("AGENTOPS_API_KEY")) main() From 007e3edfc5c785da858f11a94f14ea7a5bd28ff0 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 12:40:04 -0300 Subject: [PATCH 16/27] Update README.md --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index dd8a740a..d8e5bdcb 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ To use the API, you need to sign up on [Firecrawl](https://firecrawl.dev) and ge - **Media parsing**: pdfs, docx, images. - **Reliability first**: designed to get the data you need - no matter how hard it is. - **Actions**: click, scroll, input, wait and more before extracting data +- **Batching (New)**: scrape thousands of URLs at the same time with a new async endpoint You can find all of Firecrawl's capabilities and how to use them in our [documentation](https://docs.firecrawl.dev) @@ -350,6 +351,19 @@ curl -X POST https://api.firecrawl.dev/v1/scrape \ }' ``` +### Batch Scraping Multiple URLs (New) + +You can now batch scrape multiple URLs at the same time. It is very similar to how the /crawl endpoint works. It submits a batch scrape job and returns a job ID to check the status of the batch scrape. + +```bash +curl -X POST https://api.firecrawl.dev/v1/batch/scrape \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer YOUR_API_KEY' \ + -d '{ + "urls": ["https://docs.firecrawl.dev", "https://docs.firecrawl.dev/sdks/overview"], + "formats" : ["markdown", "html"] + }' +``` ### Search (v0) (Beta) From fa8875d64d4246f0d0b7eecbfcffbbccce473033 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 15:09:50 -0300 Subject: [PATCH 17/27] Update single_url.ts --- apps/api/src/scraper/WebScraper/single_url.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts index cd76793c..c7185b79 100644 --- a/apps/api/src/scraper/WebScraper/single_url.ts +++ b/apps/api/src/scraper/WebScraper/single_url.ts @@ -209,14 +209,15 @@ export async function scrapSingleUrl( if (action.type === "click" || action.type === "write" || action.type === "press") { const result: Action[] = []; // Don't add a wait if the previous action is a wait - if (index === 0 || array[index - 1].type !== "wait") { - result.push({ type: "wait", milliseconds: 1200 } as Action); - } + // if (index === 0 || array[index - 1].type !== "wait") { + // result.push({ type: "wait", milliseconds: 1200 } as Action); + // } + // Fire-engine now handles wait times automatically, leaving the code here for now result.push(action); // Don't add a wait if the next action is a wait - if (index === array.length - 1 || array[index + 1].type !== "wait") { - result.push({ type: "wait", milliseconds: 1200 } as Action); - } + // if (index === array.length - 1 || array[index + 1].type !== "wait") { + // result.push({ type: "wait", milliseconds: 1200 } as Action); + // } return result; } return [action as Action]; From 726430c2e641666626b400ca62f32062a31978f4 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 16:51:49 -0300 Subject: [PATCH 18/27] Nick: llm extract in batch scrape --- apps/api/src/controllers/v1/batch-scrape.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts index 7c68341b..cde4bd76 100644 --- a/apps/api/src/controllers/v1/batch-scrape.ts +++ b/apps/api/src/controllers/v1/batch-scrape.ts @@ -4,6 +4,7 @@ import { BatchScrapeRequest, batchScrapeRequestSchema, CrawlResponse, + legacyExtractorOptions, legacyScrapeOptions, RequestWithAuth, } from "./types"; @@ -34,6 +35,8 @@ export async function batchScrapeController( } const pageOptions = legacyScrapeOptions(req.body); + const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined; + const sc: StoredCrawl = { crawlerOptions: null, @@ -65,6 +68,7 @@ export async function batchScrapeController( plan: req.auth.plan, crawlerOptions: null, pageOptions, + extractorOptions, origin: "api", crawl_id: id, sitemapped: true, From 0bad436061191fd304dd66ea201eccc97f0e7474 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 17:04:42 -0300 Subject: [PATCH 19/27] Nick: fixed the batch scrape + llm extract billing --- apps/api/src/main/runWebScraper.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index 8eb679e7..8bd0c12c 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -121,8 +121,13 @@ export async function runWebScraper({ : docs; if(is_scrape === false) { - billTeam(team_id, undefined, filteredDocs.length).catch(error => { - Logger.error(`Failed to bill team ${team_id} for ${filteredDocs.length} credits: ${error}`); + let creditsToBeBilled = 1; // Assuming 1 credit per document + if (extractorOptions && (extractorOptions.mode === "llm-extraction" || extractorOptions.mode === "extract")) { + creditsToBeBilled = 5; + } + + billTeam(team_id, undefined, creditsToBeBilled * filteredDocs.length).catch(error => { + Logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled * filteredDocs.length} credits: ${error}`); // Optionally, you could notify an admin or add to a retry queue here }); } From 3d1bb82aa27865bfda143a952c4b33b0f8b7b18c Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 20:16:11 -0300 Subject: [PATCH 20/27] Nick: languages support --- apps/api/src/controllers/v1/types.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 8c60c0fb..633bbdf1 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -116,8 +116,10 @@ export const scrapeOptions = z.object({ { message: "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.", } - ).transform(val => val ? val.toUpperCase() : 'US') + ).transform(val => val ? val.toUpperCase() : 'US'), + languages: z.string().array().optional(), }).optional(), + // Deprecated geolocation: z.object({ country: z.string().optional().refine( @@ -125,7 +127,8 @@ export const scrapeOptions = z.object({ { message: "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.", } - ).transform(val => val ? val.toUpperCase() : 'US') + ).transform(val => val ? val.toUpperCase() : 'US'), + languages: z.string().array().optional(), }).optional(), skipTlsVerification: z.boolean().default(false), }).strict(strictMessage) From b6ce49e5bbc06a1e5027e794fcb3f7b4163f1704 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 20:24:16 -0300 Subject: [PATCH 21/27] Update index.ts --- apps/js-sdk/firecrawl/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 491df1e4..bbe934fe 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -82,6 +82,10 @@ export interface CrawlScrapeOptions { onlyMainContent?: boolean; waitFor?: number; timeout?: number; + location?: { + country?: string; + languages?: string[]; + }; } export type Action = { From 6d38c65466ca66731b367141480db585dd87dce9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 28 Oct 2024 20:25:28 -0300 Subject: [PATCH 22/27] Update package.json --- apps/js-sdk/firecrawl/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index a7fb2d83..b8738e5e 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "1.7.1", + "version": "1.7.2", "description": "JavaScript SDK for Firecrawl API", "main": "dist/index.js", "types": "dist/index.d.ts", From 7152ac8856c35e0e1dde39d112c27ffd679f3ec4 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 29 Oct 2024 15:58:20 -0300 Subject: [PATCH 23/27] Update auth.ts --- apps/api/src/controllers/auth.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 93327e66..1b842e3e 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -354,6 +354,7 @@ function getPlanByPriceId(price_id: string): PlanType { return "standardnew"; case process.env.STRIPE_PRICE_ID_GROWTH: case process.env.STRIPE_PRICE_ID_GROWTH_YEARLY: + case price_id: return "growth"; case process.env.STRIPE_PRICE_ID_GROWTH_DOUBLE_MONTHLY: return "growthdouble"; From 6948ca6fe15158fd4abb1c58c49cfa54ea9ebdb5 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 29 Oct 2024 16:02:21 -0300 Subject: [PATCH 24/27] Revert "Update auth.ts" This reverts commit 7152ac8856c35e0e1dde39d112c27ffd679f3ec4. --- apps/api/src/controllers/auth.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 1b842e3e..93327e66 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -354,7 +354,6 @@ function getPlanByPriceId(price_id: string): PlanType { return "standardnew"; case process.env.STRIPE_PRICE_ID_GROWTH: case process.env.STRIPE_PRICE_ID_GROWTH_YEARLY: - case price_id: return "growth"; case process.env.STRIPE_PRICE_ID_GROWTH_DOUBLE_MONTHLY: return "growthdouble"; From e0ba339c509a5f4204c8de819655bbea30d0030d Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 29 Oct 2024 16:06:12 -0300 Subject: [PATCH 25/27] Update auth.ts --- apps/api/src/controllers/auth.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 93327e66..0736ecd8 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -354,6 +354,7 @@ function getPlanByPriceId(price_id: string): PlanType { return "standardnew"; case process.env.STRIPE_PRICE_ID_GROWTH: case process.env.STRIPE_PRICE_ID_GROWTH_YEARLY: + case process.env.STRIPE_PRICE_ID_SCALE_2M: return "growth"; case process.env.STRIPE_PRICE_ID_GROWTH_DOUBLE_MONTHLY: return "growthdouble"; From 983f344fa8da984a080f8b30ae1d9c59756b3dae Mon Sep 17 00:00:00 2001 From: Eric Ciarla Date: Tue, 29 Oct 2024 15:38:57 -0400 Subject: [PATCH 26/27] Create claude_stock_analyzer.py --- .../claude_stock_analyzer.py | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 examples/claude_stock_analyzer/claude_stock_analyzer.py diff --git a/examples/claude_stock_analyzer/claude_stock_analyzer.py b/examples/claude_stock_analyzer/claude_stock_analyzer.py new file mode 100644 index 00000000..42acdc20 --- /dev/null +++ b/examples/claude_stock_analyzer/claude_stock_analyzer.py @@ -0,0 +1,180 @@ +import os +from firecrawl import FirecrawlApp +import json +from dotenv import load_dotenv +import anthropic +from e2b_code_interpreter import Sandbox +import base64 + +# ANSI color codes +class Colors: + CYAN = '\033[96m' + YELLOW = '\033[93m' + GREEN = '\033[92m' + RED = '\033[91m' + MAGENTA = '\033[95m' + BLUE = '\033[94m' + RESET = '\033[0m' + +# Load environment variables +load_dotenv() + +# Retrieve API keys from environment variables +firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY") +anthropic_api_key = os.getenv("ANTHROPIC_API_KEY") +e2b_api_key = os.getenv("E2B_API_KEY") + +# Initialize the FirecrawlApp and Anthropic client +app = FirecrawlApp(api_key=firecrawl_api_key) +client = anthropic.Anthropic(api_key=anthropic_api_key) +sandbox = Sandbox(api_key=e2b_api_key) + +# Find the relevant stock pages via map +def find_relevant_page_via_map(stock_search_term, url, app): + try: + print(f"{Colors.CYAN}Searching for stock: {stock_search_term}{Colors.RESET}") + print(f"{Colors.CYAN}Initiating search on the website: {url}{Colors.RESET}") + + map_search_parameter = stock_search_term + + print(f"{Colors.GREEN}Search parameter: {map_search_parameter}{Colors.RESET}") + + print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}") + map_website = app.map_url(url, params={"search": map_search_parameter}) + print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}") + print(f"{Colors.GREEN}Located {len(map_website['links'])} relevant links.{Colors.RESET}") + return map_website['links'] + except Exception as e: + print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}") + return None + +# Function to plot the scores using e2b +def plot_scores(stock_names, stock_scores): + print(f"{Colors.YELLOW}Plotting scores...{Colors.RESET}") + code_to_run = f""" +import matplotlib.pyplot as plt + +stock_names = {stock_names} +stock_scores = {stock_scores} + +plt.figure(figsize=(10, 5)) +plt.bar(stock_names, stock_scores, color='blue') +plt.xlabel('Stock Names') +plt.ylabel('Scores') +plt.title('Stock Investment Scores') +plt.xticks(rotation=45) +plt.tight_layout() +plt.savefig('chart.png') +plt.show() +""" + # Run the code inside the sandbox + execution = sandbox.run_code(code_to_run) + + # Check if there are any results + if execution.results and execution.results[0].png: + first_result = execution.results[0] + + # Get the directory where the current python file is located + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Save the png to a file in the examples directory. The png is in base64 format. + with open(os.path.join(current_dir, 'chart.png'), 'wb') as f: + f.write(base64.b64decode(first_result.png)) + print('Chart saved as examples/chart.png') + else: + print(f"{Colors.RED}No results returned from the sandbox execution.{Colors.RESET}") + +# Analyze the top stocks and provide investment recommendation +def analyze_top_stocks(map_website, app, client): + try: + # Get top 5 links from the map result + top_links = map_website[:10] + print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}") + + # Scrape the pages in batch + batch_scrape_result = app.batch_scrape_urls(top_links, {'formats': ['markdown']}) + print(f"{Colors.GREEN}Batch page scraping completed successfully.{Colors.RESET}") + + # Prepare content for LLM + stock_contents = [] + for scrape_result in batch_scrape_result['data']: + stock_contents.append({ + 'content': scrape_result['markdown'] + }) + + # Pass all the content to the LLM to analyze and decide which stock to invest in + analyze_prompt = f""" +Based on the following information about different stocks from their Robinhood pages, analyze and determine which stock is the best investment opportunity. DO NOT include any other text, just the JSON. + +Return the result in the following JSON format. Only return the JSON, nothing else. Do not include backticks or any other formatting, just the JSON. +{{ + "scores": [ + {{ + "stock_name": "", + "score": + }}, + ... + ] +}} + +Stock Information: +""" + + for stock in stock_contents: + analyze_prompt += f"Content:\n{stock['content']}\n" + + print(f"{Colors.YELLOW}Analyzing stock information with LLM...{Colors.RESET}") + analyze_prompt += f"\n\nStart JSON:\n" + completion = client.messages.create( + model="claude-3-5-sonnet-20240620", + max_tokens=1000, + temperature=0, + system="You are a financial analyst. Only return the JSON, nothing else.", + messages=[ + { + "role": "user", + "content": analyze_prompt + } + ] + ) + + result = completion.content[0].text + print(f"{Colors.GREEN}Analysis completed. Here is the recommendation:{Colors.RESET}") + print(f"{Colors.MAGENTA}{result}{Colors.RESET}") + + # Plot the scores using e2b + try: + result_json = json.loads(result) + scores = result_json['scores'] + stock_names = [score['stock_name'] for score in scores] + stock_scores = [score['score'] for score in scores] + + plot_scores(stock_names, stock_scores) + except json.JSONDecodeError as json_err: + print(f"{Colors.RED}Error decoding JSON response: {str(json_err)}{Colors.RESET}") + + except Exception as e: + print(f"{Colors.RED}Error encountered during stock analysis: {str(e)}{Colors.RESET}") + +# Main function to execute the process +def main(): + # Get user input + stock_search_term = input(f"{Colors.BLUE}Enter the stock you're interested in: {Colors.RESET}") + if not stock_search_term.strip(): + print(f"{Colors.RED}No stock entered. Exiting.{Colors.RESET}") + return + + url = "https://robinhood.com/stocks" + + print(f"{Colors.YELLOW}Initiating stock analysis process...{Colors.RESET}") + # Find the relevant pages + map_website = find_relevant_page_via_map(stock_search_term, url, app) + + if map_website: + print(f"{Colors.GREEN}Relevant stock pages identified. Proceeding with detailed analysis...{Colors.RESET}") + # Analyze top stocks + analyze_top_stocks(map_website, app, client) + else: + print(f"{Colors.RED}No relevant stock pages identified. Consider refining the search term or trying a different stock.{Colors.RESET}") + +if __name__ == "__main__": + main() From 96c579f1cd3b507eb7ff6e72588d6ade23398280 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 29 Oct 2024 21:01:43 -0300 Subject: [PATCH 27/27] Nick: etier2c --- apps/api/src/controllers/auth.ts | 2 ++ apps/api/src/lib/job-priority.ts | 4 ++++ apps/api/src/services/rate-limiter.ts | 4 ++++ apps/api/src/types.ts | 1 + 4 files changed, 11 insertions(+) diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts index 0736ecd8..20c4a60a 100644 --- a/apps/api/src/controllers/auth.ts +++ b/apps/api/src/controllers/auth.ts @@ -358,6 +358,8 @@ function getPlanByPriceId(price_id: string): PlanType { return "growth"; case process.env.STRIPE_PRICE_ID_GROWTH_DOUBLE_MONTHLY: return "growthdouble"; + case process.env.STRIPE_PRICE_ID_ETIER2C: + return "etier2c"; default: return "free"; } diff --git a/apps/api/src/lib/job-priority.ts b/apps/api/src/lib/job-priority.ts index 83fefcec..6108e131 100644 --- a/apps/api/src/lib/job-priority.ts +++ b/apps/api/src/lib/job-priority.ts @@ -70,6 +70,10 @@ export async function getJobPriority({ bucketLimit = 400; planModifier = 0.1; break; + case "etier2c": + bucketLimit = 1000; + planModifier = 0.05; + break; default: bucketLimit = 25; diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts index ef3d78f8..480db9f3 100644 --- a/apps/api/src/services/rate-limiter.ts +++ b/apps/api/src/services/rate-limiter.ts @@ -15,6 +15,7 @@ const RATE_LIMITS = { standardnew: 10, growth: 50, growthdouble: 50, + etier2c: 300, }, scrape: { default: 20, @@ -28,6 +29,7 @@ const RATE_LIMITS = { standardnew: 100, growth: 1000, growthdouble: 1000, + etier2c: 2500, }, search: { default: 20, @@ -41,6 +43,7 @@ const RATE_LIMITS = { standardnew: 50, growth: 500, growthdouble: 500, + etier2c: 2500, }, map:{ default: 20, @@ -54,6 +57,7 @@ const RATE_LIMITS = { standardnew: 50, growth: 500, growthdouble: 500, + etier2c: 2500, }, preview: { free: 5, diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index bd11667b..216786d9 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -157,6 +157,7 @@ export type PlanType = | "standardnew" | "growth" | "growthdouble" + | "etier2c" | "free" | "";