AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 20:29:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cost-tracking): add model tracking and more costs

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-17 11:33:01 -07:00
parent 5aa9469081
commit 7df557e59c
4 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/lib/extract/extraction-service.ts
View File

@ -72,6 +72,7 @@ export class CostTracking {
type: "smartScrape" | "other", type: "smartScrape" | "other",
metadata: Record<string, any>, metadata: Record<string, any>,
cost: number, cost: number,
model: string,
tokens?: { tokens?: {
input: number, input: number,
output: number, output: number,

2
apps/api/src/lib/extract/url-processor.ts
View File

@ -28,6 +28,7 @@ export async function generateBasicCompletion(prompt: string, costTracking: Cost
module: "extract", module: "extract",
method: "generateBasicCompletion", method: "generateBasicCompletion",
}, },
model: "openai/gpt-4o",
cost: calculateCost("openai/gpt-4o", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0), cost: calculateCost("openai/gpt-4o", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0),
tokens: { tokens: {
input: result.usage?.promptTokens ?? 0, input: result.usage?.promptTokens ?? 0,
@ -54,6 +55,7 @@ export async function generateBasicCompletion(prompt: string, costTracking: Cost
module: "extract", module: "extract",
method: "generateBasicCompletion", method: "generateBasicCompletion",
}, },
model: "openai/gpt-4o-mini",
cost: calculateCost("openai/gpt-4o-mini", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0), cost: calculateCost("openai/gpt-4o-mini", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0),
tokens: { tokens: {
input: result.usage?.promptTokens ?? 0, input: result.usage?.promptTokens ?? 0,

1
apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
View File

@ -144,6 +144,7 @@ export async function smartScrape({
costTracking.addCall({ costTracking.addCall({
type: "smartScrape", type: "smartScrape",
cost: response.tokenUsage, cost: response.tokenUsage,
model: "firecrawl/smart-scrape",
metadata: { metadata: {
module: "smartScrape", module: "smartScrape",
method: "smartScrape", method: "smartScrape",

7
apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
View File

@ -187,6 +187,8 @@ export function calculateCost(
) { ) {
const modelCosts = { const modelCosts = {
"openai/o3-mini": { input_cost: 1.1, output_cost: 4.4 }, "openai/o3-mini": { input_cost: 1.1, output_cost: 4.4 },
"openai/gpt-4o-mini": { input_cost: 0.15, output_cost: 0.6 },
"openai/gpt-4o": { input_cost: 2.5, output_cost: 10 },
"google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 }, "google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 },
"deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 }, "deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 },
"google/gemini-2.0-flash-thinking-exp:free": { "google/gemini-2.0-flash-thinking-exp:free": {
@ -289,6 +291,7 @@ export async function generateCompletions({
...costTrackingOptions.metadata, ...costTrackingOptions.metadata,
gcDetails: "no-object", gcDetails: "no-object",
}, },
model: currentModel.modelId,
cost: calculateCost( cost: calculateCost(
currentModel.modelId, currentModel.modelId,
result.usage?.promptTokens ?? 0, result.usage?.promptTokens ?? 0,
@ -340,6 +343,7 @@ export async function generateCompletions({
...costTrackingOptions.metadata, ...costTrackingOptions.metadata,
gcDetails: "no-object fallback", gcDetails: "no-object fallback",
}, },
model: currentModel.modelId,
cost: calculateCost( cost: calculateCost(
currentModel.modelId, currentModel.modelId,
result.usage?.promptTokens ?? 0, result.usage?.promptTokens ?? 0,
@ -454,6 +458,7 @@ export async function generateCompletions({
repairUsage?.promptTokens ?? 0, repairUsage?.promptTokens ?? 0,
repairUsage?.completionTokens ?? 0, repairUsage?.completionTokens ?? 0,
), ),
model: currentModel.modelId,
tokens: { tokens: {
input: repairUsage?.promptTokens ?? 0, input: repairUsage?.promptTokens ?? 0,
output: repairUsage?.completionTokens ?? 0, output: repairUsage?.completionTokens ?? 0,
@ -513,6 +518,7 @@ export async function generateCompletions({
input: result.usage?.promptTokens ?? 0, input: result.usage?.promptTokens ?? 0,
output: result.usage?.completionTokens ?? 0, output: result.usage?.completionTokens ?? 0,
}, },
model: currentModel.modelId,
cost: calculateCost( cost: calculateCost(
currentModel.modelId, currentModel.modelId,
result.usage?.promptTokens ?? 0, result.usage?.promptTokens ?? 0,
@ -547,6 +553,7 @@ export async function generateCompletions({
input: result.usage?.promptTokens ?? 0, input: result.usage?.promptTokens ?? 0,
output: result.usage?.completionTokens ?? 0, output: result.usage?.completionTokens ?? 0,
}, },
model: currentModel.modelId,
cost: calculateCost( cost: calculateCost(
currentModel.modelId, currentModel.modelId,
result.usage?.promptTokens ?? 0, result.usage?.promptTokens ?? 0,