AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 02:49:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(smart-scrape): log failed costs

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-19 01:07:00 -07:00
parent 3909b6641d
commit 2b2e648d41
1 changed files with 48 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
View File

@ -3,7 +3,7 @@ import { logger as _logger } from "../../../lib/logger";
import { robustFetch } from "./fetch";
import fs from "fs/promises";
import { configDotenv } from "dotenv";
import { CostTracking } from "../../../lib/extract/extraction-service";
import { CostLimitExceededError, CostTracking } from "../../../lib/extract/extraction-service";
configDotenv();
// Define schemas outside the function scope
@ -119,12 +119,25 @@ export async function smartScrape({
errorResponse.success === false &&
errorResponse.error
) {
if (errorResponse.error === "Cost limit exceeded") {
throw new Error("Cost limit exceeded", {
cause: { tokenUsage: (errorResponse as any).tokenUsage },
if ((errorResponse as any).tokenUsage) {
logger.info("Failed smart scrape cost $" + (errorResponse as any).tokenUsage);
costTracking.addCall({
type: "smartScrape",
cost: (errorResponse as any).tokenUsage,
model: "firecrawl/smart-scrape",
metadata: {
module: "smartScrape",
method: "smartScrape",
url,
sessionId,
},
});
}
if (errorResponse.error === "Cost limit exceeded") {
throw new CostLimitExceededError();
}
logger.error("Smart scrape returned error response", {
url,
prompt,
@ -155,6 +168,36 @@ export async function smartScrape({
return response; // The response type now matches SmartScrapeResult
} catch (error) {
if (error instanceof CostLimitExceededError) {
throw error;
}
if (error instanceof Error && error.message === "Request sent failure status" && error.cause && (error.cause as any).response) {
const response = (error.cause as any).response;
try {
const json = JSON.parse(response.body);
if (json.tokenUsage) {
logger.info("Failed smart scrape cost $" + json.tokenUsage);
costTracking.addCall({
type: "smartScrape",
cost: json.tokenUsage,
model: "firecrawl/smart-scrape",
metadata: {
module: "smartScrape",
method: "smartScrape",
url,
sessionId,
},
});
}
if (json.error === "Cost limit exceeded") {
throw new CostLimitExceededError();
}
} catch (e) {}
}
// Safely extract error information without circular references
const errorInfo = {
message: error instanceof Error ? error.message : String(error),
@ -185,7 +228,7 @@ export async function smartScrape({
};
logger.error("Smart scrape request failed", {
error: JSON.stringify(errorInfo),
error: errorInfo
});
// Rethrowing the error to be handled by the caller