AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 06:18:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(smart-scrape): log failed costs

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-19 01:07:00 -07:00
parent 3909b6641d
commit 2b2e648d41
1 changed files with 48 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
View File

@ -3,7 +3,7 @@ import { logger as _logger } from "../../../lib/logger";
import { robustFetch } from "./fetch"; import { robustFetch } from "./fetch";
import fs from "fs/promises"; import fs from "fs/promises";
import { configDotenv } from "dotenv"; import { configDotenv } from "dotenv";
import { CostTracking } from "../../../lib/extract/extraction-service"; import { CostLimitExceededError, CostTracking } from "../../../lib/extract/extraction-service";
configDotenv(); configDotenv();
// Define schemas outside the function scope // Define schemas outside the function scope
@ -119,12 +119,25 @@ export async function smartScrape({
errorResponse.success === false && errorResponse.success === false &&
errorResponse.error errorResponse.error
) { ) {
if (errorResponse.error === "Cost limit exceeded") { if ((errorResponse as any).tokenUsage) {
throw new Error("Cost limit exceeded", { logger.info("Failed smart scrape cost $" + (errorResponse as any).tokenUsage);
cause: { tokenUsage: (errorResponse as any).tokenUsage }, costTracking.addCall({
type: "smartScrape",
cost: (errorResponse as any).tokenUsage,
model: "firecrawl/smart-scrape",
metadata: {
module: "smartScrape",
method: "smartScrape",
url,
sessionId,
},
}); });
} }
if (errorResponse.error === "Cost limit exceeded") {
throw new CostLimitExceededError();
}
logger.error("Smart scrape returned error response", { logger.error("Smart scrape returned error response", {
url, url,
prompt, prompt,
@ -155,6 +168,36 @@ export async function smartScrape({
return response; // The response type now matches SmartScrapeResult return response; // The response type now matches SmartScrapeResult
} catch (error) { } catch (error) {
if (error instanceof CostLimitExceededError) {
throw error;
}
if (error instanceof Error && error.message === "Request sent failure status" && error.cause && (error.cause as any).response) {
const response = (error.cause as any).response;
try {
const json = JSON.parse(response.body);
if (json.tokenUsage) {
logger.info("Failed smart scrape cost $" + json.tokenUsage);
costTracking.addCall({
type: "smartScrape",
cost: json.tokenUsage,
model: "firecrawl/smart-scrape",
metadata: {
module: "smartScrape",
method: "smartScrape",
url,
sessionId,
},
});
}
if (json.error === "Cost limit exceeded") {
throw new CostLimitExceededError();
}
} catch (e) {}
}
// Safely extract error information without circular references // Safely extract error information without circular references
const errorInfo = { const errorInfo = {
message: error instanceof Error ? error.message : String(error), message: error instanceof Error ? error.message : String(error),
@ -185,7 +228,7 @@ export async function smartScrape({
}; };
logger.error("Smart scrape request failed", { logger.error("Smart scrape request failed", {
error: JSON.stringify(errorInfo), error: errorInfo
}); });
// Rethrowing the error to be handled by the caller // Rethrowing the error to be handled by the caller