AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 21:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

(feat/deep-research) Improvements to final analysis (#1443)

Browse Source 
* Nick: fixes

* Update research-manager.ts

* Update research-manager.ts
This commit is contained in:
Nicolas 2025-04-10 22:59:28 -07:00 committed by GitHub
parent 0bed648b0d
commit f52d6aab91
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 66 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/package.json
View File

@ -51,7 +51,7 @@
"typescript": "^5.8.3" "typescript": "^5.8.3"
}, },
"dependencies": { "dependencies": {
"@ai-sdk/openai": "^1.1.13", "@ai-sdk/openai": "^1.3.10",
"@anthropic-ai/sdk": "^0.24.3", "@anthropic-ai/sdk": "^0.24.3",
"@apidevtools/json-schema-ref-parser": "^11.7.3", "@apidevtools/json-schema-ref-parser": "^11.7.3",
"@brillout/import": "^0.2.2", "@brillout/import": "^0.2.2",
@ -68,7 +68,7 @@
"@supabase/supabase-js": "^2.44.2", "@supabase/supabase-js": "^2.44.2",
"@types/express-ws": "^3.0.4", "@types/express-ws": "^3.0.4",
"@types/ws": "^8.5.12", "@types/ws": "^8.5.12",
"ai": "^4.1.45", "ai": "^4.3.4",
"ajv": "^8.16.0", "ajv": "^8.16.0",
"async": "^3.2.5", "async": "^3.2.5",
"async-mutex": "^0.5.0", "async-mutex": "^0.5.0",

93
apps/api/pnpm-lock.yaml generated
View File

@ -9,8 +9,8 @@ importers:
.: .:
dependencies: dependencies:
'@ai-sdk/openai': '@ai-sdk/openai':
specifier: ^1.1.13 specifier: ^1.3.10
version: 1.1.13(zod@3.24.2) version: 1.3.10(zod@3.24.2)
'@anthropic-ai/sdk': '@anthropic-ai/sdk':
specifier: ^0.24.3 specifier: ^0.24.3
version: 0.24.3(encoding@0.1.13) version: 0.24.3(encoding@0.1.13)
@ -60,8 +60,8 @@ importers:
specifier: ^8.5.12 specifier: ^8.5.12
version: 8.5.12 version: 8.5.12
ai: ai:
specifier: ^4.1.45 specifier: ^4.3.4
version: 4.1.45(react@18.3.1)(zod@3.24.2) version: 4.3.4(react@18.3.1)(zod@3.24.2)
ajv: ajv:
specifier: ^8.16.0 specifier: ^8.16.0
version: 8.16.0 version: 8.16.0
@ -324,8 +324,8 @@ importers:
packages: packages:
'@ai-sdk/openai@1.1.13': '@ai-sdk/openai@1.3.10':
resolution: {integrity: sha512-IdChK1pJTW3NQis02PG/hHTG0gZSyQIMOLPt7f7ES56C0xH2yaKOU1Tp2aib7pZzWGwDlzTOW2h5TtAB8+V6CQ==} resolution: {integrity: sha512-XO0wF2lmAMWCYjkM5bLpWTKoXet61fBiIimTi+blqEGiLUjAvivt/1zZL1Lzhrv9+p19IC1rn9EWZI1dCelV8w==}
engines: {node: '>=18'} engines: {node: '>=18'}
peerDependencies: peerDependencies:
zod: ^3.0.0 zod: ^3.0.0
@ -339,30 +339,35 @@ packages:
zod: zod:
optional: true optional: true
'@ai-sdk/provider-utils@2.2.6':
resolution: {integrity: sha512-sUlZ7Gnq84DCGWMQRIK8XVbkzIBnvPR1diV4v6JwPgpn5armnLI/j+rqn62MpLrU5ZCQZlDKl/Lw6ed3ulYqaA==}
engines: {node: '>=18'}
peerDependencies:
zod: ^3.23.8
'@ai-sdk/provider@1.0.8': '@ai-sdk/provider@1.0.8':
resolution: {integrity: sha512-f9jSYwKMdXvm44Dmab1vUBnfCDSFfI5rOtvV1W9oKB7WYHR5dGvCC6x68Mk3NUfrdmNoMVHGoh6JT9HCVMlMow==} resolution: {integrity: sha512-f9jSYwKMdXvm44Dmab1vUBnfCDSFfI5rOtvV1W9oKB7WYHR5dGvCC6x68Mk3NUfrdmNoMVHGoh6JT9HCVMlMow==}
engines: {node: '>=18'} engines: {node: '>=18'}
'@ai-sdk/react@1.1.17': '@ai-sdk/provider@1.1.2':
resolution: {integrity: sha512-NAuEflFvjw1uh1AOmpyi7rBF4xasWsiWUb86JQ8ScjDGxoGDYEdBnaHOxUpooLna0dGNbSPkvDMnVRhoLKoxPQ==} resolution: {integrity: sha512-ITdgNilJZwLKR7X5TnUr1BsQW6UTX5yFp0h66Nfx8XjBYkWD9W3yugr50GOz3CnE9m/U/Cd5OyEbTMI0rgi6ZQ==}
engines: {node: '>=18'}
'@ai-sdk/react@1.2.8':
resolution: {integrity: sha512-S2FzCSi4uTF0JuSN6zYMXyiAWVAzi/Hho8ISYgHpGZiICYLNCP2si4DuXQOsnWef3IXzQPLVoE11C63lILZIkw==}
engines: {node: '>=18'} engines: {node: '>=18'}
peerDependencies: peerDependencies:
react: ^18 || ^19 || ^19.0.0-rc react: ^18 || ^19 || ^19.0.0-rc
zod: ^3.0.0 zod: ^3.23.8
peerDependenciesMeta: peerDependenciesMeta:
react:
optional: true
zod: zod:
optional: true optional: true
'@ai-sdk/ui-utils@1.1.15': '@ai-sdk/ui-utils@1.2.7':
resolution: {integrity: sha512-NsV/3CMmjc4m53snzRdtZM6teTQUXIKi8u0Kf7GBruSzaMSuZ4DWaAAlUshhR3p2FpZgtsogW+vYG1/rXsGu+Q==} resolution: {integrity: sha512-OVRxa4SDj0wVsMZ8tGr/whT89oqNtNoXBKmqWC2BRv5ZG6azL2LYZ5ZK35u3lb4l1IE7cWGsLlmq0py0ttsL7A==}
engines: {node: '>=18'} engines: {node: '>=18'}
peerDependencies: peerDependencies:
zod: ^3.0.0 zod: ^3.23.8
peerDependenciesMeta:
zod:
optional: true
'@ampproject/remapping@2.3.0': '@ampproject/remapping@2.3.0':
resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==}
@ -1809,17 +1814,15 @@ packages:
resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==}
engines: {node: '>= 8.0.0'} engines: {node: '>= 8.0.0'}
ai@4.1.45: ai@4.3.4:
resolution: {integrity: sha512-nQkxQ2zCD+O/h8zJ+PxmBv9coyMaG1uP9kGJvhNaGAA25hbZRQWL0NbTsSJ/QMOUraXKLa+6fBm3VF1NkJK9Kg==} resolution: {integrity: sha512-uMjzrowIqfU8CCCxhx8QGl7ETydHBROeNL0VoEwetkmDCY6Q8ZTacj6jNNqGJOiCk595aUrGR9VHPY9Ylvy1fg==}
engines: {node: '>=18'} engines: {node: '>=18'}
peerDependencies: peerDependencies:
react: ^18 || ^19 || ^19.0.0-rc react: ^18 || ^19 || ^19.0.0-rc
zod: ^3.0.0 zod: ^3.23.8
peerDependenciesMeta: peerDependenciesMeta:
react: react:
optional: true optional: true
zod:
optional: true
ajv@8.16.0: ajv@8.16.0:
resolution: {integrity: sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==} resolution: {integrity: sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==}
@ -4749,10 +4752,10 @@ packages:
snapshots: snapshots:
'@ai-sdk/openai@1.1.13(zod@3.24.2)': '@ai-sdk/openai@1.3.10(zod@3.24.2)':
dependencies: dependencies:
'@ai-sdk/provider': 1.0.8 '@ai-sdk/provider': 1.1.2
'@ai-sdk/provider-utils': 2.1.9(zod@3.24.2) '@ai-sdk/provider-utils': 2.2.6(zod@3.24.2)
zod: 3.24.2 zod: 3.24.2
'@ai-sdk/provider-utils@2.1.9(zod@3.24.2)': '@ai-sdk/provider-utils@2.1.9(zod@3.24.2)':
@ -4764,27 +4767,37 @@ snapshots:
optionalDependencies: optionalDependencies:
zod: 3.24.2 zod: 3.24.2
'@ai-sdk/provider-utils@2.2.6(zod@3.24.2)':
dependencies:
'@ai-sdk/provider': 1.1.2
nanoid: 3.3.8
secure-json-parse: 2.7.0
zod: 3.24.2
'@ai-sdk/provider@1.0.8': '@ai-sdk/provider@1.0.8':
dependencies: dependencies:
json-schema: 0.4.0 json-schema: 0.4.0
'@ai-sdk/react@1.1.17(react@18.3.1)(zod@3.24.2)': '@ai-sdk/provider@1.1.2':
dependencies: dependencies:
'@ai-sdk/provider-utils': 2.1.9(zod@3.24.2) json-schema: 0.4.0
'@ai-sdk/ui-utils': 1.1.15(zod@3.24.2)
'@ai-sdk/react@1.2.8(react@18.3.1)(zod@3.24.2)':
dependencies:
'@ai-sdk/provider-utils': 2.2.6(zod@3.24.2)
'@ai-sdk/ui-utils': 1.2.7(zod@3.24.2)
react: 18.3.1
swr: 2.3.2(react@18.3.1) swr: 2.3.2(react@18.3.1)
throttleit: 2.1.0 throttleit: 2.1.0
optionalDependencies: optionalDependencies:
react: 18.3.1
zod: 3.24.2 zod: 3.24.2
'@ai-sdk/ui-utils@1.1.15(zod@3.24.2)': '@ai-sdk/ui-utils@1.2.7(zod@3.24.2)':
dependencies: dependencies:
'@ai-sdk/provider': 1.0.8 '@ai-sdk/provider': 1.1.2
'@ai-sdk/provider-utils': 2.1.9(zod@3.24.2) '@ai-sdk/provider-utils': 2.2.6(zod@3.24.2)
zod-to-json-schema: 3.24.2(zod@3.24.2)
optionalDependencies:
zod: 3.24.2 zod: 3.24.2
zod-to-json-schema: 3.24.2(zod@3.24.2)
'@ampproject/remapping@2.3.0': '@ampproject/remapping@2.3.0':
dependencies: dependencies:
@ -6978,17 +6991,17 @@ snapshots:
dependencies: dependencies:
humanize-ms: 1.2.1 humanize-ms: 1.2.1
ai@4.1.45(react@18.3.1)(zod@3.24.2): ai@4.3.4(react@18.3.1)(zod@3.24.2):
dependencies: dependencies:
'@ai-sdk/provider': 1.0.8 '@ai-sdk/provider': 1.1.2
'@ai-sdk/provider-utils': 2.1.9(zod@3.24.2) '@ai-sdk/provider-utils': 2.2.6(zod@3.24.2)
'@ai-sdk/react': 1.1.17(react@18.3.1)(zod@3.24.2) '@ai-sdk/react': 1.2.8(react@18.3.1)(zod@3.24.2)
'@ai-sdk/ui-utils': 1.1.15(zod@3.24.2) '@ai-sdk/ui-utils': 1.2.7(zod@3.24.2)
'@opentelemetry/api': 1.9.0 '@opentelemetry/api': 1.9.0
jsondiffpatch: 0.6.0 jsondiffpatch: 0.6.0
zod: 3.24.2
optionalDependencies: optionalDependencies:
react: 18.3.1 react: 18.3.1
zod: 3.24.2
ajv@8.16.0: ajv@8.16.0:
dependencies: dependencies:

12
apps/api/src/lib/deep-research/research-manager.ts
View File

@ -7,6 +7,8 @@ import {
} from "./deep-research-redis"; } from "./deep-research-redis";
import { generateCompletions, trimToTokenLimit } from "../../scraper/scrapeURL/transformers/llmExtract"; import { generateCompletions, trimToTokenLimit } from "../../scraper/scrapeURL/transformers/llmExtract";
import { ExtractOptions } from "../../controllers/v1/types"; import { ExtractOptions } from "../../controllers/v1/types";
import { openai } from "@ai-sdk/openai/dist";
import { getModel } from "../generic-ai";
interface AnalysisResult { interface AnalysisResult {
gaps: string[]; gaps: string[];
nextSteps: string[]; nextSteps: string[];
@ -277,7 +279,7 @@ export class ResearchLLMService {
}), }),
systemPrompt: formats.includes('json') systemPrompt: formats.includes('json')
? "You are an expert research analyst who creates comprehensive, structured analysis following the provided JSON schema exactly." ? "You are an expert research analyst who creates comprehensive, structured analysis following the provided JSON schema exactly."
: "You are an expert research analyst who creates comprehensive, well-structured reports. Your reports are detailed, properly formatted in Markdown, and include clear sections with citations. Today's date is " + : "You are an expert research analyst who creates comprehensive, well-structured reports. Don't begin the report by saying 'Here is the report', nor 'Below is the report', nor something similar. ALWAYS start with a great title that reflects the research topic and findings. Your reports are detailed, properly formatted in Markdown, and include clear sections with citations. Today's date is " +
new Date().toISOString().split("T")[0], new Date().toISOString().split("T")[0],
prompt: trimToTokenLimit( prompt: trimToTokenLimit(
analysisPrompt analysisPrompt
@ -296,11 +298,17 @@ export class ResearchLLMService {
- Make it comprehensive and thorough (aim for 4+ pages worth of content) - Make it comprehensive and thorough (aim for 4+ pages worth of content)
- Include all relevant findings and insights from the research - Include all relevant findings and insights from the research
- Cite sources - Cite sources
- Use bullet points and lists where appropriate for readability`, - Cite sources throughout the report
- Use bullet points and lists where appropriate for readability
- Don't begin the report by saying "Here is the report", nor "Below is the report", nor something similar.
- ALWAYS Start with a great title that reflects the research topic and findings - concise and to the point. That's the first thing you should output.
Begin!`,
100000, 100000,
).text, ).text,
}, },
markdown: "", markdown: "",
model: getModel('o3-mini'),
}); });
return extract; return extract;

2
apps/api/src/services/billing/batch_billing.ts
View File

@ -122,7 +122,7 @@ export async function processBillingBatch() {
group.is_extract group.is_extract
); );
logger.info(`✅ Successfully billed team ${group.team_id} for ${group.total_credits} credits`); logger.info(`✅ Successfully billed team ${group.team_id} for ${group.total_credits} ${group.is_extract ? 'tokens' : 'credits'}`);
} catch (error) { } catch (error) {
logger.error(`❌ Failed to bill team ${group.team_id}`, { error, group }); logger.error(`❌ Failed to bill team ${group.team_id}`, { error, group });
Sentry.captureException(error, { Sentry.captureException(error, {