diff --git a/.gitignore b/.gitignore index a19465c1..37baa733 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,5 @@ apps/js-sdk/firecrawl/dist .vscode llm-links.txt -mapped-links.txt \ No newline at end of file +mapped-links.txt +gke-key.json \ No newline at end of file diff --git a/apps/api/package.json b/apps/api/package.json index 528e39f8..fbcdee8f 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -55,6 +55,7 @@ "@ai-sdk/deepinfra": "^0.2.4", "@ai-sdk/fireworks": "^0.2.4", "@ai-sdk/google": "^1.2.3", + "@ai-sdk/google-vertex": "^2.2.13", "@ai-sdk/groq": "^1.2.1", "@ai-sdk/openai": "^1.1.13", "@anthropic-ai/sdk": "^0.24.3", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 3c980877..d85ce22a 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@ai-sdk/google': specifier: ^1.2.3 version: 1.2.3(zod@3.24.2) + '@ai-sdk/google-vertex': + specifier: ^2.2.13 + version: 2.2.13(encoding@0.1.13)(zod@3.24.2) '@ai-sdk/groq': specifier: ^1.2.1 version: 1.2.1(zod@3.24.2) @@ -348,6 +351,12 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/anthropic@1.2.8': + resolution: {integrity: sha512-UchR+xGtTASkHyJ5uX4jVJGfxo2SCoqxtHJQzuO8INYEbpFwa9cUD06934tNqICxsNOhupS9y4b1sd/Uep6OXw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/deepinfra@0.2.4': resolution: {integrity: sha512-JBF3tUOLYgQDCwkvN9I5ZbSqsAxTJWOKmIpyJXJl5RpLXOEviJUqpKSZufs11J9S4Z0U9vZX9jfhO1+DBjS56w==} engines: {node: '>=18'} @@ -360,6 +369,18 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/google-vertex@2.2.13': + resolution: {integrity: sha512-ddtrFYQnPRwr5/Cf9tIJ7Q/kAlYOtxA1xk3ZFnnfEjOArHAb+3KcLeanBSGpd+xuPm9nwK0DgYlm3lElGNjRIg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + + '@ai-sdk/google@1.2.10': + resolution: {integrity: sha512-YmZ9DIO6Un0+RU9PtjM9TfoExmUQg2fk8vTlwT+NOaARyhv8eskRCUTne0zf5uUOazPIJuBEv2I6YE9XnS+tUg==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/google@1.2.3': resolution: {integrity: sha512-zsgwko7T+MFIdEfhg4fIXv6O2dnzTLFr6BOpAA21eo/moOBA5szVzOto1jTwIwoBYsF2ixPGNZBoc+k/fQ2AWw==} engines: {node: '>=18'} @@ -414,6 +435,12 @@ packages: peerDependencies: zod: ^3.23.8 + '@ai-sdk/provider-utils@2.2.6': + resolution: {integrity: sha512-sUlZ7Gnq84DCGWMQRIK8XVbkzIBnvPR1diV4v6JwPgpn5armnLI/j+rqn62MpLrU5ZCQZlDKl/Lw6ed3ulYqaA==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + '@ai-sdk/provider@1.0.8': resolution: {integrity: sha512-f9jSYwKMdXvm44Dmab1vUBnfCDSFfI5rOtvV1W9oKB7WYHR5dGvCC6x68Mk3NUfrdmNoMVHGoh6JT9HCVMlMow==} engines: {node: '>=18'} @@ -426,6 +453,10 @@ packages: resolution: {integrity: sha512-0M+qjp+clUD0R1E5eWQFhxEvWLNaOtGQRUaBn8CUABnSKredagq92hUS9VjOzGsTm37xLfpaxl97AVtbeOsHew==} engines: {node: '>=18'} + '@ai-sdk/provider@1.1.2': + resolution: {integrity: sha512-ITdgNilJZwLKR7X5TnUr1BsQW6UTX5yFp0h66Nfx8XjBYkWD9W3yugr50GOz3CnE9m/U/Cd5OyEbTMI0rgi6ZQ==} + engines: {node: '>=18'} + '@ai-sdk/react@1.2.5': resolution: {integrity: sha512-0jOop3S2WkDOdO4X5I+5fTGqZlNX8/h1T1eYokpkR9xh8Vmrxqw8SsovqGvrddTsZykH8uXRsvI+G4FTyy894A==} engines: {node: '>=18'} @@ -2033,6 +2064,9 @@ packages: resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==} engines: {node: '>=10.0.0'} + bignumber.js@9.2.0: + resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==} + bin-links@4.0.4: resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -2089,6 +2123,9 @@ packages: buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -2477,6 +2514,9 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + editorconfig@1.0.4: resolution: {integrity: sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==} engines: {node: '>=14'} @@ -2615,6 +2655,9 @@ packages: resolution: {integrity: sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==} engines: {node: '>= 0.10.0'} + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + extract-zip@2.0.1: resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} engines: {node: '>= 10.17.0'} @@ -2740,6 +2783,14 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + gaxios@6.7.1: + resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} + engines: {node: '>=14'} + + gcp-metadata@6.1.1: + resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} + engines: {node: '>=14'} + generic-pool@3.9.0: resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} engines: {node: '>= 4'} @@ -2789,6 +2840,14 @@ packages: resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==} engines: {node: '>=4'} + google-auth-library@9.15.1: + resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} + engines: {node: '>=14'} + + google-logging-utils@0.0.2: + resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} + engines: {node: '>=14'} + gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} @@ -2799,6 +2858,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + gtoken@7.1.0: + resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} + engines: {node: '>=14.0.0'} + handlebars@4.7.8: resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} engines: {node: '>=0.4.7'} @@ -3229,6 +3292,9 @@ packages: engines: {node: '>=4'} hasBin: true + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-parse-even-better-errors@2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} @@ -3262,6 +3328,12 @@ packages: jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + jwa@2.0.0: + resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==} + + jws@4.0.0: + resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + kareem@2.6.3: resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==} engines: {node: '>=12.0.0'} @@ -4666,8 +4738,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.8.2: - resolution: {integrity: sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==} + typescript@5.8.3: + resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} engines: {node: '>=14.17'} hasBin: true @@ -4966,6 +5038,12 @@ snapshots: '@ai-sdk/provider-utils': 2.2.3(zod@3.24.2) zod: 3.24.2 + '@ai-sdk/anthropic@1.2.8(zod@3.24.2)': + dependencies: + '@ai-sdk/provider': 1.1.2 + '@ai-sdk/provider-utils': 2.2.6(zod@3.24.2) + zod: 3.24.2 + '@ai-sdk/deepinfra@0.2.4(zod@3.24.2)': dependencies: '@ai-sdk/openai-compatible': 0.2.4(zod@3.24.2) @@ -4980,6 +5058,24 @@ snapshots: '@ai-sdk/provider-utils': 2.2.3(zod@3.24.2) zod: 3.24.2 + '@ai-sdk/google-vertex@2.2.13(encoding@0.1.13)(zod@3.24.2)': + dependencies: + '@ai-sdk/anthropic': 1.2.8(zod@3.24.2) + '@ai-sdk/google': 1.2.10(zod@3.24.2) + '@ai-sdk/provider': 1.1.2 + '@ai-sdk/provider-utils': 2.2.6(zod@3.24.2) + google-auth-library: 9.15.1(encoding@0.1.13) + zod: 3.24.2 + transitivePeerDependencies: + - encoding + - supports-color + + '@ai-sdk/google@1.2.10(zod@3.24.2)': + dependencies: + '@ai-sdk/provider': 1.1.2 + '@ai-sdk/provider-utils': 2.2.6(zod@3.24.2) + zod: 3.24.2 + '@ai-sdk/google@1.2.3(zod@3.24.2)': dependencies: '@ai-sdk/provider': 1.1.0 @@ -5036,6 +5132,13 @@ snapshots: secure-json-parse: 2.7.0 zod: 3.24.2 + '@ai-sdk/provider-utils@2.2.6(zod@3.24.2)': + dependencies: + '@ai-sdk/provider': 1.1.2 + nanoid: 3.3.8 + secure-json-parse: 2.7.0 + zod: 3.24.2 + '@ai-sdk/provider@1.0.8': dependencies: json-schema: 0.4.0 @@ -5048,6 +5151,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@1.1.2': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/react@1.2.5(react@18.3.1)(zod@3.24.2)': dependencies: '@ai-sdk/provider-utils': 2.2.3(zod@3.24.2) @@ -7448,6 +7555,8 @@ snapshots: basic-ftp@5.0.5: {} + bignumber.js@9.2.0: {} + bin-links@4.0.4: dependencies: cmd-shim: 6.0.3 @@ -7516,6 +7625,8 @@ snapshots: buffer-crc32@0.2.13: {} + buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} buffer@5.7.1: @@ -7899,6 +8010,10 @@ snapshots: eastasianwidth@0.2.0: {} + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + editorconfig@1.0.4: dependencies: '@one-ini/wasm': 0.1.1 @@ -8048,6 +8163,8 @@ snapshots: transitivePeerDependencies: - supports-color + extend@3.0.2: {} + extract-zip@2.0.1: dependencies: debug: 4.3.5 @@ -8174,6 +8291,26 @@ snapshots: function-bind@1.1.2: {} + gaxios@6.7.1(encoding@0.1.13): + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + is-stream: 2.0.1 + node-fetch: 2.7.0(encoding@0.1.13) + uuid: 9.0.1 + transitivePeerDependencies: + - encoding + - supports-color + + gcp-metadata@6.1.1(encoding@0.1.13): + dependencies: + gaxios: 6.7.1(encoding@0.1.13) + google-logging-utils: 0.0.2 + json-bigint: 1.0.0 + transitivePeerDependencies: + - encoding + - supports-color + generic-pool@3.9.0: {} gensync@1.0.0-beta.2: {} @@ -8229,6 +8366,20 @@ snapshots: globals@11.12.0: {} + google-auth-library@9.15.1(encoding@0.1.13): + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 6.7.1(encoding@0.1.13) + gcp-metadata: 6.1.1(encoding@0.1.13) + gtoken: 7.1.0(encoding@0.1.13) + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + + google-logging-utils@0.0.2: {} + gopd@1.0.1: dependencies: get-intrinsic: 1.2.4 @@ -8239,6 +8390,14 @@ snapshots: graceful-fs@4.2.11: {} + gtoken@7.1.0(encoding@0.1.13): + dependencies: + gaxios: 6.7.1(encoding@0.1.13) + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + handlebars@4.7.8: dependencies: minimist: 1.2.8 @@ -8921,6 +9080,10 @@ snapshots: jsesc@2.5.2: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.2.0 + json-parse-even-better-errors@2.3.1: {} json-schema-to-zod@2.3.0: {} @@ -8952,6 +9115,17 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 + jwa@2.0.0: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.0: + dependencies: + jwa: 2.0.0 + safe-buffer: 5.2.1 + kareem@2.6.3: {} keyword-extractor@0.0.28: {} @@ -9663,7 +9837,7 @@ snapshots: csv-parse: 5.5.6 gpt3-tokenizer: 1.1.5 openai: 3.3.0 - typescript: 5.8.2 + typescript: 5.8.3 uuid: 9.0.1 zod: 3.24.2 transitivePeerDependencies: @@ -10277,7 +10451,7 @@ snapshots: typescript@5.4.5: {} - typescript@5.8.2: {} + typescript@5.8.3: {} typesense@1.8.2(@babel/runtime@7.24.6): dependencies: diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts index 177fa32c..8d5bc965 100644 --- a/apps/api/src/lib/extract/reranker.ts +++ b/apps/api/src/lib/extract/reranker.ts @@ -314,42 +314,7 @@ export async function rerankLinksWithLLM( }); completion = await completionPromise; - // completion = await Promise.race([ - // completionPromise, - // timeoutPromise, - // ]); - - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!˜", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log({ completion }); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); + } catch (error) { console.warn( `Error processing chunk ${chunkIndex + 1} attempt ${retry + 1}:`, diff --git a/apps/api/src/lib/generic-ai.ts b/apps/api/src/lib/generic-ai.ts index c585c43a..5ee35db9 100644 --- a/apps/api/src/lib/generic-ai.ts +++ b/apps/api/src/lib/generic-ai.ts @@ -6,6 +6,8 @@ import { google } from "@ai-sdk/google"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { fireworks } from "@ai-sdk/fireworks"; import { deepinfra } from "@ai-sdk/deepinfra"; +import { createVertex } from "@ai-sdk/google-vertex"; + type Provider = | "openai" | "ollama" @@ -14,7 +16,8 @@ type Provider = | "google" | "openrouter" | "fireworks" - | "deepinfra"; + | "deepinfra" + | "vertex"; const defaultProvider: Provider = process.env.OLLAMA_BASE_URL ? "ollama" : "openai"; @@ -32,6 +35,13 @@ const providerList: Record = { }), fireworks, //FIREWORKS_API_KEY deepinfra, //DEEPINFRA_API_KEY + vertex: createVertex({ + project: "firecrawl", + location: "us-central1", + googleAuthOptions: { + keyFile: "./gke-key.json", + }, + }), }; export function getModel(name: string, provider: Provider = defaultProvider) { diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts index 1ca50d83..45c4d47b 100644 --- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts +++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts @@ -7,6 +7,7 @@ import { import { smartScrape } from "./smartScrape"; import { parseMarkdown } from "../../../lib/html-to-markdown"; import { getModel } from "../../../lib/generic-ai"; +import { TokenUsage } from "../../../controllers/v1/types"; const commonSmartScrapeProperties = { shouldUseSmartscrape: { @@ -184,6 +185,9 @@ export async function extractData({ extractOptions: GenerateCompletionsOptions; urls: string[]; }): Promise<{ extractedDataArray: any[]; warning: any }> { + // TODO: receive from user + const useSmartScrape = true; + //WRAP SCHEMA const schema = extractOptions.options.schema; const logger = extractOptions.logger; @@ -197,26 +201,23 @@ export async function extractData({ console.log("schema", schema); console.log("schemaToUse", schemaToUse); - let extract, warning, totalUsage; + let extract: any, + warning: string | undefined, + totalUsage: TokenUsage | undefined; + // checks if using smartScrape is needed for this case try { const { extract: e, warning: w, totalUsage: t } = await generateCompletions( - { ...extractOptionsNewSchema, model: getModel("gemini-2.5-pro-exp-03-25", "google") } - ); + { ...extractOptionsNewSchema, + model: getModel("gemini-2.5-pro-preview-03-25", "vertex"), + retryModel: getModel("o3-mini", "openai"), + }); extract = e; warning = w; totalUsage = t; } catch (error) { console.log("failed during extractSmartScrape.ts:generateCompletions", error); } - console.log("extract", extract); - - // const { - // extractedData, - // shouldUseSmartscrape, - // smartscrape_reasoning, - // smartscrape_prompt, - // } = processSmartScrapeResult(extract, logger); let extractedData = extract?.extractedData; @@ -224,7 +225,7 @@ export async function extractData({ console.log("smartscrape_reasoning", extract?.smartscrape_reasoning); console.log("smartscrape_prompt", extract?.smartscrape_prompt); try { - if (extract?.shouldUseSmartscrape) { + if (useSmartScrape && extract?.shouldUseSmartscrape) { let smartscrapeResults; if (isSingleUrl) { smartscrapeResults = [ diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts index b6c493d8..98e1cc26 100644 --- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts +++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts @@ -55,14 +55,16 @@ export async function smartScrape( prompt, models: { thinkingModel: { - model: "gemini-2.5-pro-exp-03-25", - provider: "google", + model: "gemini-2.5-pro-preview-03-25", + provider: "vertex", supportTools: true, toolChoice: "required", cost: { input: 1.3, output: 5, }, + retryModel: "qwen-qwq-32b", + retryProvider: "groq" }, toolModel: { model: "gemini-2.0-flash", diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts index d12817c0..f358bdc5 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts @@ -9,7 +9,7 @@ import { Logger } from "winston"; import { EngineResultsTracker, Meta } from ".."; import { logger } from "../../../lib/logger"; import { modelPrices } from "../../../lib/extract/usage/model-prices"; -import { generateObject, generateText, LanguageModel } from "ai"; +import { generateObject, generateText, LanguageModel, NoObjectGeneratedError } from "ai"; import { jsonSchema } from "ai"; import { getModel } from "../../../lib/generic-ai"; import { z } from "zod"; @@ -178,6 +178,7 @@ export type GenerateCompletionsOptions = { isExtractEndpoint?: boolean; mode?: "object" | "no-object"; providerOptions?: LanguageModelV1ProviderMetadata; + retryModel?: LanguageModel; }; export async function generateCompletions({ logger, @@ -188,6 +189,7 @@ export async function generateCompletions({ model = getModel("gpt-4o-mini"), mode = "object", providerOptions, + retryModel = getModel("claude-3-7-sonnet-latest", "anthropic"), }: GenerateCompletionsOptions): Promise<{ extract: any; numTokens: number; @@ -197,12 +199,14 @@ export async function generateCompletions({ }> { let extract: any; let warning: string | undefined; + let currentModel = model; + let lastError: Error | null = null; if (markdown === undefined) { throw new Error("document.markdown is undefined -- this is unexpected"); } - const { maxInputTokens, maxOutputTokens } = getModelLimits(model.modelId); + const { maxInputTokens, maxOutputTokens } = getModelLimits(currentModel.modelId); // Calculate 80% of max input tokens (for content) const maxTokensSafe = Math.floor(maxInputTokens * 0.8); @@ -224,31 +228,72 @@ export async function generateCompletions({ : `Transform the following content into structured JSON output based on the provided schema if any.\n\n${markdown}`; if (mode === "no-object") { - const result = await generateText({ - model: model, - prompt: options.prompt + (markdown ? `\n\nData:${markdown}` : ""), - // temperature: options.temperature ?? 0, - system: options.systemPrompt, - providerOptions: { - anthropic: { - thinking: { type: "enabled", budgetTokens: 12000 }, + try { + const result = await generateText({ + model: currentModel, + prompt: options.prompt + (markdown ? `\n\nData:${markdown}` : ""), + system: options.systemPrompt, + providerOptions: { + anthropic: { + thinking: { type: "enabled", budgetTokens: 12000 }, + }, }, - }, - }); + }); - extract = result.text; + extract = result.text; - return { - extract, - warning, - numTokens, - totalUsage: { - promptTokens: numTokens, - completionTokens: result.usage?.completionTokens ?? 0, - totalTokens: numTokens + (result.usage?.completionTokens ?? 0), - }, - model: model.modelId, - }; + return { + extract, + warning, + numTokens, + totalUsage: { + promptTokens: numTokens, + completionTokens: result.usage?.completionTokens ?? 0, + totalTokens: numTokens + (result.usage?.completionTokens ?? 0), + }, + model: currentModel.modelId, + }; + } catch (error) { + lastError = error as Error; + if (error.message?.includes("Quota exceeded") || error.message?.includes("rate limit")) { + logger.warn("Quota exceeded, retrying with fallback model", { error: lastError.message }); + currentModel = retryModel; + try { + const result = await generateText({ + model: currentModel, + prompt: options.prompt + (markdown ? `\n\nData:${markdown}` : ""), + system: options.systemPrompt, + providerOptions: { + anthropic: { + thinking: { type: "enabled", budgetTokens: 12000 }, + }, + }, + }); + + extract = result.text; + + return { + extract, + warning, + numTokens, + totalUsage: { + promptTokens: numTokens, + completionTokens: result.usage?.completionTokens ?? 0, + totalTokens: numTokens + (result.usage?.completionTokens ?? 0), + }, + model: currentModel.modelId, + }; + } catch (retryError) { + lastError = retryError as Error; + logger.error("Failed with fallback model", { + originalError: lastError.message, + model: currentModel.modelId + }); + throw lastError; + } + } + throw lastError; + } } let schema = options.schema; @@ -305,26 +350,31 @@ export async function generateCompletions({ } catch (_) {} } - const { text: fixedText } = await generateText({ - model: model, - prompt: `Fix this JSON that had the following error: ${error}\n\nOriginal text:\n${text}\n\nReturn only the fixed JSON, no explanation.`, - system: - "You are a JSON repair expert. Your only job is to fix malformed JSON and return valid JSON that matches the original structure and intent as closely as possible. Do not include any explanation or commentary - only return the fixed JSON. Do not return it in a Markdown code block, just plain JSON.", - providerOptions: { - anthropic: { - thinking: { type: "enabled", budgetTokens: 12000 }, + try { + const { text: fixedText } = await generateText({ + model: currentModel, + prompt: `Fix this JSON that had the following error: ${error}\n\nOriginal text:\n${text}\n\nReturn only the fixed JSON, no explanation.`, + system: + "You are a JSON repair expert. Your only job is to fix malformed JSON and return valid JSON that matches the original structure and intent as closely as possible. Do not include any explanation or commentary - only return the fixed JSON. Do not return it in a Markdown code block, just plain JSON.", + providerOptions: { + anthropic: { + thinking: { type: "enabled", budgetTokens: 12000 }, + }, }, - }, - }); - return fixedText; + }); + return fixedText; + } catch (repairError) { + lastError = repairError as Error; + logger.error("Failed to repair JSON", { error: lastError.message }); + throw lastError; + } }, }; const generateObjectConfig = { - model: model, + model: currentModel, prompt: prompt, providerOptions: providerOptions || undefined, - // temperature: options.temperature ?? 0, system: options.systemPrompt, ...(schema && { schema: schema instanceof z.ZodType ? schema : jsonSchema(schema), @@ -333,32 +383,67 @@ export async function generateCompletions({ ...repairConfig, ...(!schema && { onError: (error: Error) => { + lastError = error; console.error(error); }, }), } satisfies Parameters[0]; - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); const now = new Date().getTime(); - console.log(now); - console.log({ generateObjectConfig }); - await fs.writeFile( `logs/generateObjectConfig-${now}.json`, JSON.stringify(generateObjectConfig, null, 2), ); - const result = await generateObject(generateObjectConfig); - extract = result.object; - - const now2 = new Date().getTime(); - console.log(">>>>>>", now2 - now); - console.log({ extract }); - console.log( - "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", - ); + let result: { object: any, usage: TokenUsage } | undefined; + try { + result = await generateObject(generateObjectConfig); + } catch (error) { + lastError = error as Error; + if (error.message?.includes("Quota exceeded") || error.message?.includes("rate limit")) { + logger.warn("Quota exceeded, retrying with fallback model", { error: lastError.message }); + currentModel = retryModel; + try { + const retryConfig = { + ...generateObjectConfig, + model: currentModel, + }; + result = await generateObject(retryConfig); + } catch (retryError) { + lastError = retryError as Error; + logger.error("Failed with fallback model", { + originalError: lastError.message, + model: currentModel.modelId + }); + throw lastError; + } + } else if (NoObjectGeneratedError.isInstance(error)) { + console.log("No object generated", error); + if (error.text && error.text.startsWith("```json") && error?.text.endsWith("```")) { + try { + extract = JSON.parse(error.text.slice("```json".length, -"```".length).trim()); + result = { + object: extract, + usage: { + promptTokens: error.usage?.promptTokens ?? 0, + completionTokens: error.usage?.completionTokens ?? 0, + totalTokens: error.usage?.totalTokens ?? 0, + }, + }; + } catch (parseError) { + lastError = parseError as Error; + logger.error("Failed to parse JSON from error text", { error: lastError.message }); + throw lastError; + } + } else { + throw lastError; + } + } else { + throw lastError; + } + } + + extract = result?.object; // If the users actually wants the items object, they can specify it as 'required' in the schema // otherwise, we just return the items array @@ -383,13 +468,19 @@ export async function generateCompletions({ completionTokens, totalTokens: promptTokens + completionTokens, }, - model: model.modelId, + model: currentModel.modelId, }; } catch (error) { + lastError = error as Error; if (error.message?.includes("refused")) { throw new LLMRefusalError(error.message); } - throw error; + logger.error("LLM extraction failed", { + error: lastError.message, + model: currentModel.modelId, + mode + }); + throw lastError; } } @@ -412,9 +503,10 @@ export async function performLLMExtract( // ... existing model and provider options ... // model: getModel("o3-mini", "openai"), // Keeping existing model selection // model: getModel("o3-mini", "openai"), - // model: getModel("qwen-qwq-32b", "groq"), + model: getModel("qwen-qwq-32b", "groq"), // model: getModel("gemini-2.0-flash", "google"), - model: getModel("gemini-2.5-pro-exp-03-25", "google"), + // model: getModel("gemini-2.5-pro-preview-03-25", "vertex"), + retryModel: getModel("o3-mini", "openai"), }; const { extractedDataArray, warning } = await extractData({ @@ -555,7 +647,8 @@ export function removeDefaultProperty(schema: any): any { } export async function generateSchemaFromPrompt(prompt: string): Promise { - const model = getModel("gpt-4o"); + const model = getModel("qwen-qwq-32b", "groq"); + const retryModel = getModel("gpt-4o", "openai"); const temperatures = [0, 0.1, 0.3]; // Different temperatures to try let lastError: Error | null = null; @@ -565,7 +658,8 @@ export async function generateSchemaFromPrompt(prompt: string): Promise { logger: logger.child({ method: "generateSchemaFromPrompt/generateCompletions", }), - model: model, + model, + retryModel, options: { mode: "llm", systemPrompt: `You are a schema generator for a web scraping system. Generate a JSON schema based on the user's prompt.