AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 16:38:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extractSmartScrape): better pagination handling

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-16 16:23:12 -07:00
parent 509e6e658c
commit 751c30f139
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
View File

@ -16,7 +16,7 @@ const commonSmartScrapeProperties = {
shouldUseSmartscrape: { shouldUseSmartscrape: {
type: "boolean", type: "boolean",
description: description:
"Set to `true` if any of the extractedData is null and you think you can find the information by performing user-like interactions (e.g., clicking buttons/accordions to reveal hidden text, login, inputs etc.). SmartScrape can perform these actions to access the data.", "Set to `true` if any of the extractedData is null and you think you can find the information by performing user-like interactions (e.g., clicking buttons/accordions to reveal hidden text, login, inputs, pagination etc.). SmartScrape can perform these actions to access the data.",
}, },
// Note: extractedData is added dynamically in prepareSmartScrapeSchema // Note: extractedData is added dynamically in prepareSmartScrapeSchema
}; };

2
apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
View File

@ -386,7 +386,7 @@ export async function generateCompletions({
const repairConfig = { const repairConfig = {
experimental_repairText: async ({ text, error }) => { experimental_repairText: async ({ text, error }) => {
// AI may output a markdown JSON code block. Remove it - mogery // AI may output a markdown JSON code block. Remove it - mogery
logger.debug("Repairing text", { textType: typeof text, error }); logger.debug("Repairing text", { textType: typeof text, textPeek: JSON.stringify(text).slice(0, 100) + "...", error });
if (typeof text === "string" && text.trim().startsWith("```")) { if (typeof text === "string" && text.trim().startsWith("```")) {
if (text.trim().startsWith("```json")) { if (text.trim().startsWith("```json")) {