AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 20:19:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extract): run on original links if reranker is weird

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-15 22:57:20 -07:00
parent 13bd50ad2b
commit 512a2b1cd4
1 changed files with 6 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
apps/api/src/lib/extract/extraction-service.ts
View File

@ -273,28 +273,21 @@ export async function performExtraction(
);
const processedUrls = await Promise.all(urlPromises);
const links = processedUrls.flat().filter((url) => url);
let links = processedUrls.flat().filter((url) => url);
logger.debug("Processed URLs.", {
linkCount: links.length,
});
log["links"] = links;
log["linksLength"] = links.length;
if (links.length === 0) {
logger.error("0 links! Bailing.", {
links = urls.map(x => x.replace(/\*$/g, ""));
logger.warn("0 links! Doing just the original URLs. (without * wildcard)", {
linkCount: links.length,
});
return {
success: false,
error:
"No valid URLs found to scrape. Try adjusting your search criteria or including more URLs.",
extractId,
urlTrace: urlTraces,
totalUrlsScraped: 0,
};
}
log["links"] = links;
log["linksLength"] = links.length;
await updateExtract(extractId, {
status: "processing",
steps: [