AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 09:49:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

layers

Browse Source
This commit is contained in:
rafaelmmiller 2025-01-24 10:25:45 -03:00
parent 64d116540f
commit 3184e91f66
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/lib/extract/url-processor.ts
View File

@ -204,19 +204,28 @@ export async function processUrl(
}); });
let rerankedLinks = mappedLinks; let rerankedLinks = mappedLinks;
logger.info("Reranking pass 1 (threshold 0.6)..."); logger.info("Reranking pass 1 (threshold 0.8)...");
const rerankerResult = await rerankLinksWithLLM({ const rerankerResult = await rerankLinksWithLLM({
links: rerankedLinks, links: rerankedLinks,
searchQuery: rephrasedPrompt, searchQuery: rephrasedPrompt,
urlTraces urlTraces
}); });
rerankedLinks = rerankerResult.mapDocument.filter((x) => x.relevanceScore && x.relevanceScore > 0.6); rerankedLinks = rerankerResult.mapDocument.filter((x) => x.relevanceScore && x.relevanceScore > 0.8);
let tokensUsed = rerankerResult.tokensUsed; let tokensUsed = rerankerResult.tokensUsed;
logger.info("Reranked! (threshold 0.6)", { logger.info("Reranked! (threshold 0.8)", {
linkCount: rerankedLinks.length, linkCount: rerankedLinks.length,
}); });
// lower threshold to 0.6 if no links are found
if (rerankedLinks.length === 0) {
logger.info("No links found. Reranking with threshold 0.6");
rerankedLinks = rerankerResult.mapDocument.filter((x) => x.relevanceScore && x.relevanceScore > 0.6);
logger.info("Reranked! (threshold 0.6)", {
linkCount: rerankedLinks.length,
});
}
// lower threshold to 0.3 if no links are found // lower threshold to 0.3 if no links are found
if (rerankedLinks.length === 0) { if (rerankedLinks.length === 0) {
logger.info("No links found. Reranking with threshold 0.3"); logger.info("No links found. Reranking with threshold 0.3");
@ -235,8 +244,9 @@ export async function processUrl(
urlTraces, urlTraces,
}); });
// why 0.6? average? experimental results?
if (secondPassRerankerResult.mapDocument.length > 0) { if (secondPassRerankerResult.mapDocument.length > 0) {
rerankedLinks = secondPassRerankerResult.mapDocument; rerankedLinks = secondPassRerankerResult.mapDocument.filter((x) => x.relevanceScore && x.relevanceScore > 0.6);
logger.info("Reranked! (threshold 0.6)", { logger.info("Reranked! (threshold 0.6)", {
linkCount: rerankedLinks.length, linkCount: rerankedLinks.length,
}); });