AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 19:46:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update extract.ts

Browse Source
This commit is contained in:
Nicolas 2024-11-24 19:36:18 -08:00
parent 95bea6a391
commit b693c6c23b
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/controllers/v1/extract.ts
View File

@ -93,14 +93,20 @@ export async function extractController(
// }
if (req.body.prompt) {
// Get similarity scores between the search query and each link's context
const linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);
mappedLinks = linksAndScores
// Only keep links that have a similarity score above the threshold
.filter(x => x.score > SCORE_THRESHOLD)
// Map back to the original link objects
.map(x => mappedLinks.find(link => link.url === x.link))
// Remove any undefined links, links without URLs, and blocked URLs
.filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url))
// Limit the number of results
.slice(0, MAX_RANKING_LIMIT);
// console.log("linksAndScores", linksAndScores);
// console.log("linksAndScores", linksAndScores.length);
// TODO: handle case where no links are returned
}
return mappedLinks.map(x => x.url) as string[];