AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 17:10:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update extract.ts

Browse Source
This commit is contained in:
Nicolas 2024-11-20 16:40:55 -08:00
parent d196b9d93d
commit 945183ffbd
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/controllers/v1/extract.ts
View File

@ -87,7 +87,6 @@ export async function extractController(
let mappedLinksRerank = mappedLinks.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);
// Filter by path prefix if present
// console.log("pathPrefix", pathPrefix);
// wrong
// if (pathPrefix) {
// mappedLinks = mappedLinks.filter(x => x.url && x.url.includes(`/${pathPrefix}/`));
@ -100,8 +99,8 @@ export async function extractController(
.map(x => mappedLinks.find(link => link.url === x.link))
.filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url))
.slice(0, MAX_RANKING_LIMIT);
console.log("linksAndScores", linksAndScores);
console.log("linksAndScores", linksAndScores.length);
// console.log("linksAndScores", linksAndScores);
// console.log("linksAndScores", linksAndScores.length);
}
return mappedLinks.map(x => x.url) as string[];
@ -119,7 +118,7 @@ export async function extractController(
const processedUrls = await Promise.all(urlPromises);
links.push(...processedUrls.flat());
console.log("links", links.length);
// console.log("links", links.length);
// Scrape all links in parallel
const scrapePromises = links.map(async (url) => {
const origin = req.body.origin || "api";
@ -208,7 +207,7 @@ export async function extractController(
});
console.log("completions.extract", completions.extract);
// console.log("completions.extract", completions.extract);
let data: any;
try {