AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 15:00:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update index.ts

Browse Source
This commit is contained in:
Nicolas 2024-07-23 15:15:56 -04:00
parent 252bc09ee2
commit f0b07b509b
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/index.ts
View File

@ -355,7 +355,7 @@ export class WebScraperDataProvider {
documents: Document[], documents: Document[],
links: string[] links: string[]
): Promise<Document[]> { ): Promise<Document[]> {
await this.setCachedDocuments(documents, links); // await this.setCachedDocuments(documents, links);
documents = this.removeChildLinks(documents); documents = this.removeChildLinks(documents);
return documents.splice(0, this.limit); return documents.splice(0, this.limit);
} }
@ -451,7 +451,7 @@ export class WebScraperDataProvider {
...document, ...document,
childrenLinks: childrenLinks || [], childrenLinks: childrenLinks || [],
}), }),
60 * 60 * 24 * 10 60 * 60
); // 10 days ); // 10 days
} }
} }