AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-18 10:05:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

minor spacing and comment stuff

Browse Source
This commit is contained in:
rafaelsideguide 2024-07-01 16:05:34 -03:00
parent a5fb45988c
commit 4d6e25619b
1 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -286,8 +286,6 @@ export class WebCrawler {
if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS
if (this.isInternalLink(fullUrl) &&
this.noSections(fullUrl) &&
// The idea here to comment this out is to allow wider website coverage as we filter this anyway afterwards
// this.matchesIncludes(path) &&
!this.matchesExcludes(path) &&
this.isRobotsAllowed(fullUrl)
) {
@ -295,7 +293,7 @@ export class WebCrawler {
}
} else { // EXTERNAL LINKS
if (
this.isInternalLink(url) && //its avoid to add links from external pages on the queue
this.isInternalLink(url) &&
this.allowExternalContentLinks &&
!this.isSocialMediaOrEmail(fullUrl) &&
!this.matchesExcludes(fullUrl, true) &&