AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-18 01:25:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

minor spacing and comment stuff

Browse Source
This commit is contained in:
rafaelsideguide 2024-07-01 16:05:34 -03:00
parent a5fb45988c
commit 4d6e25619b
1 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -283,24 +283,22 @@ export class WebCrawler {
const path = urlObj.pathname;
if(this.isInternalLink(fullUrl)){ // INTERNAL LINKS
if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS
if (this.isInternalLink(fullUrl) &&
this.noSections(fullUrl) &&
// The idea here to comment this out is to allow wider website coverage as we filter this anyway afterwards
// this.matchesIncludes(path) &&
!this.matchesExcludes(path) &&
this.isRobotsAllowed(fullUrl)
) {
links.push({ url: fullUrl, html: content, pageStatusCode, pageError });
}
}else{ // EXTERNAL LINKS
if(
this.isInternalLink(url) && //its avoid to add links from external pages on the queue
} else { // EXTERNAL LINKS
if (
this.isInternalLink(url) &&
this.allowExternalContentLinks &&
!this.isSocialMediaOrEmail(fullUrl) &&
!this.matchesExcludes(fullUrl, true) &&
!this.isExternalMainPage(fullUrl)
){
) {
links.push({ url: fullUrl, html: content, pageStatusCode, pageError });
}
}