AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 12:00:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into nsc/new-extract

Browse Source
This commit is contained in:
Nicolas 2024-11-12 12:24:47 -05:00
commit 464b41a5d2
4 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/index.ts
View File

@ -80,6 +80,17 @@ function startServer(port = DEFAULT_PORT) {
`For the Queue UI, open: http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`
);
});
const exitHandler = () => {
logger.info('SIGTERM signal received: closing HTTP server')
server.close(() => {
logger.info("Server closed.");
process.exit(0);
});
};
process.on('SIGTERM', exitHandler);
process.on('SIGINT', exitHandler);
return server;
}

4
apps/api/src/lib/crawl-redis.ts
View File

@ -166,10 +166,10 @@ export async function lockURLs(id: string, sc: StoredCrawl, urls: string[]): Pro
return res;
}
export function crawlToCrawler(id: string, sc: StoredCrawl): WebCrawler {
export function crawlToCrawler(id: string, sc: StoredCrawl, initialUrl?: string): WebCrawler {
const crawler = new WebCrawler({
jobId: id,
initialUrl: sc.originUrl!,
initialUrl: initialUrl ?? sc.originUrl!,
includes: sc.crawlerOptions?.includes ?? [],
excludes: sc.crawlerOptions?.excludes ?? [],
maxCrawledLinks: sc.crawlerOptions?.maxCrawledLinks ?? 1000,

2
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -171,7 +171,7 @@ export class WebCrawler {
let fullUrl = href;
if (!href.startsWith("http")) {
try {
fullUrl = new URL(href, this.baseUrl).toString();
fullUrl = new URL(href, url).toString();
} catch (_) {
return null;
}

22
apps/api/src/services/queue-worker.ts
View File

@ -352,10 +352,10 @@ async function processJob(job: Job & { id: string }, token: string) {
if (!job.data.sitemapped && job.data.crawlerOptions !== null) {
if (!sc.cancelled) {
const crawler = crawlToCrawler(job.data.crawl_id, sc);
const crawler = crawlToCrawler(job.data.crawl_id, sc, doc.metadata?.url ?? doc.metadata?.sourceURL ?? undefined);
const links = crawler.filterLinks(
crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl as string),
crawler.extractLinksFromHTML(rawHtml ?? "", doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl as string),
Infinity,
sc.crawlerOptions?.maxDepth ?? 10
);
@ -504,19 +504,19 @@ async function processJob(job: Job & { id: string }, token: string) {
job: job.id,
},
});
if (error instanceof CustomError) {
// Here we handle the error, then save the failed job
logger.error(error.message); // or any other error handling
}
logger.error(error);
if (error.stack) {
logger.error(error.stack);
}
} else {
logger.error(`🐂 Job timed out ${job.id}`);
}
if (error instanceof CustomError) {
// Here we handle the error, then save the failed job
logger.error(error.message); // or any other error handling
}
logger.error(error);
if (error.stack) {
logger.error(error.stack);
}
const data = {
success: false,
document: null,