AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 11:59:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: formatting

Browse Source
This commit is contained in:
Nicolas 2024-12-15 02:54:49 -03:00
parent 4987880b32
commit 588f747ee8
3 changed files with 64 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
apps/api/src/controllers/v0/admin/check-fire-engine.ts
View File

@ -2,7 +2,6 @@ import { logger } from "../../../lib/logger";
import * as Sentry from "@sentry/node";
import { Request, Response } from "express";
export async function checkFireEngine(req: Request, res: Response) {
try {
if (!process.env.FIRE_ENGINE_BETA_URL) {
@ -17,7 +16,9 @@ export async function checkFireEngine(req: Request, res: Response) {
const timeout = setTimeout(() => controller.abort(), 30000);
try {
const response = await fetch(`${process.env.FIRE_ENGINE_BETA_URL}/scrape`, {
const response = await fetch(
`${process.env.FIRE_ENGINE_BETA_URL}/scrape`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
@ -27,7 +28,8 @@ export async function checkFireEngine(req: Request, res: Response) {
url: "https://example.com",
}),
signal: controller.signal,
});
},
);
clearTimeout(timeout);
@ -43,7 +45,7 @@ export async function checkFireEngine(req: Request, res: Response) {
});
}
} catch (error) {
if (error.name === 'AbortError') {
if (error.name === "AbortError") {
return res.status(504).json({
success: false,
error: "Request timed out after 30 seconds",

17
apps/api/src/main/runWebScraper.ts
View File

@ -74,7 +74,16 @@ export async function runWebScraper({
for (let i = 0; i < tries; i++) {
if (i > 0) {
logger.debug("Retrying scrape...", { scrapeId: bull_job_id, jobId: bull_job_id, method: "runWebScraper", module: "runWebScraper", tries, i, previousStatusCode: (response as any)?.document?.metadata?.statusCode, previousError: error });
logger.debug("Retrying scrape...", {
scrapeId: bull_job_id,
jobId: bull_job_id,
method: "runWebScraper",
module: "runWebScraper",
tries,
i,
previousStatusCode: (response as any)?.document?.metadata?.statusCode,
previousError: error,
});
}
response = undefined;
@ -106,7 +115,11 @@ export async function runWebScraper({
engines = response.engines;
if ((response.document.metadata.statusCode >= 200 && response.document.metadata.statusCode < 300) || response.document.metadata.statusCode === 304) {
if (
(response.document.metadata.statusCode >= 200 &&
response.document.metadata.statusCode < 300) ||
response.document.metadata.statusCode === 304
) {
// status code is good -- do not attempt retry
break;
}

7
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -10,7 +10,7 @@ function encryptAES(plaintext: string, key: Buffer): string {
const cipher = crypto.createCipheriv(algorithm, key, null);
const encrypted = Buffer.concat([
cipher.update(plaintext, "utf-8"),
cipher.final()
cipher.final(),
]);
return encrypted.toString("base64");
}
@ -68,7 +68,10 @@ const urlBlocklist = [
"l8GDVI8w/ueHnNzdN1ODuQ==",
];
const decryptedBlocklist = hashKey.length > 0 ? urlBlocklist.map((ciphertext) => decryptAES(ciphertext, hashKey)) : [];
const decryptedBlocklist =
hashKey.length > 0
? urlBlocklist.map((ciphertext) => decryptAES(ciphertext, hashKey))
: [];
const allowedKeywords = [
"pulse",