AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 00:36:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: random fixes

Browse Source
This commit is contained in:
Nicolas 2024-12-15 02:52:06 -03:00
parent 664ba69f08
commit 4987880b32
2 changed files with 40 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
apps/api/src/scraper/WebScraper/utils/blocklist.ts
View File

@ -6,6 +6,15 @@ configDotenv();
const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8"); const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");
const algorithm = "aes-256-ecb"; const algorithm = "aes-256-ecb";
function encryptAES(plaintext: string, key: Buffer): string {
const cipher = crypto.createCipheriv(algorithm, key, null);
const encrypted = Buffer.concat([
cipher.update(plaintext, "utf-8"),
cipher.final()
]);
return encrypted.toString("base64");
}
function decryptAES(ciphertext: string, key: Buffer): string { function decryptAES(ciphertext: string, key: Buffer): string {
const decipher = crypto.createDecipheriv(algorithm, key, null); const decipher = crypto.createDecipheriv(algorithm, key, null);
const decrypted = Buffer.concat([ const decrypted = Buffer.concat([
@ -42,6 +51,21 @@ const urlBlocklist = [
"PTbGg8PK/h0Seyw4HEpK4Q==", "PTbGg8PK/h0Seyw4HEpK4Q==",
"lZdQMknjHb7+4+sjF3qNTw==", "lZdQMknjHb7+4+sjF3qNTw==",
"LsgSq54q5oDysbva29JxnQ==", "LsgSq54q5oDysbva29JxnQ==",
"KZfBtpwjOpdSoqacRbz7og==",
"Indtl4yxJMHCKBGF4KABCQ==",
"e3HFXLVgxhaVoadYpwb2BA==",
"b+asgLayXQ5Jq+se+q56jA==",
"86ZDUI7vmp4MvNq3fvZrGQ==",
"sEGFoYZ6GEg4Zocd+TiyfQ==",
"6OOL72eXthgnJ1Hj4PfOQQ==",
"g/ME+Sh1CAFboKrwkVb+5Q==",
"Pw+xawUoX8xBYbX2yqqGWQ==",
"k6vBalxYFhAvkPsF19t9gQ==",
"e3HFXLVgxhaVoadYpwb2BA==",
"b+asgLayXQ5Jq+se+q56jA==",
"KKttwRz4w+AMJrZcB828WQ==",
"vMdzZ33BXoyWVZnAPOBcrg==",
"l8GDVI8w/ueHnNzdN1ODuQ==",
]; ];
const decryptedBlocklist = hashKey.length > 0 ? urlBlocklist.map((ciphertext) => decryptAES(ciphertext, hashKey)) : []; const decryptedBlocklist = hashKey.length > 0 ? urlBlocklist.map((ciphertext) => decryptAES(ciphertext, hashKey)) : [];

31
apps/api/src/services/queue-worker.ts
View File

@ -391,22 +391,21 @@ async function processJob(job: Job & { id: string }, token: string) {
// Check if the job URL is researchhub and block it immediately // Check if the job URL is researchhub and block it immediately
// TODO: remove this once solve the root issue // TODO: remove this once solve the root issue
if ( // if (
job.data.url && // job.data.url &&
(job.data.url.includes("researchhub.com") || // (job.data.url.includes("researchhub.com") ||
job.data.url.includes("ebay.com") || // job.data.url.includes("ebay.com"))
job.data.url.includes("youtube.com")) // ) {
) { // logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`);
logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`); // const data = {
const data = { // success: false,
success: false, // document: null,
document: null, // project_id: job.data.project_id,
project_id: job.data.project_id, // error:
error: // "URL is blocked. Suspecious activity detected. Please contact help@firecrawl.com if you believe this is an error.",
"URL is blocked. Suspecious activity detected. Please contact help@firecrawl.com if you believe this is an error.", // };
}; // return data;
return data; // }
}
try { try {
job.updateProgress({ job.updateProgress({