From 22a5e85899eb893c9a68f53201e13f5fb569bc46 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 9 Sep 2024 12:26:55 -0300 Subject: [PATCH 1/2] Update index.ts --- apps/api/src/index.ts | 102 ++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 34 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 58370158..1edf3759 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import "./services/sentry" +import "./services/sentry"; import * as Sentry from "@sentry/node"; import express, { NextFunction, Request, Response } from "express"; import bodyParser from "body-parser"; @@ -12,9 +12,9 @@ import os from "os"; import { Logger } from "./lib/logger"; import { adminRouter } from "./routes/admin"; import { ScrapeEvents } from "./lib/scrape-events"; -import http from 'node:http'; -import https from 'node:https'; -import CacheableLookup from 'cacheable-lookup'; +import http from "node:http"; +import https from "node:https"; +import CacheableLookup from "cacheable-lookup"; import { v1Router } from "./routes/v1"; import expressWs from "express-ws"; import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws"; @@ -31,11 +31,11 @@ Logger.info(`Number of CPUs: ${numCPUs} available`); const cacheable = new CacheableLookup({ // this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme - lookup:false + lookup: false, }); cacheable.install(http.globalAgent); -cacheable.install(https.globalAgent) +cacheable.install(https.globalAgent); if (cluster.isMaster) { Logger.info(`Master ${process.pid} is running`); @@ -115,9 +115,7 @@ if (cluster.isMaster) { app.get(`/serverHealthCheck`, async (req, res) => { try { const scrapeQueue = getScrapeQueue(); - const [waitingJobs] = await Promise.all([ - scrapeQueue.getWaitingCount(), - ]); + const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]); const noWaitingJobs = waitingJobs === 0; // 200 if no active jobs, 503 if there are active jobs @@ -190,38 +188,77 @@ if (cluster.isMaster) { res.send({ isProduction: global.isProduction }); }); - app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response, next: NextFunction) => { - if (err instanceof ZodError) { - res.status(400).json({ success: false, error: "Bad Request", details: err.errors }); - } else { + app.use( + ( + err: unknown, + req: Request<{}, ErrorResponse, undefined>, + res: Response, + next: NextFunction + ) => { + if (err instanceof ZodError) { + res + .status(400) + .json({ success: false, error: "Bad Request", details: err.errors }); + } else { next(err); + } } - }); + ); Sentry.setupExpressErrorHandler(app); - app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry, next: NextFunction) => { - const id = res.sentry ?? uuidv4(); - let verbose = JSON.stringify(err); - if (verbose === "{}") { - if (err instanceof Error) { - verbose = JSON.stringify({ - message: err.message, - name: err.name, - stack: err.stack, - }); - } - } + app.use( + ( + err: unknown, + req: Request<{}, ErrorResponse, undefined>, + res: ResponseWithSentry, + next: NextFunction + ) => { + if ( + err instanceof SyntaxError && + "status" in err && + err.status === 400 && + "body" in err + ) { + return res + .status(400) + .json({ success: false, error: "Bad request, malformed JSON" }); + } - Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose); - res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id }); - }); + const id = res.sentry ?? uuidv4(); + let verbose = JSON.stringify(err); + if (verbose === "{}") { + if (err instanceof Error) { + verbose = JSON.stringify({ + message: err.message, + name: err.name, + stack: err.stack, + }); + } + } + + Logger.error( + "Error occurred in request! (" + + req.path + + ") -- ID " + + id + + " -- " + + verbose + ); + res + .status(500) + .json({ + success: false, + error: + "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + + id, + }); + } + ); Logger.info(`Worker ${process.pid} started`); } - - // const sq = getScrapeQueue(); // sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting")); @@ -230,6 +267,3 @@ if (cluster.isMaster) { // sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused")); // sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed")); // sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed")); - - - From ca9a781eb7fbadf7aee7dd6926aea3a0b1ca5e07 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 9 Sep 2024 12:27:55 -0300 Subject: [PATCH 2/2] Update index.ts --- apps/api/src/index.ts | 106 +++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 68 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 1edf3759..7d8817af 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import "./services/sentry"; +import "./services/sentry" import * as Sentry from "@sentry/node"; import express, { NextFunction, Request, Response } from "express"; import bodyParser from "body-parser"; @@ -12,9 +12,9 @@ import os from "os"; import { Logger } from "./lib/logger"; import { adminRouter } from "./routes/admin"; import { ScrapeEvents } from "./lib/scrape-events"; -import http from "node:http"; -import https from "node:https"; -import CacheableLookup from "cacheable-lookup"; +import http from 'node:http'; +import https from 'node:https'; +import CacheableLookup from 'cacheable-lookup'; import { v1Router } from "./routes/v1"; import expressWs from "express-ws"; import { crawlStatusWSController } from "./controllers/v1/crawl-status-ws"; @@ -31,11 +31,11 @@ Logger.info(`Number of CPUs: ${numCPUs} available`); const cacheable = new CacheableLookup({ // this is important to avoid querying local hostnames see https://github.com/szmarczak/cacheable-lookup readme - lookup: false, + lookup:false }); cacheable.install(http.globalAgent); -cacheable.install(https.globalAgent); +cacheable.install(https.globalAgent) if (cluster.isMaster) { Logger.info(`Master ${process.pid} is running`); @@ -115,7 +115,9 @@ if (cluster.isMaster) { app.get(`/serverHealthCheck`, async (req, res) => { try { const scrapeQueue = getScrapeQueue(); - const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]); + const [waitingJobs] = await Promise.all([ + scrapeQueue.getWaitingCount(), + ]); const noWaitingJobs = waitingJobs === 0; // 200 if no active jobs, 503 if there are active jobs @@ -188,77 +190,42 @@ if (cluster.isMaster) { res.send({ isProduction: global.isProduction }); }); - app.use( - ( - err: unknown, - req: Request<{}, ErrorResponse, undefined>, - res: Response, - next: NextFunction - ) => { - if (err instanceof ZodError) { - res - .status(400) - .json({ success: false, error: "Bad Request", details: err.errors }); - } else { + app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response, next: NextFunction) => { + if (err instanceof ZodError) { + res.status(400).json({ success: false, error: "Bad Request", details: err.errors }); + } else { next(err); - } } - ); + }); Sentry.setupExpressErrorHandler(app); - app.use( - ( - err: unknown, - req: Request<{}, ErrorResponse, undefined>, - res: ResponseWithSentry, - next: NextFunction - ) => { - if ( - err instanceof SyntaxError && - "status" in err && - err.status === 400 && - "body" in err - ) { - return res - .status(400) - .json({ success: false, error: "Bad request, malformed JSON" }); - } - - const id = res.sentry ?? uuidv4(); - let verbose = JSON.stringify(err); - if (verbose === "{}") { - if (err instanceof Error) { - verbose = JSON.stringify({ - message: err.message, - name: err.name, - stack: err.stack, - }); - } - } - - Logger.error( - "Error occurred in request! (" + - req.path + - ") -- ID " + - id + - " -- " + - verbose - ); - res - .status(500) - .json({ - success: false, - error: - "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + - id, - }); + app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry, next: NextFunction) => { + if (err instanceof SyntaxError && 'status' in err && err.status === 400 && 'body' in err) { + return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' }); } - ); + + const id = res.sentry ?? uuidv4(); + let verbose = JSON.stringify(err); + if (verbose === "{}") { + if (err instanceof Error) { + verbose = JSON.stringify({ + message: err.message, + name: err.name, + stack: err.stack, + }); + } + } + + Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose); + res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id }); + }); Logger.info(`Worker ${process.pid} started`); } + + // const sq = getScrapeQueue(); // sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting")); @@ -267,3 +234,6 @@ if (cluster.isMaster) { // sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused")); // sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed")); // sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed")); + + +