AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-05 20:46:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/mog/no-cluster' into nsc/no-cluster-all

Browse Source
This commit is contained in:
Nicolas 2024-10-09 19:26:11 -03:00
commit 7847404e1f
2 changed files with 149 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/Dockerfile
View File

@ -28,9 +28,6 @@ RUN cd /app/src/lib/go-html-to-md && \
chmod +x html-to-markdown.so chmod +x html-to-markdown.so
FROM base FROM base
RUN apt-get update -qq && \
apt-get install --no-install-recommends -y chromium chromium-sandbox && \
rm -rf /var/lib/apt/lists /var/cache/apt/archives
COPY --from=prod-deps /app/node_modules /app/node_modules COPY --from=prod-deps /app/node_modules /app/node_modules
COPY --from=build /app /app COPY --from=build /app /app
COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/dist/src/lib/go-html-to-md/html-to-markdown.so COPY --from=go-base /app/src/lib/go-html-to-md/html-to-markdown.so /app/dist/src/lib/go-html-to-md/html-to-markdown.so

104
apps/api/src/index.ts
View File

@ -7,7 +7,6 @@ import cors from "cors";
import { getScrapeQueue } from "./services/queue-service"; import { getScrapeQueue } from "./services/queue-service";
import { v0Router } from "./routes/v0"; import { v0Router } from "./routes/v0";
import { initSDK } from "@hyperdx/node-opentelemetry"; import { initSDK } from "@hyperdx/node-opentelemetry";
import cluster from "cluster";
import os from "os"; import os from "os";
import { Logger } from "./lib/logger"; import { Logger } from "./lib/logger";
import { adminRouter } from "./routes/admin"; import { adminRouter } from "./routes/admin";
@ -37,68 +36,52 @@ const cacheable = new CacheableLookup({
cacheable.install(http.globalAgent); cacheable.install(http.globalAgent);
cacheable.install(https.globalAgent) cacheable.install(https.globalAgent)
if (cluster.isMaster) { const ws = expressWs(express());
Logger.info(`Master ${process.pid} is running`); const app = ws.app;
// Fork workers. global.isProduction = process.env.IS_PRODUCTION === "true";
for (let i = 0; i < numCPUs; i++) {
cluster.fork();
}
cluster.on("exit", (worker, code, signal) => { app.use(bodyParser.urlencoded({ extended: true }));
if (code !== null) { app.use(bodyParser.json({ limit: "10mb" }));
Logger.info(`Worker ${worker.process.pid} exited`);
Logger.info("Starting a new worker");
cluster.fork();
}
});
} else {
const ws = expressWs(express());
const app = ws.app;
global.isProduction = process.env.IS_PRODUCTION === "true"; app.use(cors()); // Add this line to enable CORS
app.use(bodyParser.urlencoded({ extended: true })); const serverAdapter = new ExpressAdapter();
app.use(bodyParser.json({ limit: "10mb" })); serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
app.use(cors()); // Add this line to enable CORS const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [new BullAdapter(getScrapeQueue())], queues: [new BullAdapter(getScrapeQueue())],
serverAdapter: serverAdapter, serverAdapter: serverAdapter,
}); });
app.use( app.use(
`/admin/${process.env.BULL_AUTH_KEY}/queues`, `/admin/${process.env.BULL_AUTH_KEY}/queues`,
serverAdapter.getRouter() serverAdapter.getRouter()
); );
app.get("/", (req, res) => { app.get("/", (req, res) => {
res.send("SCRAPERS-JS: Hello, world! Fly.io"); res.send("SCRAPERS-JS: Hello, world! Fly.io");
}); });
//write a simple test function //write a simple test function
app.get("/test", async (req, res) => { app.get("/test", async (req, res) => {
res.send("Hello, world!"); res.send("Hello, world!");
}); });
// register router // register router
app.use(v0Router); app.use(v0Router);
app.use("/v1", v1Router); app.use("/v1", v1Router);
app.use(adminRouter); app.use(adminRouter);
const DEFAULT_PORT = process.env.PORT ?? 3002; const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost"; const HOST = process.env.HOST ?? "localhost";
// HyperDX OpenTelemetry // HyperDX OpenTelemetry
if (process.env.ENV === "production") { if (process.env.ENV === "production") {
initSDK({ consoleCapture: true, additionalInstrumentations: [] }); initSDK({ consoleCapture: true, additionalInstrumentations: [] });
} }
function startServer(port = DEFAULT_PORT) { function startServer(port = DEFAULT_PORT) {
const server = app.listen(Number(port), HOST, () => { const server = app.listen(Number(port), HOST, () => {
Logger.info(`Worker ${process.pid} listening on port ${port}`); Logger.info(`Worker ${process.pid} listening on port ${port}`);
Logger.info( Logger.info(
@ -106,13 +89,13 @@ if (cluster.isMaster) {
); );
}); });
return server; return server;
} }
if (require.main === module) { if (require.main === module) {
startServer(); startServer();
} }
app.get(`/serverHealthCheck`, async (req, res) => { app.get(`/serverHealthCheck`, async (req, res) => {
try { try {
const scrapeQueue = getScrapeQueue(); const scrapeQueue = getScrapeQueue();
const [waitingJobs] = await Promise.all([ const [waitingJobs] = await Promise.all([
@ -129,9 +112,9 @@ if (cluster.isMaster) {
Logger.error(error); Logger.error(error);
return res.status(500).json({ error: error.message }); return res.status(500).json({ error: error.message });
} }
}); });
app.get("/serverHealthCheck/notify", async (req, res) => { app.get("/serverHealthCheck/notify", async (req, res) => {
if (process.env.SLACK_WEBHOOK_URL) { if (process.env.SLACK_WEBHOOK_URL) {
const treshold = 1; // The treshold value for the active jobs const treshold = 1; // The treshold value for the active jobs
const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
@ -184,13 +167,13 @@ if (cluster.isMaster) {
checkWaitingJobs(); checkWaitingJobs();
} }
}); });
app.get("/is-production", (req, res) => { app.get("/is-production", (req, res) => {
res.send({ isProduction: global.isProduction }); res.send({ isProduction: global.isProduction });
}); });
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => { app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: Response<ErrorResponse>, next: NextFunction) => {
if (err instanceof ZodError) { if (err instanceof ZodError) {
if (Array.isArray(err.errors) && err.errors.find(x => x.message === "URL uses unsupported protocol")) { if (Array.isArray(err.errors) && err.errors.find(x => x.message === "URL uses unsupported protocol")) {
Logger.warn("Unsupported protocol error: " + JSON.stringify(req.body)); Logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));
@ -200,11 +183,11 @@ if (cluster.isMaster) {
} else { } else {
next(err); next(err);
} }
}); });
Sentry.setupExpressErrorHandler(app); Sentry.setupExpressErrorHandler(app);
app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry<ErrorResponse>, next: NextFunction) => { app.use((err: unknown, req: Request<{}, ErrorResponse, undefined>, res: ResponseWithSentry<ErrorResponse>, next: NextFunction) => {
if (err instanceof SyntaxError && 'status' in err && err.status === 400 && 'body' in err) { if (err instanceof SyntaxError && 'status' in err && err.status === 400 && 'body' in err) {
return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' }); return res.status(400).json({ success: false, error: 'Bad request, malformed JSON' });
} }
@ -223,12 +206,9 @@ if (cluster.isMaster) {
Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose); Logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- " + verbose);
res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id }); res.status(500).json({ success: false, error: "An unexpected error occurred. Please contact hello@firecrawl.com for help. Your exception ID is " + id });
}); });
Logger.info(`Worker ${process.pid} started`);
}
Logger.info(`Worker ${process.pid} started`);
// const sq = getScrapeQueue(); // const sq = getScrapeQueue();