From 3d5dc9d90aa657f1d532e52ab4ebacedd5933506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 19:39:10 +0200 Subject: [PATCH 1/7] feat(sentry): add log + server name --- apps/api/src/services/sentry.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts index 04a0b1e4..11dcab45 100644 --- a/apps/api/src/services/sentry.ts +++ b/apps/api/src/services/sentry.ts @@ -1,8 +1,10 @@ // Import with `import * as Sentry from "@sentry/node"` if you are using ESM import * as Sentry from "@sentry/node"; import { nodeProfilingIntegration } from "@sentry/profiling-node"; +import { Logger } from "../lib/logger"; if (process.env.SENTRY_DSN) { + Logger.info("Setting up Sentry..."); Sentry.init({ dsn: process.env.SENTRY_DSN, integrations: [ @@ -10,5 +12,6 @@ if (process.env.SENTRY_DSN) { ], tracesSampleRate: 1.0, profilesSampleRate: 1.0, + serverName: process.env.FLY_MACHINE_ID, }); } From 6104d7421342182c3297ea317e31f74a75dece7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 20:12:47 +0200 Subject: [PATCH 2/7] fix(sentry): drop profiling sample rate --- apps/api/src/services/sentry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts index 11dcab45..266fe4c7 100644 --- a/apps/api/src/services/sentry.ts +++ b/apps/api/src/services/sentry.ts @@ -11,7 +11,7 @@ if (process.env.SENTRY_DSN) { nodeProfilingIntegration(), ], tracesSampleRate: 1.0, - profilesSampleRate: 1.0, + profilesSampleRate: 0.25, serverName: process.env.FLY_MACHINE_ID, }); } From 9579f03c4be33468ac0760d1f8c0592181ab9fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 20:16:06 +0200 Subject: [PATCH 3/7] fix: import resolution --- apps/api/tsconfig.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json index 239d9b3a..dd7f0ed2 100644 --- a/apps/api/tsconfig.json +++ b/apps/api/tsconfig.json @@ -17,12 +17,7 @@ "*": ["node_modules/*", "src/types/*"], }, - "inlineSources": true, - - // Set `sourceRoot` to "/" to strip the build path prefix - // from generated source code references. - // This improves issue grouping in Sentry. - "sourceRoot": "/" + "inlineSources": true }, "include": ["src/","src/**/*", "services/db/supabase.ts", "utils/utils.ts", "services/db/supabaseEmbeddings.ts", "utils/EventEmmitter.ts", "src/services/queue-service.ts"] } From ac9783ed2f70d4ea907d838e1c3e792a98d214ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 20:21:16 +0200 Subject: [PATCH 4/7] fix(sentry): adjust profiles sample rate to be even lower --- apps/api/src/services/sentry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts index 266fe4c7..7bd318ad 100644 --- a/apps/api/src/services/sentry.ts +++ b/apps/api/src/services/sentry.ts @@ -11,7 +11,7 @@ if (process.env.SENTRY_DSN) { nodeProfilingIntegration(), ], tracesSampleRate: 1.0, - profilesSampleRate: 0.25, + profilesSampleRate: 0.045, serverName: process.env.FLY_MACHINE_ID, }); } From dae1408e66ff312f8cca6a2fc05dbfa1b7ba5170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 20:40:42 +0200 Subject: [PATCH 5/7] fix(Dockerfile): retain sentry auth token properly --- apps/api/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 201b2ce9..3ffede0d 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -15,8 +15,7 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile RUN apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates RUN pnpm install RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \ - SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)" \ - bash -c "if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi" + bash -c 'export SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)"; if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi' # Install packages needed for deployment From 55009e51f5338c05a913da3c2a9b2e88c1d9f5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 20:49:25 +0200 Subject: [PATCH 6/7] fix: filter out invalid URLs from crawl links --- apps/api/src/scraper/WebScraper/crawler.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index af3a9d69..02894cfc 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -267,9 +267,18 @@ export class WebCrawler { public filterURL(href: string, url: string): string | null { let fullUrl = href; if (!href.startsWith("http")) { - fullUrl = new URL(href, this.baseUrl).toString(); + try { + fullUrl = new URL(href, this.baseUrl).toString(); + } catch (_) { + return null; + } + } + let urlObj; + try { + urlObj = new URL(fullUrl); + } catch (_) { + return null; } - const urlObj = new URL(fullUrl); const path = urlObj.pathname; if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS From 629da74a5c673f641b613066042c511538c78f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 21 Aug 2024 20:51:35 +0200 Subject: [PATCH 7/7] fix(sentry): decrease tracesSampleRate --- apps/api/src/services/sentry.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts index 7bd318ad..1292773a 100644 --- a/apps/api/src/services/sentry.ts +++ b/apps/api/src/services/sentry.ts @@ -10,8 +10,8 @@ if (process.env.SENTRY_DSN) { integrations: [ nodeProfilingIntegration(), ], - tracesSampleRate: 1.0, - profilesSampleRate: 0.045, + tracesSampleRate: 0.045, + profilesSampleRate: 1.0, serverName: process.env.FLY_MACHINE_ID, }); }