AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-11 17:28:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into nsc/job-priority

Browse Source
This commit is contained in:
Nicolas 2024-08-21 20:54:57 -03:00
commit e64d3815ea
4 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/api/Dockerfile
View File

@ -15,8 +15,7 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
RUN apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates RUN apt-get update -qq && apt-get install -y ca-certificates && update-ca-certificates
RUN pnpm install RUN pnpm install
RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \ RUN --mount=type=secret,id=SENTRY_AUTH_TOKEN \
SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)" \ bash -c 'export SENTRY_AUTH_TOKEN="$(cat /run/secrets/SENTRY_AUTH_TOKEN)"; if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi'
bash -c "if [ -z $SENTRY_AUTH_TOKEN ]; then pnpm run build:nosentry; else pnpm run build; fi"
# Install packages needed for deployment # Install packages needed for deployment

13
apps/api/src/scraper/WebScraper/crawler.ts
View File

@ -267,9 +267,18 @@ export class WebCrawler {
public filterURL(href: string, url: string): string | null { public filterURL(href: string, url: string): string | null {
let fullUrl = href; let fullUrl = href;
if (!href.startsWith("http")) { if (!href.startsWith("http")) {
fullUrl = new URL(href, this.baseUrl).toString(); try {
fullUrl = new URL(href, this.baseUrl).toString();
} catch (_) {
return null;
}
}
let urlObj;
try {
urlObj = new URL(fullUrl);
} catch (_) {
return null;
} }
const urlObj = new URL(fullUrl);
const path = urlObj.pathname; const path = urlObj.pathname;
if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS if (this.isInternalLink(fullUrl)) { // INTERNAL LINKS

5
apps/api/src/services/sentry.ts
View File

@ -1,14 +1,17 @@
// Import with `import * as Sentry from "@sentry/node"` if you are using ESM // Import with `import * as Sentry from "@sentry/node"` if you are using ESM
import * as Sentry from "@sentry/node"; import * as Sentry from "@sentry/node";
import { nodeProfilingIntegration } from "@sentry/profiling-node"; import { nodeProfilingIntegration } from "@sentry/profiling-node";
import { Logger } from "../lib/logger";
if (process.env.SENTRY_DSN) { if (process.env.SENTRY_DSN) {
Logger.info("Setting up Sentry...");
Sentry.init({ Sentry.init({
dsn: process.env.SENTRY_DSN, dsn: process.env.SENTRY_DSN,
integrations: [ integrations: [
nodeProfilingIntegration(), nodeProfilingIntegration(),
], ],
tracesSampleRate: 1.0, tracesSampleRate: 0.045,
profilesSampleRate: 1.0, profilesSampleRate: 1.0,
serverName: process.env.FLY_MACHINE_ID,
}); });
} }

7
apps/api/tsconfig.json
View File

@ -17,12 +17,7 @@
"*": ["node_modules/*", "src/types/*"], "*": ["node_modules/*", "src/types/*"],
}, },
"inlineSources": true, "inlineSources": true
// Set `sourceRoot` to "/" to strip the build path prefix
// from generated source code references.
// This improves issue grouping in Sentry.
"sourceRoot": "/"
}, },
"include": ["src/","src/**/*", "services/db/supabase.ts", "utils/utils.ts", "services/db/supabaseEmbeddings.ts", "utils/EventEmmitter.ts", "src/services/queue-service.ts"] "include": ["src/","src/**/*", "services/db/supabase.ts", "utils/utils.ts", "services/db/supabaseEmbeddings.ts", "utils/EventEmmitter.ts", "src/services/queue-service.ts"]
} }