diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts index ebdcfba3..f95364a3 100644 --- a/apps/api/src/controllers/v1/crawl.ts +++ b/apps/api/src/controllers/v1/crawl.ts @@ -73,6 +73,7 @@ export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, Cr pageOptions: pageOptions, origin: "api", crawl_id: id, + webhook: req.body.webhook, }, { priority: 15, }); diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index f43b433a..c047bd16 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -78,6 +78,7 @@ export const crawlRequestSchema = z.object({ origin: z.string().optional().default("api"), crawlerOptions: crawlerOptions.default({}), scrapeOptions: scrapeOptions.omit({ timeout: true }).default({}), + webhook: z.string().url().optional(), }); // export type CrawlRequest = { diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index e50aae84..cd4e03a1 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -152,7 +152,7 @@ async function processJob(job: Job, token: string) { }; if (job.data.mode === "crawl") { - await callWebhook(job.data.team_id, job.id as string, data); + await callWebhook(job.data.team_id, job.id as string, data, job.data.webhook); } if (job.data.crawl_id) { diff --git a/apps/api/src/services/webhook.ts b/apps/api/src/services/webhook.ts index b0222ea3..2b008667 100644 --- a/apps/api/src/services/webhook.ts +++ b/apps/api/src/services/webhook.ts @@ -1,15 +1,15 @@ import { Logger } from "../../src/lib/logger"; import { supabase_service } from "./supabase"; -export const callWebhook = async (teamId: string, jobId: string,data: any) => { +export const callWebhook = async (teamId: string, jobId: string, data: any, specified?: string) => { try { const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}", jobId); const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; - let webhookUrl = selfHostedUrl; + let webhookUrl = specified ?? selfHostedUrl; - // Only fetch the webhook URL from the database if the self-hosted webhook URL is not set + // Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set // and the USE_DB_AUTHENTICATION environment variable is set to true - if (!selfHostedUrl && useDbAuthentication) { + if (!webhookUrl && useDbAuthentication) { const { data: webhooksData, error } = await supabase_service .from("webhooks") .select("url") diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index ca5ff3e3..5e63ac78 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -30,6 +30,7 @@ export interface WebScraperOptions { origin?: string; crawl_id?: string; sitemapped?: boolean; + webhook?: string; } export interface RunWebScraperParams {