AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-18 05:45:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(crawl): webhook

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-16 23:42:40 +02:00
parent 086ba6280b
commit 803577eeba
5 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/src/controllers/v1/crawl.ts
View File

@ -73,6 +73,7 @@ export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, Cr
pageOptions: pageOptions, pageOptions: pageOptions,
origin: "api", origin: "api",
crawl_id: id, crawl_id: id,
webhook: req.body.webhook,
}, { }, {
priority: 15, priority: 15,
}); });

1
apps/api/src/controllers/v1/types.ts
View File

@ -78,6 +78,7 @@ export const crawlRequestSchema = z.object({
origin: z.string().optional().default("api"), origin: z.string().optional().default("api"),
crawlerOptions: crawlerOptions.default({}), crawlerOptions: crawlerOptions.default({}),
scrapeOptions: scrapeOptions.omit({ timeout: true }).default({}), scrapeOptions: scrapeOptions.omit({ timeout: true }).default({}),
webhook: z.string().url().optional(),
}); });
// export type CrawlRequest = { // export type CrawlRequest = {

2
apps/api/src/services/queue-worker.ts
View File

@ -152,7 +152,7 @@ async function processJob(job: Job, token: string) {
}; };
if (job.data.mode === "crawl") { if (job.data.mode === "crawl") {
await callWebhook(job.data.team_id, job.id as string, data); await callWebhook(job.data.team_id, job.id as string, data, job.data.webhook);
} }
if (job.data.crawl_id) { if (job.data.crawl_id) {

8
apps/api/src/services/webhook.ts
View File

@ -1,15 +1,15 @@
import { Logger } from "../../src/lib/logger"; import { Logger } from "../../src/lib/logger";
import { supabase_service } from "./supabase"; import { supabase_service } from "./supabase";
export const callWebhook = async (teamId: string, jobId: string,data: any) => { export const callWebhook = async (teamId: string, jobId: string, data: any, specified?: string) => {
try { try {
const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}", jobId); const selfHostedUrl = process.env.SELF_HOSTED_WEBHOOK_URL?.replace("{{JOB_ID}}", jobId);
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true'; const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
let webhookUrl = selfHostedUrl; let webhookUrl = specified ?? selfHostedUrl;
// Only fetch the webhook URL from the database if the self-hosted webhook URL is not set // Only fetch the webhook URL from the database if the self-hosted webhook URL and specified webhook are not set
// and the USE_DB_AUTHENTICATION environment variable is set to true // and the USE_DB_AUTHENTICATION environment variable is set to true
if (!selfHostedUrl && useDbAuthentication) { if (!webhookUrl && useDbAuthentication) {
const { data: webhooksData, error } = await supabase_service const { data: webhooksData, error } = await supabase_service
.from("webhooks") .from("webhooks")
.select("url") .select("url")

1
apps/api/src/types.ts
View File

@ -30,6 +30,7 @@ export interface WebScraperOptions {
origin?: string; origin?: string;
crawl_id?: string; crawl_id?: string;
sitemapped?: boolean; sitemapped?: boolean;
webhook?: string;
} }
export interface RunWebScraperParams { export interface RunWebScraperParams {