AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-19 01:55:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

move crawl to read replica

Browse Source
This commit is contained in:
Gergő Móricz 2025-03-06 21:47:10 +01:00
parent 982b3da27d
commit a1e6c13b67
2 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -21,7 +21,7 @@ import {
import { configDotenv } from "dotenv"; import { configDotenv } from "dotenv";
import type { Job, JobState, Queue } from "bullmq"; import type { Job, JobState, Queue } from "bullmq";
import { logger } from "../../lib/logger"; import { logger } from "../../lib/logger";
import { supabase_service } from "../../services/supabase"; import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit"; import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";
configDotenv(); configDotenv();
@ -246,7 +246,7 @@ export async function crawlStatusController(
let totalCount = jobIDs.length; let totalCount = jobIDs.length;
if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") { if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {
const x = await supabase_service const x = await supabase_rr_service
.from('firecrawl_jobs') .from('firecrawl_jobs')
.select('*', { count: 'exact', head: true }) .select('*', { count: 'exact', head: true })
.eq("crawl_id", req.params.jobId) .eq("crawl_id", req.params.jobId)

10
apps/api/src/lib/supabase-jobs.ts
View File

@ -1,4 +1,4 @@
import { supabase_service } from "../services/supabase"; import { supabase_rr_service, supabase_service } from "../services/supabase";
import { logger } from "./logger"; import { logger } from "./logger";
import * as Sentry from "@sentry/node"; import * as Sentry from "@sentry/node";
@ -8,7 +8,7 @@ import * as Sentry from "@sentry/node";
* @returns {any | null} Job * @returns {any | null} Job
*/ */
export const supabaseGetJobById = async (jobId: string) => { export const supabaseGetJobById = async (jobId: string) => {
const { data, error } = await supabase_service const { data, error } = await supabase_rr_service
.from("firecrawl_jobs") .from("firecrawl_jobs")
.select("*") .select("*")
.eq("job_id", jobId) .eq("job_id", jobId)
@ -31,7 +31,7 @@ export const supabaseGetJobById = async (jobId: string) => {
* @returns {any[]} Jobs * @returns {any[]} Jobs
*/ */
export const supabaseGetJobsById = async (jobIds: string[]) => { export const supabaseGetJobsById = async (jobIds: string[]) => {
const { data, error } = await supabase_service const { data, error } = await supabase_rr_service
.from("firecrawl_jobs") .from("firecrawl_jobs")
.select() .select()
.in("job_id", jobIds); .in("job_id", jobIds);
@ -55,7 +55,7 @@ export const supabaseGetJobsById = async (jobIds: string[]) => {
* @returns {any[]} Jobs * @returns {any[]} Jobs
*/ */
export const supabaseGetJobsByCrawlId = async (crawlId: string) => { export const supabaseGetJobsByCrawlId = async (crawlId: string) => {
const { data, error } = await supabase_service const { data, error } = await supabase_rr_service
.from("firecrawl_jobs") .from("firecrawl_jobs")
.select() .select()
.eq("crawl_id", crawlId); .eq("crawl_id", crawlId);
@ -74,7 +74,7 @@ export const supabaseGetJobsByCrawlId = async (crawlId: string) => {
}; };
export const supabaseGetJobByIdOnlyData = async (jobId: string) => { export const supabaseGetJobByIdOnlyData = async (jobId: string) => {
const { data, error } = await supabase_service const { data, error } = await supabase_rr_service
.from("firecrawl_jobs") .from("firecrawl_jobs")
.select("docs, team_id") .select("docs, team_id")
.eq("job_id", jobId) .eq("job_id", jobId)