AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-13 05:19:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
Gergő Móricz 2025-04-10 12:50:03 +02:00
parent 4294face78
commit 415603acb0
2 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
apps/api/src/scraper/scrapeURL/transformers/diff.ts
View File

@ -1,22 +1,29 @@
import { supabase_service } from "../../../services/supabase";
import { Document } from "../../../controllers/v1/types";
import { Meta } from "../index";
import { getJob } from "../../../controllers/v1/crawl-status";
export async function deriveDiff(meta: Meta, document: Document): Promise<Document> {
if (meta.options.formats.includes("changeTracking")) {
const res = await supabase_service
.rpc("diff_get_last_scrape_1", {
.rpc("diff_get_last_scrape_2", {
i_team_id: meta.internalOptions.teamId,
i_url: document.metadata.sourceURL ?? meta.url,
});
const data: {
o_docs: Document[],
o_job_id: string,
o_date_added: string,
} | undefined | null = (res.data ?? [])[0] as any;
if (data && data.o_docs.length > 0) {
const previousMarkdown = data.o_docs[0].markdown!;
const job: {
returnvalue: Document,
} | null = data?.o_job_id ? await getJob(data.o_job_id) : null;
console.log(data, job);
if (data && job && job?.returnvalue) {
const previousMarkdown = job.returnvalue.markdown!;
const currentMarkdown = document.markdown!;
const transformer = (x: string) => [...x.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")].sort().join("");

2
apps/api/src/services/logging/log_job.ts
View File

@ -89,7 +89,7 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {
success: job.success,
message: job.message,
num_docs: job.num_docs,
docs: (job.mode === "single_urls" || job.mode === "scrape") ? null : cleanOfNull(job.docs),
docs: ((job.mode === "single_urls" || job.mode === "scrape") && process.env.GCS_BUCKET_NAME) ? null : cleanOfNull(job.docs),
time_taken: job.time_taken,
team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_"))? null : job.team_id,
mode: job.mode,