AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 05:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(v1/batch/scrape): appendToId

Browse Source
This commit is contained in:
Gergő Móricz 2024-12-04 23:35:29 +01:00
parent 7bde034020
commit 41d859203f
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/controllers/v1/batch-scrape.ts
View File

@ -8,6 +8,7 @@ import {
} from "./types";
import {
addCrawlJobs,
getCrawl,
lockURLs,
saveCrawl,
StoredCrawl,
@ -24,9 +25,11 @@ export async function batchScrapeController(
) {
req.body = batchScrapeRequestSchema.parse(req.body);
const id = uuidv4();
const id = req.body.appendToId ?? uuidv4();
await logCrawl(id, req.auth.team_id);
if (!req.body.appendToId) {
await logCrawl(id, req.auth.team_id);
}
let { remainingCredits } = req.account!;
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
@ -34,7 +37,7 @@ export async function batchScrapeController(
remainingCredits = Infinity;
}
const sc: StoredCrawl = {
const sc: StoredCrawl = req.body.appendToId ? await getCrawl(req.body.appendToId) as StoredCrawl : {
crawlerOptions: null,
scrapeOptions: req.body,
internalOptions: {},
@ -43,7 +46,9 @@ export async function batchScrapeController(
plan: req.auth.plan,
};
await saveCrawl(id, sc);
if (!req.body.appendToId) {
await saveCrawl(id, sc);
}
let jobPriority = 20;

1
apps/api/src/controllers/v1/types.ts
View File

@ -210,6 +210,7 @@ export const batchScrapeRequestSchema = scrapeOptions.extend({
urls: url.array(),
origin: z.string().optional().default("api"),
webhook: webhookSchema.optional(),
appendToId: z.string().uuid().optional(),
}).strict(strictMessage).refine(
(obj) => {
const hasExtractFormat = obj.formats?.includes("extract");