mirror of
https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl
synced 2025-06-04 11:24:40 +08:00
39 lines
2.0 KiB
JavaScript
39 lines
2.0 KiB
JavaScript
const fs = require("fs");
|
|
|
|
// METHOD: Winston log file
|
|
// const logs = fs.readFileSync("7a373219-0eb4-4e47-b2df-e90e12afd5c1.log", "utf8")
|
|
// .split("\n").filter(x => x.trim().length > 0).map(x => JSON.parse(x));
|
|
|
|
// METHOD: GCloud export
|
|
const logs = [
|
|
"downloaded-logs-20241213-225607.json",
|
|
"downloaded-logs-20241213-225654.json",
|
|
"downloaded-logs-20241213-225720.json",
|
|
"downloaded-logs-20241213-225758.json",
|
|
"downloaded-logs-20241213-225825.json",
|
|
"downloaded-logs-20241213-225843.json",
|
|
].flatMap(x => JSON.parse(fs.readFileSync(x, "utf8"))).map(x => x.jsonPayload);
|
|
|
|
|
|
const crawlIds = [...new Set(logs.map(x => x.crawlId).filter(x => x))];
|
|
|
|
const urlFilter = x => new URL(x).pathname.slice(1) || "root"
|
|
|
|
for (const crawlId of crawlIds) {
|
|
const crawlLogs = logs.filter(x => x.crawlId === crawlId);
|
|
fs.writeFileSync("crawl-" + crawlId + ".log", crawlLogs.map(x => JSON.stringify(x)).join("\n"));
|
|
|
|
const jobAdds = crawlLogs.filter(x => x.jobPriority !== undefined && x.message.startsWith("Added job for URL "));
|
|
const jobStarts = crawlLogs.filter(x => x.message.startsWith("🐂 Worker taking job"));
|
|
const ttl = [...new Set(crawlLogs.filter(x => x.method === "lockURL" && x.res !== undefined).map(x => x.url))]
|
|
|
|
fs.writeFileSync(crawlId + ".md",
|
|
"```mermaid\nflowchart LR\n "
|
|
+ jobStarts.map(x => `${x.jobId}[${urlFilter(x.url)}]`).join("\n ") + "\n "
|
|
+ jobAdds.map(x => `${x.jobId}[${urlFilter(jobStarts.find(y => y.jobId === x.jobId).url)}] --> ${x.newJobId}[${urlFilter(x.url)}]`).join("\n ")
|
|
+ "\n```\n\nURLs scraped: (" + jobStarts.length + ")\n"
|
|
+ jobStarts.map(x => "- " + x.url).join("\n") + "\n\nURLs tried to lock: (" + ttl.length + ")\n"
|
|
+ ttl.map(x => "- " + x + " ("+ crawlLogs.filter(y => y.method === "lockURL" && y.res !== undefined && y.url === x).length + "; " + crawlLogs.filter(y => y.method === "lockURL" && y.res === true && y.url === x).length + ")").join("\n")
|
|
);
|
|
}
|