AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-14 04:26:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Nick: sitemap only

Browse Source
This commit is contained in:
Nicolas 2024-11-14 17:29:53 -05:00
parent 431e64e752
commit f155449458
4 changed files with 87 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
apps/api/src/controllers/v1/map.ts
View File

@ -1,10 +1,6 @@
import { Response } from "express"; import { Response } from "express";
import { v4 as uuidv4 } from "uuid"; import { v4 as uuidv4 } from "uuid";
import { import { mapRequestSchema, RequestWithAuth, scrapeOptions } from "./types";
mapRequestSchema,
RequestWithAuth,
scrapeOptions,
} from "./types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis"; import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { MapResponse, MapRequest } from "./types"; import { MapResponse, MapRequest } from "./types";
import { configDotenv } from "dotenv"; import { configDotenv } from "dotenv";
@ -46,6 +42,7 @@ export async function mapController(
originUrl: req.body.url, originUrl: req.body.url,
crawlerOptions: { crawlerOptions: {
...req.body, ...req.body,
limit: req.body.sitemapOnly ? 10000000 : limit,
scrapeOptions: undefined, scrapeOptions: undefined,
}, },
scrapeOptions: scrapeOptions.parse({}), scrapeOptions: scrapeOptions.parse({}),
@ -57,6 +54,15 @@ export async function mapController(
const crawler = crawlToCrawler(id, sc); const crawler = crawlToCrawler(id, sc);
// If sitemapOnly is true, only get links from sitemap
if (req.body.sitemapOnly) {
const sitemap = await crawler.tryGetSitemap();
if (sitemap !== null) {
sitemap.forEach((x) => {
links.push(x.url);
});
}
} else {
let urlWithoutWww = req.body.url.replace("www.", ""); let urlWithoutWww = req.body.url.replace("www.", "");
let mapUrl = req.body.search let mapUrl = req.body.search
@ -64,7 +70,9 @@ export async function mapController(
: `site:${req.body.url}`; : `site:${req.body.url}`;
const resultsPerPage = 100; const resultsPerPage = 100;
const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage); const maxPages = Math.ceil(
Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage
);
const cacheKey = `fireEngineMap:${mapUrl}`; const cacheKey = `fireEngineMap:${mapUrl}`;
const cachedResult = null; const cachedResult = null;
@ -82,7 +90,9 @@ export async function mapController(
}); });
}; };
pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1)); pagePromises = Array.from({ length: maxPages }, (_, i) =>
fetchPage(i + 1)
);
allResults = await Promise.all(pagePromises); allResults = await Promise.all(pagePromises);
await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
@ -128,6 +138,8 @@ export async function mapController(
} }
} }
}
// Perform cosine similarity between the search query and the list of links // Perform cosine similarity between the search query and the list of links
if (req.body.search) { if (req.body.search) {
const searchQuery = req.body.search.toLowerCase(); const searchQuery = req.body.search.toLowerCase();

1
apps/api/src/controllers/v1/types.ts
View File

@ -261,6 +261,7 @@ export const mapRequestSchema = crawlerOptions.extend({
includeSubdomains: z.boolean().default(true), includeSubdomains: z.boolean().default(true),
search: z.string().optional(), search: z.string().optional(),
ignoreSitemap: z.boolean().default(false), ignoreSitemap: z.boolean().default(false),
sitemapOnly: z.boolean().default(false),
limit: z.number().min(1).max(5000).default(5000), limit: z.number().min(1).max(5000).default(5000),
}).strict(strictMessage); }).strict(strictMessage);

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "@mendable/firecrawl-js", "name": "@mendable/firecrawl-js",
"version": "1.8.3", "version": "1.8.4",
"description": "JavaScript SDK for Firecrawl API", "description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js", "main": "dist/index.js",
"types": "dist/index.d.ts", "types": "dist/index.d.ts",

1
apps/js-sdk/firecrawl/src/index.ts
View File

@ -221,6 +221,7 @@ export interface MapParams {
search?: string; search?: string;
ignoreSitemap?: boolean; ignoreSitemap?: boolean;
includeSubdomains?: boolean; includeSubdomains?: boolean;
sitemapOnly?: boolean;
limit?: number; limit?: number;
} }