AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-16 19:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(index): integrate into map

Browse Source
This commit is contained in:
Gergő Móricz 2025-05-28 14:47:38 +02:00
parent 550d6bf7bd
commit d904ec676f
2 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
apps/api/src/controllers/v1/map.ts
View File

@ -25,6 +25,7 @@ import { logger } from "../../lib/logger";
import Redis from "ioredis";
import { querySitemapIndex } from "../../scraper/WebScraper/sitemap-index";
import { getIndexQueue } from "../../services/queue-service";
import { hashURL, index_supabase_service, normalizeURLForIndex } from "../../services/index";
configDotenv();
const redis = new Redis(process.env.REDIS_URL!);
@ -165,11 +166,23 @@ export async function getMapResults({
}
// Parallelize sitemap index query with search results
const [sitemapIndexResult, ...searchResults] = await Promise.all([
const [sitemapIndexResult, { data: indexResults, error: indexError }, ...searchResults] = await Promise.all([
querySitemapIndex(url, abort),
index_supabase_service
.from("index")
.select("resolved_url")
.overlaps("url_splits_hash", [await hashURL(normalizeURLForIndex(url))])
.gte("created_at", new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString())
.limit(limit),
...(cachedResult ? [] : pagePromises),
]);
if (indexError) {
logger.warn("Error querying index", { error: indexError });
} else if (indexResults.length > 0) {
links.push(...indexResults.map((x) => x.resolved_url));
}
const twoDaysAgo = new Date();
twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);

3
apps/api/src/scraper/scrapeURL/transformers/index.ts
View File

@ -11,7 +11,8 @@ import { saveToCache } from "./cache";
import { performAgent } from "./agent";
import { deriveDiff } from "./diff";
import { sendDocumentToIndex, useIndex } from "../engines/index/index";
import { useIndex } from "../../../services/index";
import { sendDocumentToIndex } from "../engines/index/index";
export type Transformer = (
meta: Meta,