AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-16 12:15:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix map endpoint including non-existent URLs in results

Browse Source 
Co-Authored-By: Nicolas Camara <nick@sideguide.dev>
This commit is contained in:
Devin AI 2025-04-08 23:21:25 +00:00
parent ab6fb48e6e
commit 257556f424
2 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
apps/api/src/__tests__/snips/map.test.ts
View File

@ -31,4 +31,23 @@ describe("Map tests", () => {
expect(response.body.success).toBe(true);
expect(response.body.links.some(x => x.match(/^https:\/\/www\.hfea\.gov\.uk\/choose-a-clinic\/clinic-search\/results\/?\?options=\d+$/))).toBe(true);
}, 60000);
it.concurrent("does not include non-existent child URLs in parent results", async () => {
const nonExistentPath = "/non-existent-path-" + Date.now(); // Use timestamp to ensure uniqueness
const childUrl = "https://firecrawl.dev" + nonExistentPath;
await map({
url: childUrl,
useMock: "map-non-existent-child",
});
const parentResponse = await map({
url: "https://firecrawl.dev",
useMock: "map-parent-after-child",
});
expect(parentResponse.statusCode).toBe(200);
expect(parentResponse.body.success).toBe(true);
expect(parentResponse.body.links.some(link => link.includes(nonExistentPath))).toBe(false);
}, 30000);
});

4
apps/api/src/controllers/v1/map.ts
View File

@ -161,7 +161,9 @@ export async function getMapResults({
);
allResults = await Promise.all(pagePromises);
await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours
if (allResults.flat().filter(result => result !== null && result !== undefined).length > 0) {
await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours
}
}
// Parallelize sitemap index query with search results