AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-20 12:39:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

further fixes

Browse Source
This commit is contained in:
Gergő Móricz 2025-06-03 16:12:31 +02:00
parent d7fef33224
commit 7426e54e6c
5 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/src/__tests__/queue-concurrency-integration.test.ts
View File

@ -65,7 +65,7 @@ describe("Queue Concurrency Integration", () => {
fastMode: false, fastMode: false,
blockAds: true, blockAds: true,
maxAge: 0, maxAge: 0,
dontStoreInCache: false, storeInCache: true,
}; };
beforeEach(() => { beforeEach(() => {

4
apps/api/src/__tests__/snips/scrape.test.ts
View File

@ -97,8 +97,8 @@ describe("Scrape tests", () => {
}, 30000); }, 30000);
}); });
describe("Index", () => { describe.only("Index", () => {
it.concurrent("caches properly", async () => { it.only("caches properly", async () => {
const id = crypto.randomUUID(); const id = crypto.randomUUID();
const url = "https://firecrawl.dev/?testId=" + id; const url = "https://firecrawl.dev/?testId=" + id;

2
apps/api/src/lib/deep-research/deep-research-service.ts
View File

@ -132,7 +132,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
fastMode: false, fastMode: false,
blockAds: false, blockAds: false,
maxAge: 0, maxAge: 0,
dontStoreInCache: false, storeInCache: true,
__experimental_cache: true, __experimental_cache: true,
}, },
}, logger, costTracking, acuc?.flags ?? null); }, logger, costTracking, acuc?.flags ?? null);

2
apps/api/src/scraper/scrapeURL/engines/index.ts
View File

@ -191,6 +191,7 @@ export const engineOptions: {
skipTlsVerification: true, skipTlsVerification: true,
useFastMode: true, useFastMode: true,
stealthProxy: false, stealthProxy: false,
disableAdblock: false,
}, },
quality: 999, // index should always be tried second ? - MG quality: 999, // index should always be tried second ? - MG
}, },
@ -244,6 +245,7 @@ export const engineOptions: {
skipTlsVerification: true, skipTlsVerification: true,
useFastMode: true, useFastMode: true,
stealthProxy: false, stealthProxy: false,
disableAdblock: false,
}, },
quality: -1, quality: -1,
}, },

2
apps/api/src/scraper/scrapeURL/engines/index/index.ts
View File

@ -6,7 +6,7 @@ import { EngineError, IndexMissError } from "../../error";
import crypto from "crypto"; import crypto from "crypto";
export async function sendDocumentToIndex(meta: Meta, document: Document) { export async function sendDocumentToIndex(meta: Meta, document: Document) {
const shouldCache = !meta.options.dontStoreInCache const shouldCache = meta.options.storeInCache
&& meta.winnerEngine !== "cache" && meta.winnerEngine !== "cache"
&& meta.winnerEngine !== "index" && meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents" && meta.winnerEngine !== "index;documents"