AI/firecrawl
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/mendableai/firecrawl synced 2025-08-12 23:09:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "fix(scrapeURL): better timeToRun distribution"

Browse Source 
This reverts commit 284a6ccedd1baede825571ee933eb7e4f773e2de.
This commit is contained in:
Gergő Móricz 2024-12-16 23:08:20 +01:00
parent 284a6ccedd
commit 7f57c868be
1 changed files with 4 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/scrapeURL/index.ts
View File

@ -203,20 +203,15 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const results: EngineResultsTracker = {};
let result: EngineScrapeResultWithContext | null = null;
let ttrInstanceCount = Math.min(fallbackList.length, 3);
let ttrRatios = new Array(ttrInstanceCount).fill(0).map((_, i) => ttrInstanceCount - i);
let ttrRatioSum = ttrRatios.reduce((a, x) => a + x, 0);
const timeToRun = meta.options.timeout !== undefined
? ttrRatios.map(ratio => Math.round(meta.options.timeout! * ratio / ttrRatioSum)).map(ratio => isNaN(ratio) ? undefined : ratio)
: [undefined]
? Math.round(meta.options.timeout / Math.min(fallbackList.length, 3))
: undefined
for (const i in fallbackList) {
const { engine, unsupportedFeatures } = fallbackList[i];
for (const { engine, unsupportedFeatures } of fallbackList) {
const startedAt = Date.now();
try {
meta.logger.info("Scraping via " + engine + "...");
const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun[i] ?? timeToRun.slice(-1)[0]);
const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);
if (_engineResult.markdown === undefined) {
// Some engines emit Markdown directly.
_engineResult.markdown = await parseMarkdown(_engineResult.html);