GitHub-Proxy/jina_reader
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/jina-ai/reader.git synced 2025-08-19 22:09:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: suspend data crunching

Browse Source
This commit is contained in:
yanlong.wang 2024-10-21 12:07:14 +08:00
parent cf32ab4fa7
commit d6ad9e75d6
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
1 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
backend/functions/src/cloud-functions/data-crunching.ts
View File

@ -5,7 +5,7 @@ import {
} from 'civkit'; } from 'civkit';
import { singleton } from 'tsyringe'; import { singleton } from 'tsyringe';
import { import {
CloudScheduleV2, CloudTaskV2, // CloudScheduleV2, CloudTaskV2,
FirebaseStorageBucketControl, Logger, Param, TempFileManager FirebaseStorageBucketControl, Logger, Param, TempFileManager
} from '../shared'; } from '../shared';
import _ from 'lodash'; import _ from 'lodash';
@ -48,24 +48,24 @@ export class DataCrunchingHost extends RPCHost {
this.emit('ready'); this.emit('ready');
} }
@CloudTaskV2({ // @CloudTaskV2({
runtime: { // runtime: {
cpu: 2, // cpu: 2,
memory: '4GiB', // memory: '4GiB',
timeoutSeconds: 3600, // timeoutSeconds: 3600,
concurrency: 2, // concurrency: 2,
maxInstances: 200, // maxInstances: 200,
retryConfig: { // retryConfig: {
maxAttempts: 3, // maxAttempts: 3,
minBackoffSeconds: 60, // minBackoffSeconds: 60,
}, // },
rateLimits: { // rateLimits: {
maxConcurrentDispatches: 150, // maxConcurrentDispatches: 150,
maxDispatchesPerSecond: 2, // maxDispatchesPerSecond: 2,
}, // },
}, // },
tags: ['DataCrunching'], // tags: ['DataCrunching'],
}) // })
async crunchPageCacheWorker( async crunchPageCacheWorker(
@Param('date') date: string, @Param('date') date: string,
@Param('offset', { default: 0 }) offset: number @Param('offset', { default: 0 }) offset: number
@ -87,18 +87,18 @@ export class DataCrunchingHost extends RPCHost {
return true; return true;
} }
@CloudScheduleV2('2 0 * * *', { // @CloudScheduleV2('2 0 * * *', {
name: 'crunchPageCacheEveryday', // name: 'crunchPageCacheEveryday',
runtime: { // runtime: {
cpu: 2, // cpu: 2,
memory: '4GiB', // memory: '4GiB',
timeoutSeconds: 1800, // timeoutSeconds: 1800,
timeZone: 'UTC', // timeZone: 'UTC',
retryCount: 3, // retryCount: 3,
minBackoffSeconds: 60, // minBackoffSeconds: 60,
}, // },
tags: ['DataCrunching'], // tags: ['DataCrunching'],
}) // })
async dispatchPageCacheCrunching() { async dispatchPageCacheCrunching() {
for await (const { fileName, date, offset } of this.iterPageCacheChunks()) { for await (const { fileName, date, offset } of this.iterPageCacheChunks()) {
this.logger.info(`Dispatching ${fileName}...`); this.logger.info(`Dispatching ${fileName}...`);