From 9d6fd26d4b487aec83dc6c15493013bcd08f10c5 Mon Sep 17 00:00:00 2001 From: Yanlong Wang Date: Fri, 11 Apr 2025 22:27:54 +0800 Subject: [PATCH] wip --- .docker/mongo/Dockerfile | 9 ++++++ .docker/mongo/init-repl-set.sh | 46 +++++++++++++++++++++++++++++++ .docker/mongo/random.key | 1 + docker-compose.yml | 50 ++++++++++++++++++++++++++++++++++ src/services/mongodb.ts | 1 + src/services/storage-bucket.ts | 27 ++++++++++++++++++ thinapps-shared | 2 +- 7 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 .docker/mongo/Dockerfile create mode 100755 .docker/mongo/init-repl-set.sh create mode 100644 .docker/mongo/random.key create mode 100644 docker-compose.yml create mode 100644 src/services/storage-bucket.ts diff --git a/.docker/mongo/Dockerfile b/.docker/mongo/Dockerfile new file mode 100644 index 0000000..8be5a9b --- /dev/null +++ b/.docker/mongo/Dockerfile @@ -0,0 +1,9 @@ +FROM mongo:latest + +COPY ./random.key /app/repl.key +COPY ./init-repl-set.sh /docker-entrypoint-initdb.d/init-repl-set.sh +RUN chmod 400 /app/repl.key && chown 999:999 /app/repl.key + +ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 27017 +CMD ["mongod", "--bind_ip_all", "--replSet", "rs0", "--keyFile", "/app/repl.key"] diff --git a/.docker/mongo/init-repl-set.sh b/.docker/mongo/init-repl-set.sh new file mode 100755 index 0000000..6adb056 --- /dev/null +++ b/.docker/mongo/init-repl-set.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copied from https://gist.github.com/zhangyoufu/d1d43ac0fa268cda4dd2dfe55a8c834e + +: "${FORKED:=}" +if [ -z "${FORKED}" ]; then + echo >&2 'mongod for initdb is going to shutdown' + mongod --pidfilepath /tmp/docker-entrypoint-temp-mongod.pid --shutdown + echo >&2 'replica set will be initialized later' + FORKED=1 "${BASH_SOURCE[0]}" & + unset FORKED + mongodHackedArgs=(:) # bypass mongod --shutdown in docker-entrypoint.sh + return +fi + +# FIXME: assume mongod listens on 127.0.0.1:27017 +mongo=( mongosh --host 127.0.0.1 --port 27017 -u "${MONGO_INITDB_ROOT_USERNAME}" -p ${MONGO_INITDB_ROOT_PASSWORD} --quiet ) + +tries=30 +while true; do + sleep 1 + if "${mongo[@]}" --eval 'quit(0)' &> /dev/null; then + # success! + break + fi + (( tries-- )) + if [ "$tries" -le 0 ]; then + echo >&2 + echo >&2 'error: unable to initialize replica set' + echo >&2 + kill -STOP 1 # initdb won't be executed twice, so fail loudly + exit 1 + fi +done + +echo 'about to initialize replica set' +# FIXME: hard-coded replica set name & member host +"${mongo[@]}" <<-EOF + rs.initiate({ + _id: "rs0", + version: 1, + members: [ + { _id: 0, host : "mongo-replset-1.dev.jina.ai:27017" }, + ] + }); +EOF diff --git a/.docker/mongo/random.key b/.docker/mongo/random.key new file mode 100644 index 0000000..26d6274 --- /dev/null +++ b/.docker/mongo/random.key @@ -0,0 +1 @@ +aabbccddeeff123 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f6cd105 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +services: + mongo: + build: + context: ./.docker/mongo + restart: always + ports: + - 27017:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: localdev + MONGO_INITDB_ROOT_PASSWORD: localdev123 + MONGO_INITDB_DATABASE: dev + volumes: + - mongo-data:/data/db + networks: + default: + aliases: + - mongo-replset-1.dev.jina.ai + - mongo.dev.jina.ai + + minio: + image: minio/minio + hostname: minio + ports: + - 9001:9001 + - 9000:9000 + environment: + MINIO_ROOT_USER: minio + MINIO_ROOT_PASSWORD: minio123 + volumes: + - minio-data:/data/minio + command: server /data/minio --console-address ":9001" + healthcheck: + test: + [ + 'CMD', + 'curl', + '-f', + 'http://localhost:9001/minio/health/live' + ] + interval: 30s + timeout: 20s + retries: 3 + networks: + default: + aliases: + - minio.dev.jina.ai + +volumes: + minio-data: + mongo-data: diff --git a/src/services/mongodb.ts b/src/services/mongodb.ts index 10ff0de..f22405d 100644 --- a/src/services/mongodb.ts +++ b/src/services/mongodb.ts @@ -36,6 +36,7 @@ export class MongoDB extends AbstractMongoDB { this.options = { minPoolSize: 2, maxPoolSize: 10, + maxIdleTimeMS: 1000 * 60, }; } if (!this.url) { diff --git a/src/services/storage-bucket.ts b/src/services/storage-bucket.ts new file mode 100644 index 0000000..12da31d --- /dev/null +++ b/src/services/storage-bucket.ts @@ -0,0 +1,27 @@ +import { AbstractObjectStorageService, ObjectStorageOptions } from 'civkit/abstract/object-storage'; +import { GlobalLogger } from './logger'; +import { SecretExposer } from '../shared'; + + +export class StorageBucket extends AbstractObjectStorageService { + + logger = this.globalLogger.child({ service: this.constructor.name }); + options!: ObjectStorageOptions; + + constructor( + protected globalLogger: GlobalLogger, + protected config: SecretExposer, + ) { + super(...arguments); + + this.options = { + bucket: process.env.GCP_STORAGE_BUCKET || `${process.env.GCLOUD_PROJECT}.appspot.com`, + endPoint: 'https://storage.googleapis.com', + region: 'auto', + accessKey: this.config.GCP_STORAGE_ACCESS_KEY, + secretKey: this.config.GCP_STORAGE_SECRET_KEY, + }; + + this.dependsOn(globalLogger); + } +} diff --git a/thinapps-shared b/thinapps-shared index 4552b18..0a59b8f 160000 --- a/thinapps-shared +++ b/thinapps-shared @@ -1 +1 @@ -Subproject commit 4552b18a12f249a8413ba73700bbe0816e9fde0d +Subproject commit 0a59b8f84c03b2099a9785769bedc98573e65847