From 6b23308f2604467733f3a74720466b3215c0c511 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 6 Sep 2024 16:02:44 +0800 Subject: [PATCH] Added kibana (#2286) Added kibana to make elastic management easier. PR #1710 did this. PR #1714 revert this. This PR did again and fix some bugs. - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) --- README.md | 2 +- README_ko.md | 2 +- README_zh.md | 2 +- SECURITY.md | 2 +- docker/docker-compose-admin-tool.yml | 6 ++- docker/entrypoint.sh | 3 ++ docker/init-kibana.sh | 75 +++++++++++++++++++++------- 7 files changed, 68 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1f72718e..317c75ce 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ Try our demo at [https://demo.ragflow.io](https://demo.ragflow.io). * Running on http://x.x.x.x:9380 INFO:werkzeug:Press CTRL+C to quit ``` - > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network anomaly` error because, at that moment, your RAGFlow may not be fully initialized. + > If you skip this confirmation step and directly log in to RAGFlow, your browser may prompt a `network abnormal` error because, at that moment, your RAGFlow may not be fully initialized. 5. In your web browser, enter the IP address of your server and log in to RAGFlow. > With the default settings, you only need to enter `http://IP_OF_YOUR_MACHINE` (**sans** port number) as the default HTTP serving port `80` can be omitted when using the default configurations. diff --git a/README_ko.md b/README_ko.md index 2df4c43e..a41833da 100644 --- a/README_ko.md +++ b/README_ko.md @@ -178,7 +178,7 @@ * Running on http://x.x.x.x:9380 INFO:werkzeug:Press CTRL+C to quit ``` - > 만약 확인 단계를 건너뛰고 바로 RAGFlow에 로그인하면, RAGFlow가 완전히 초기화되지 않았기 때문에 브라우저에서 `network anomaly` 오류가 발생할 수 있습니다. + > 만약 확인 단계를 건너뛰고 바로 RAGFlow에 로그인하면, RAGFlow가 완전히 초기화되지 않았기 때문에 브라우저에서 `network abnormal` 오류가 발생할 수 있습니다. 5. 웹 브라우저에 서버의 IP 주소를 입력하고 RAGFlow에 로그인하세요. > 기본 설정을 사용할 경우, `http://IP_OF_YOUR_MACHINE`만 입력하면 됩니다 (포트 번호는 제외). 기본 HTTP 서비스 포트 `80`은 기본 구성으로 사용할 때 생략할 수 있습니다. diff --git a/README_zh.md b/README_zh.md index a78bc5e4..c0b05033 100644 --- a/README_zh.md +++ b/README_zh.md @@ -167,7 +167,7 @@ * Running on http://x.x.x.x:9380 INFO:werkzeug:Press CTRL+C to quit ``` - > 如果您跳过这一步系统确认步骤就登录 RAGFlow，你的浏览器有可能会提示 `network anomaly` 或 `网络异常`，因为 RAGFlow 可能并未完全启动成功。 + > 如果您跳过这一步系统确认步骤就登录 RAGFlow，你的浏览器有可能会提示 `network abnormal` 或 `网络异常`，因为 RAGFlow 可能并未完全启动成功。 5. 在你的浏览器中输入你的服务器对应的 IP 地址并登录 RAGFlow。 > 上面这个例子中，您只需输入 http://IP_OF_YOUR_MACHINE 即可：未改动过配置则无需输入端口（默认的 HTTP 服务端口 80）。 diff --git a/SECURITY.md b/SECURITY.md index 066911d5..3ccc48b6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,7 +18,7 @@ main ### Actual behavior The restricted_loads function at [api/utils/__init__.py#L215](https://github.com/infiniflow/ragflow/blob/main/api/utils/__init__.py#L215) is still vulnerable leading via code execution. -The main reson is that numpy module has a numpy.f2py.diagnose.run_command function directly execute commands, but the restricted_loads function allows users import functions in module numpy. +The main reason is that numpy module has a numpy.f2py.diagnose.run_command function directly execute commands, but the restricted_loads function allows users import functions in module numpy. ### Steps to reproduce diff --git a/docker/docker-compose-admin-tool.yml b/docker/docker-compose-admin-tool.yml index 1d52891a..5a0c0c3d 100644 --- a/docker/docker-compose-admin-tool.yml +++ b/docker/docker-compose-admin-tool.yml @@ -1,3 +1,7 @@ +include: + - path: ./docker-compose.yml + env_file: ./.env + services: kibana: image: kibana:${STACK_VERSION} @@ -12,7 +16,7 @@ services: es01: condition: service_healthy kibana-user-init: - condition: service_completed_successfully + condition: service_completed_successfully networks: - ragflow diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 3f731789..1c2c3bc3 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,5 +1,8 @@ #!/bin/bash +# unset http proxy which maybe set by docker daemon +export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" + /usr/sbin/nginx export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/ diff --git a/docker/init-kibana.sh b/docker/init-kibana.sh index 1e5ca690..f5f4f8c2 100755 --- a/docker/init-kibana.sh +++ b/docker/init-kibana.sh @@ -1,30 +1,67 @@ #!/bin/bash -# 等待 Elasticsearch 啟動 -until curl -u "elastic:${ELASTIC_PASSWORD}" -s http://es01:9200 >/dev/null; do - echo "等待 Elasticsearch 啟動..." - sleep 5 +# unset http proxy which maybe set by docker daemon +export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY="" + +echo "Elasticsearch built-in user: elastic:${ELASTIC_PASSWORD}" + +# Wait Elasticsearch be healthy +while true; do + response=$(curl -s -v -w "\n%{http_code}" -u "elastic:${ELASTIC_PASSWORD}" "http://es01:9200") + exit_code=$? + status=$(echo "$response" | tail -n1) + if [ $exit_code -eq 0 ] && [ "$status" = "200" ]; then + echo "Elasticsearch is healthy" + break + else + echo "Elasticsearch is unhealthy: $exit_code $status" + echo "$response" + sleep 5 + fi done +# Create new role with all privileges to all indices +# https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html#privileges-list-indices +echo "Going to create Elasticsearch role own_indices with all privileges to all indices" +while true; do + response=$(curl -s -v -w "\n%{http_code}" -u "elastic:${ELASTIC_PASSWORD}" -X POST http://es01:9200/_security/role/own_indices -H 'Content-Type: application/json' -d '{"indices": [{"names": ["*"], "privileges": ["all"]}]}') + exit_code=$? + status=$(echo "$response" | tail -n1) + if [ $exit_code -eq 0 ] && [ "$status" = "200" ]; then + echo "Elasticsearch role own_indices created" + break + else + echo "Elasticsearch role own_indices failure: $exit_code $status" + echo "$response" + sleep 5 + fi +done -echo "使用者: elastic:${ELASTIC_PASSWORD}" +echo "Elasticsearch role own_indices:" +curl -u "elastic:${ELASTIC_PASSWORD}" -X GET "http://es01:9200/_security/role/own_indices" +echo "" +PAYLOAD="{\"password\": \"${KIBANA_PASSWORD}\", \"roles\": [\"kibana_admin\", \"kibana_system\", \"own_indices\"], \"full_name\": \"${KIBANA_USER}\", \"email\": \"${KIBANA_USER}@example.com\"}" +echo "Going to create Elasticsearch user ${KIBANA_USER}: ${PAYLOAD}" -PAYLOAD="{ - \"password\" : \"${KIBANA_PASSWORD}\", - \"roles\" : [ \"kibana_admin\",\"kibana_system\" ], - \"full_name\" : \"${KIBANA_USER}\", - \"email\" : \"${KIBANA_USER}@example.com\" -}" -echo "新用戶帳戶: $PAYLOAD" +# Create new user +while true; do + response=$(curl -s -v -w "\n%{http_code}" -u "elastic:${ELASTIC_PASSWORD}" -X POST http://es01:9200/_security/user/${KIBANA_USER} -H "Content-Type: application/json" -d "${PAYLOAD}") + exit_code=$? + status=$(echo "$response" | tail -n1) + if [ $exit_code -eq 0 ] && [ "$status" = "200" ]; then + echo "Elasticsearch user ${KIBANA_USER} created" + break + else + echo "Elasticsearch user ${KIBANA_USER} failure: $exit_code $status" + echo "$response" + sleep 5 + fi +done -# 創建新用戶帳戶 -curl -X POST "http://es01:9200/_security/user/${KIBANA_USER}" \ --u "elastic:${ELASTIC_PASSWORD}" \ --H "Content-Type: application/json" \ --d "$PAYLOAD"s - -echo "新用戶帳戶已創建" +echo "Elasticsearch user ${KIBANA_USER}:" +curl -u "elastic:${ELASTIC_PASSWORD}" -X GET "http://es01:9200/_security/user/${KIBANA_USER}" +echo "" exit 0