chore: remove port expose in docker compose (#5754)

Co-authored-by: Chenhe Gu <guchenhe@gmail.com>

.github/workflows/api-tests.yml

@@ -60,6 +60,9 @@ jobs:
           cp docker/.env.example docker/.env
           cp docker/middleware.env.example docker/middleware.env
 
+      - name: Expose Service Ports
+        run: sh .github/workflows/expose_service_ports.sh
+
       - name: Set up Sandbox
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:

.github/workflows/expose_service_ports.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml
+yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml
+yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
+yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
+yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
+yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
+
+echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs."

docker/.env.example

@@ -585,7 +585,7 @@ NGINX_PROXY_SEND_TIMEOUT=3600s
 # ------------------------------
 # Environment Variables for SSRF Proxy
 # ------------------------------
-HTTP_PORT=3128
+SSRF_HTTP_PORT=3128
 COREDUMP_DIR=/var/spool/squid
 REVERSE_PROXY_PORT=8194
 SANDBOX_HOST=sandbox
@@ -595,4 +595,10 @@ SANDBOX_HOST=sandbox
 # (based on the vector db type, the corresponding docker 
 # compose profile will be used)
 # ------------------------------
-COMPOSE_PROFILES=${VECTOR_STORE:-weaviate}
+COMPOSE_PROFILES=${VECTOR_STORE:-weaviate}
+
+# ------------------------------
+# Docker Compose Service Expose Host Port Configurations
+# ------------------------------
+EXPOSE_NGINX_PORT=80
+EXPOSE_NGINX_SSL_PORT=443

docker/docker-compose.middleware.yaml

@@ -12,7 +12,7 @@ services:
     volumes:
       - ./volumes/db/data:/var/lib/postgresql/data
     ports:
-      - "5432:5432"
+      - "${EXPOSE_POSTGRES_PORT:-5432}:5432"
 
   # The redis cache.
   redis:
@@ -24,32 +24,7 @@ services:
     # Set the redis password when startup redis server.
     command: redis-server --requirepass difyai123456
     ports:
-      - "6379:6379"
+      - "${EXPOSE_REDIS_PORT:-6379}:6379"
-
-  # The Weaviate vector store.
-  weaviate:
-    image: semitechnologies/weaviate:1.19.0
-    restart: always
-    volumes:
-      # Mount the Weaviate data directory to the container.
-      - ./volumes/weaviate:/var/lib/weaviate
-    env_file:
-      - ./middleware.env
-    environment:
-      # The Weaviate configurations
-      # You can refer to the [Weaviate](https://weaviate.io/developers/weaviate/config-refs/env-vars) documentation for more information.
-      PERSISTENCE_DATA_PATH: ${PERSISTENCE_DATA_PATH:-'/var/lib/weaviate'}
-      QUERY_DEFAULTS_LIMIT: ${QUERY_DEFAULTS_LIMIT:-25}
-      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-false}
-      DEFAULT_VECTORIZER_MODULE: ${DEFAULT_VECTORIZER_MODULE:-none}
-      CLUSTER_HOSTNAME: ${CLUSTER_HOSTNAME:-node1}
-      AUTHENTICATION_APIKEY_ENABLED: ${AUTHENTICATION_APIKEY_ENABLED:-true}
-      AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}
-      AUTHENTICATION_APIKEY_USERS: ${AUTHENTICATION_APIKEY_USERS:-hello@dify.ai}
-      AUTHORIZATION_ADMINLIST_ENABLED: ${AUTHORIZATION_ADMINLIST_ENABLED:-true}
-      AUTHORIZATION_ADMINLIST_USERS: ${AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai}
-    ports:
-      - "8080:8080"
 
   # The DifySandbox
   sandbox:
@@ -81,20 +56,47 @@ services:
       - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template
       - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint.sh
     entrypoint: /docker-entrypoint.sh
-    ports:
-      - "3128:3128"
-      - "8194:8194"
     environment:
       # pls clearly modify the squid env vars to fit your network environment.
-      HTTP_PORT: ${HTTP_PORT:-3128}
+      HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
       COREDUMP_DIR: ${COREDUMP_DIR:-/var/spool/squid}
       REVERSE_PROXY_PORT: ${REVERSE_PROXY_PORT:-8194}
       SANDBOX_HOST: ${SANDBOX_HOST:-sandbox}
       SANDBOX_PORT: ${SANDBOX_PORT:-8194}
+    ports:
+      - "${EXPOSE_SSRF_PROXY_PORT:-3128}:${SSRF_HTTP_PORT:-3128}"
+      - "${EXPOSE_SANDBOX_PORT:-8194}:${SANDBOX_PORT:-8194}"
     networks:
       - ssrf_proxy_network
       - default
 
+  # The Weaviate vector store.
+  weaviate:
+    image: semitechnologies/weaviate:1.19.0
+    profiles:
+      - weaviate
+    restart: always
+    volumes:
+      # Mount the Weaviate data directory to the container.
+      - ./volumes/weaviate:/var/lib/weaviate
+    env_file:
+      - ./middleware.env
+    environment:
+      # The Weaviate configurations
+      # You can refer to the [Weaviate](https://weaviate.io/developers/weaviate/config-refs/env-vars) documentation for more information.
+      PERSISTENCE_DATA_PATH: ${PERSISTENCE_DATA_PATH:-'/var/lib/weaviate'}
+      QUERY_DEFAULTS_LIMIT: ${QUERY_DEFAULTS_LIMIT:-25}
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-false}
+      DEFAULT_VECTORIZER_MODULE: ${DEFAULT_VECTORIZER_MODULE:-none}
+      CLUSTER_HOSTNAME: ${CLUSTER_HOSTNAME:-node1}
+      AUTHENTICATION_APIKEY_ENABLED: ${AUTHENTICATION_APIKEY_ENABLED:-true}
+      AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih}
+      AUTHENTICATION_APIKEY_USERS: ${AUTHENTICATION_APIKEY_USERS:-hello@dify.ai}
+      AUTHORIZATION_ADMINLIST_ENABLED: ${AUTHORIZATION_ADMINLIST_ENABLED:-true}
+      AUTHORIZATION_ADMINLIST_USERS: ${AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai}
+    ports:
+      - "${EXPOSE_WEAVIATE_PORT:-8080}:8080"
+
 networks:
   # create a network between sandbox, api and ssrf_proxy, and can not access outside.
   ssrf_proxy_network:

docker/docker-compose.yaml

@@ -272,9 +272,6 @@ services:
     volumes:
       # Mount the storage directory to the container, for storing user files.
       - ./volumes/app/storage:/app/api/storage
-    # uncomment to expose dify-api port to host
-    # ports:
-    #   - "5001:5001"
     networks:
       - ssrf_proxy_network
       - default
@@ -307,9 +304,6 @@ services:
       CONSOLE_API_URL: ${CONSOLE_API_URL:-}
       APP_API_URL: ${APP_API_URL:-}
       SENTRY_DSN: ${SENTRY_DSN:-}
-    # uncomment to expose dify-web port to host
-    # ports:
-    #   - "3000:3000"
 
   # The postgres database.
   db:
@@ -322,9 +316,6 @@ services:
       PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata}
     volumes:
       - ./volumes/db/data:/var/lib/postgresql/data
-    # uncomment to expose db(postgresql) port to host
-    # ports:
-    #   - "5432:5432"
     healthcheck:
       test: [ "CMD", "pg_isready" ]
       interval: 1s
@@ -342,9 +333,6 @@ services:
     command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456}
     healthcheck:
       test: [ "CMD", "redis-cli", "ping" ]
-    # uncomment to expose redis port to host
-    # ports:
-    #   - "6379:6379"
 
   # The DifySandbox
   sandbox:
@@ -378,7 +366,7 @@ services:
     entrypoint: ["sh", "-c", "chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh"]
     environment:
       # pls clearly modify the squid env vars to fit your network environment.
-      HTTP_PORT: ${HTTP_PORT:-3128}
+      HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
       COREDUMP_DIR: ${COREDUMP_DIR:-/var/spool/squid}
       REVERSE_PROXY_PORT: ${REVERSE_PROXY_PORT:-8194}
       SANDBOX_HOST: ${SANDBOX_HOST:-sandbox}
@@ -417,8 +405,8 @@ services:
       - api
       - web
     ports:
-      - "${NGINX_PORT:-80}:80"
+      - "${EXPOSE_NGINX_PORT:-80}:80"
-      - "${NGINX_SSL_PORT:-443}:443"
+      - "${EXPOSE_NGINX_SSL_PORT:-443}:443"
 
   # The Weaviate vector store.
   weaviate:
@@ -442,10 +430,6 @@ services:
       AUTHENTICATION_APIKEY_USERS: ${AUTHENTICATION_APIKEY_USERS:-hello@dify.ai}
       AUTHORIZATION_ADMINLIST_ENABLED: ${AUTHORIZATION_ADMINLIST_ENABLED:-true}
       AUTHORIZATION_ADMINLIST_USERS: ${AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai}
-    # uncomment to expose weaviate port to host
-    ports:
-      - "8080:8080"
-
 
   # Qdrant vector store.
   # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.)
@@ -458,10 +442,6 @@ services:
       - ./volumes/qdrant:/qdrant/storage
     environment:
       QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456}
-    # uncomment to expose qdrant port to host
-    ports:
-      - "6333:6333"
-      - "6334:6334"
 
   # The pgvector vector database.
   pgvector:
@@ -479,9 +459,6 @@ services:
       PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata}
     volumes:
       - ./volumes/pgvector/data:/var/lib/postgresql/data
-    # uncomment to expose db(postgresql) port to host
-    ports:
-      - "5433:5432"
     healthcheck:
       test: [ "CMD", "pg_isready" ]
       interval: 1s
@@ -504,9 +481,6 @@ services:
       PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata}
     volumes:
       - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data
-    # uncomment to expose db(postgresql) port to host
-    ports:
-      - "5431:5432"
     healthcheck:
       test: [ "CMD", "pg_isready" ]
       interval: 1s
@@ -525,16 +499,12 @@ services:
       CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-difyai123456}
       CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider}
       IS_PERSISTENT: ${IS_PERSISTENT:-TRUE}
-    ports:
-      - "8000:8000"
 
   oracle:
     image: container-registry.oracle.com/database/free:latest
     profiles:
       - oracle
     restart: always
-    ports:
-      - 1521:1521
     volumes:
       - type: volume
         source: oradata
@@ -574,9 +544,6 @@ services:
     environment:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
-    ports:
-      - "9001:9001"
-      - "9000:9000"
     volumes:
       - ./volumes/milvus/minio:/minio_data
     command: minio server /minio_data --console-address ":9001"
@@ -606,9 +573,6 @@ services:
       start_period: 90s
       timeout: 20s
       retries: 3
-    ports:
-      - "19530:19530"
-      - "9091:9091"
     depends_on:
       - "etcd"
       - "minio"
@@ -634,9 +598,6 @@ services:
         hard: ${OPENSEARCH_NOFILE_HARD:-65536}
     volumes:
       - ./volumes/opensearch/data:/usr/share/opensearch/data
-    ports:
-      - "9200:9200"
-      - "9600:9600"
     networks:
       - opensearch-net
 
@@ -645,10 +606,6 @@ services:
     image: opensearchproject/opensearch-dashboards:latest
     profiles:
       - opensearch
-    ports:
-      - "5601:5601"
-    expose:
-      - "5601"
     environment:
       OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
     volumes:

docker/middleware.env.example

@@ -40,3 +40,12 @@ AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
 AUTHENTICATION_APIKEY_USERS=hello@dify.ai
 AUTHORIZATION_ADMINLIST_ENABLED=true
 AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai
+
+# ------------------------------
+# Docker Compose Service Expose Host Port Configurations
+# ------------------------------
+EXPOSE_POSTGRES_PORT=5432
+EXPOSE_REDIS_PORT=6379
+EXPOSE_SANDBOX_PORT=8194
+EXPOSE_SSRF_PROXY_PORT=3128
+EXPOSE_WEAVIATE_PORT=8080