From 63b503a9fb20e373bcf1d52ae941f95a7c24b6fd Mon Sep 17 00:00:00 2001 From: Raj Kamal Singh <1133322+rkssisodiya@users.noreply.github.com> Date: Thu, 19 Oct 2023 20:01:58 +0530 Subject: [PATCH] Feat/opamp managed otel collector all deployments (#3774) * chore: get working and add config for opamp managed collector * chore: add config for opamp managed collector to pkg/query-service/tests/test-deploy/ * chore: add config for opamp managed collector to docker-swarm deployment * chore: update signoz-otel-collector image version & specify --copy-path * chore: retain root user comment in docker-compose-core.yaml * chore: update collector image tag in swarm conf and add qs dependencies * chore: some more cleanup --- .../clickhouse-setup/docker-compose.yaml | 10 ++++--- .../otel-collector-opamp-config.yaml | 1 + .../clickhouse-setup/docker-compose-core.yaml | 27 ++++++++++++++++++- .../docker-compose-local.yaml | 4 +-- .../clickhouse-setup/docker-compose.yaml | 9 +++++-- .../tests/test-deploy/docker-compose.yaml | 12 +++++++-- .../otel-collector-opamp-config.yaml | 1 + 7 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 deploy/docker-swarm/clickhouse-setup/otel-collector-opamp-config.yaml create mode 100644 pkg/query-service/tests/test-deploy/otel-collector-opamp-config.yaml diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 378d47aebd..2516d96728 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -197,15 +197,17 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/signoz-otel-collector:0.79.8 + image: signoz/signoz-otel-collector:0.79.9 command: [ "--config=/etc/otel-collector-config.yaml", + "--manager-config=/etc/manager-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName" ] user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml - /var/lib/docker/containers:/var/lib/docker/containers:ro environment: - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}} @@ -227,10 +229,12 @@ services: mode: global restart_policy: condition: on-failure - <<: *clickhouse-depend + depends_on: + - clickhouse + - query-service otel-collector-metrics: - image: signoz/signoz-otel-collector:0.79.8 + image: signoz/signoz-otel-collector:0.79.9 command: [ "--config=/etc/otel-collector-metrics-config.yaml", diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-opamp-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-opamp-config.yaml new file mode 100644 index 0000000000..e408b55ef6 --- /dev/null +++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-opamp-config.yaml @@ -0,0 +1 @@ +server_endpoint: ws://query-service:4320/v1/opamp diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml index 4912e3a1b5..6c4edc2827 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml @@ -1,6 +1,23 @@ version: "2.4" services: + zookeeper-1: + image: bitnami/zookeeper:3.7.1 + container_name: signoz-zookeeper-1 + hostname: zookeeper-1 + user: root + ports: + - "2181:2181" + - "2888:2888" + - "3888:3888" + volumes: + - ./data/zookeeper-1:/bitnami/zookeeper + environment: + - ZOO_SERVER_ID=1 + # - ZOO_SERVERS=0.0.0.0:2888:3888,zookeeper-2:2888:3888,zookeeper-3:2888:3888 + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_AUTOPURGE_INTERVAL=1 + clickhouse: image: clickhouse/clickhouse-server:23.7.3-alpine container_name: signoz-clickhouse @@ -11,8 +28,11 @@ services: volumes: - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml + - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml + - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml - ./data/clickhouse/:/var/lib/clickhouse/ + - ./user_scripts:/var/lib/clickhouse/user_scripts/ restart: on-failure logging: options: @@ -48,15 +68,18 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` otel-collector: container_name: signoz-otel-collector - image: signoz/signoz-otel-collector:0.79.8 + image: signoz/signoz-otel-collector:0.79.9 command: [ "--config=/etc/otel-collector-config.yaml", + "--manager-config=/etc/manager-config.yaml", + "--copy-path=/var/tmp/collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName" ] # user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml environment: - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux ports: @@ -75,6 +98,8 @@ services: depends_on: clickhouse: condition: service_healthy + query-service: + condition: service_healthy otel-collector-metrics: container_name: signoz-otel-collector-metrics diff --git a/deploy/docker/clickhouse-setup/docker-compose-local.yaml b/deploy/docker/clickhouse-setup/docker-compose-local.yaml index 78aa72ff75..a92c3dbcd9 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-local.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-local.yaml @@ -4,8 +4,8 @@ services: query-service: hostname: query-service build: - context: "../../../pkg/query-service" - dockerfile: "./Dockerfile" + context: "../../../" + dockerfile: "./pkg/query-service/Dockerfile" args: LDFLAGS: "" TARGETPLATFORM: "${GOOS}/${GOARCH}" diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 163179ffb9..83e3c6812d 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -213,12 +213,13 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.8} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.9} container_name: signoz-otel-collector command: [ "--config=/etc/otel-collector-config.yaml", "--manager-config=/etc/manager-config.yaml", + "--copy-path=/var/tmp/collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName" ] user: root # required for reading docker container logs @@ -243,7 +244,11 @@ services: # - "55678:55678" # OpenCensus receiver # - "55679:55679" # zPages extension restart: on-failure - <<: *clickhouse-depend + depends_on: + clickhouse: + condition: service_healthy + query-service: + condition: service_healthy otel-collector-metrics: image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.8} diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index 6d7bc5160b..df879f4099 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -190,16 +190,19 @@ services: <<: *clickhouse-depends otel-collector: - image: signoz/signoz-otel-collector:0.79.8 + image: signoz/signoz-otel-collector:0.79.9 container_name: signoz-otel-collector command: [ "--config=/etc/otel-collector-config.yaml", + "--manager-config=/etc/manager-config.yaml", + "--copy-path=/var/tmp/collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName" ] user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml - /var/lib/docker/containers:/var/lib/docker/containers:ro environment: - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux @@ -218,7 +221,12 @@ services: # - "55678:55678" # OpenCensus receiver # - "55679:55679" # zPages extension restart: on-failure - <<: *clickhouse-depends + depends_on: + clickhouse: + condition: service_healthy + query-service: + condition: service_healthy + otel-collector-metrics: image: signoz/signoz-otel-collector:0.79.8 diff --git a/pkg/query-service/tests/test-deploy/otel-collector-opamp-config.yaml b/pkg/query-service/tests/test-deploy/otel-collector-opamp-config.yaml new file mode 100644 index 0000000000..e408b55ef6 --- /dev/null +++ b/pkg/query-service/tests/test-deploy/otel-collector-opamp-config.yaml @@ -0,0 +1 @@ +server_endpoint: ws://query-service:4320/v1/opamp