Feat/opamp managed otel collector all deployments (#3774)

* chore: get  working and add config for opamp managed collector

* chore: add config for opamp managed collector to pkg/query-service/tests/test-deploy/

* chore: add config for opamp managed collector to docker-swarm deployment

* chore: update signoz-otel-collector image version & specify --copy-path

* chore: retain root user comment in docker-compose-core.yaml

* chore: update collector image tag in swarm conf and add qs dependencies

* chore: some more cleanup

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -197,15 +197,17 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.79.8
+    image: signoz/signoz-otel-collector:0.79.9
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
+        "--manager-config=/etc/manager-config.yaml",
         "--feature-gates=-pkg.translator.prometheus.NormalizeName"
       ]
     user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}}
@@ -227,10 +229,12 @@ services:
       mode: global
       restart_policy:
         condition: on-failure
-    <<: *clickhouse-depend
+    depends_on:
+      - clickhouse
+      - query-service
 
   otel-collector-metrics:
-    image: signoz/signoz-otel-collector:0.79.8
+    image: signoz/signoz-otel-collector:0.79.9
     command:
       [
         "--config=/etc/otel-collector-metrics-config.yaml",

deploy/docker-swarm/clickhouse-setup/otel-collector-opamp-config.yaml

@@ -0,0 +1 @@
+server_endpoint: ws://query-service:4320/v1/opamp

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -1,6 +1,23 @@
 version: "2.4"
 
 services:
+  zookeeper-1:
+    image: bitnami/zookeeper:3.7.1
+    container_name: signoz-zookeeper-1
+    hostname: zookeeper-1
+    user: root
+    ports:
+      - "2181:2181"
+      - "2888:2888"
+      - "3888:3888"
+    volumes:
+      - ./data/zookeeper-1:/bitnami/zookeeper
+    environment:
+      - ZOO_SERVER_ID=1
+      # - ZOO_SERVERS=0.0.0.0:2888:3888,zookeeper-2:2888:3888,zookeeper-3:2888:3888
+      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOO_AUTOPURGE_INTERVAL=1
+
   clickhouse:
     image: clickhouse/clickhouse-server:23.7.3-alpine
     container_name: signoz-clickhouse
@@ -11,8 +28,11 @@ services:
     volumes:
       - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
       - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+      - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+      - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
       # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
       - ./data/clickhouse/:/var/lib/clickhouse/
+      - ./user_scripts:/var/lib/clickhouse/user_scripts/
     restart: on-failure
     logging:
       options:
@@ -48,15 +68,18 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.79.8
+    image: signoz/signoz-otel-collector:0.79.9
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
+        "--manager-config=/etc/manager-config.yaml",
+        "--copy-path=/var/tmp/collector-config.yaml",
         "--feature-gates=-pkg.translator.prometheus.NormalizeName"
       ]
     # user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
     ports:
@@ -75,6 +98,8 @@ services:
     depends_on:
       clickhouse:
         condition: service_healthy
+      query-service:
+        condition: service_healthy
 
   otel-collector-metrics:
     container_name: signoz-otel-collector-metrics

deploy/docker/clickhouse-setup/docker-compose-local.yaml

@@ -4,8 +4,8 @@ services:
   query-service:
     hostname: query-service
     build:
-      context: "../../../pkg/query-service"
+      context: "../../../"
-      dockerfile: "./Dockerfile"
+      dockerfile: "./pkg/query-service/Dockerfile"
       args:
         LDFLAGS: ""
         TARGETPLATFORM: "${GOOS}/${GOARCH}"

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -213,12 +213,13 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.8}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.9}
     container_name: signoz-otel-collector
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
         "--manager-config=/etc/manager-config.yaml",
+        "--copy-path=/var/tmp/collector-config.yaml",
         "--feature-gates=-pkg.translator.prometheus.NormalizeName"
       ]
     user: root # required for reading docker container logs
@@ -243,7 +244,11 @@ services:
       # - "55678:55678"   # OpenCensus receiver
       # - "55679:55679"   # zPages extension
     restart: on-failure
-    <<: *clickhouse-depend
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+      query-service:
+        condition: service_healthy
 
   otel-collector-metrics:
     image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.8}

pkg/query-service/tests/test-deploy/docker-compose.yaml

@@ -190,16 +190,19 @@ services:
     <<: *clickhouse-depends
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.79.8
+    image: signoz/signoz-otel-collector:0.79.9
     container_name: signoz-otel-collector
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
+        "--manager-config=/etc/manager-config.yaml",
+        "--copy-path=/var/tmp/collector-config.yaml",
         "--feature-gates=-pkg.translator.prometheus.NormalizeName"
       ]
     user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
@@ -218,7 +221,12 @@ services:
       # - "55678:55678"   # OpenCensus receiver
       # - "55679:55679"   # zPages extension
     restart: on-failure
-    <<: *clickhouse-depends
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+      query-service:
+        condition: service_healthy
+
 
   otel-collector-metrics:
     image: signoz/signoz-otel-collector:0.79.8

pkg/query-service/tests/test-deploy/otel-collector-opamp-config.yaml

@@ -0,0 +1 @@
+server_endpoint: ws://query-service:4320/v1/opamp