Bump version of clickhouse to 22.8.8 LTS and deploy file changes (#1711)

* chore: 🔥 remove docker-compose-prod.yaml as redundant and update Makefile
* chore: 🔧 scrape otel-collector internal metrics in same container and related changes
* chore: 📌 Bump version of clickhouse to 22.8.8 LTS

Signed-off-by: Prashant Shahi <prashant@signoz.io>
Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>

Makefile

@@ -120,14 +120,10 @@ down-local:
 	down -v
 
 run-x86:
-	@docker-compose -f \
+	@docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml up --build -d
-	$(STANDALONE_DIRECTORY)/docker-compose-core.yaml -f $(STANDALONE_DIRECTORY)/docker-compose-prod.yaml \
-	up --build -d
 
 down-x86:
-	@docker-compose -f \
+	@docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml down -v
-	$(STANDALONE_DIRECTORY)/docker-compose-core.yaml -f $(STANDALONE_DIRECTORY)/docker-compose-prod.yaml \
-	down -v
 
 clear-standalone-data:
 	@docker run --rm -v "$(PWD)/$(STANDALONE_DIRECTORY)/data:/pwd" busybox \

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:22.4.5-alpine
+    image: clickhouse/clickhouse-server:22.8.8-alpine
     # ports:
     #   - "9000:9000"
     #   - "8123:8123"

deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml

@@ -55,6 +55,16 @@ receivers:
       disk: {}
       filesystem: {}
       network: {}
+  prometheus:
+    config:
+      global:
+        scrape_interval: 30s
+      scrape_configs:
+        # otel-collector internal metrics
+        - job_name: otel-collector
+          static_configs:
+          - targets:
+            - localhost:8888
 
 processors:
   batch:
@@ -134,8 +144,8 @@ service:
       receivers: [otlp]
       processors: [batch]
       exporters: [clickhousemetricswrite]
-    metrics/hostmetrics:
+    metrics/generic:
-      receivers: [hostmetrics]
+      receivers: [hostmetrics, prometheus]
       processors: [resourcedetection, batch]
       exporters: [clickhousemetricswrite]
     metrics/spanmetrics:

deploy/docker-swarm/clickhouse-setup/otel-collector-metrics-config.yaml

@@ -2,27 +2,19 @@ receivers:
   prometheus:
     config:
       scrape_configs:
-        # otel-collector internal metrics
-        - job_name: "otel-collector"
-          scrape_interval: 60s
-          dns_sd_configs:
-            - names:
-                - 'tasks.otel-collector'
-              type: 'A'
-              port: 8888
         # otel-collector-metrics internal metrics
-        - job_name: "otel-collector-metrics"
+        - job_name: otel-collector-metrics
           scrape_interval: 60s
           static_configs:
             - targets:
               - localhost:8888
         # SigNoz span metrics
-        - job_name: "signozspanmetrics-collector"
+        - job_name: signozspanmetrics-collector
           scrape_interval: 60s
           dns_sd_configs:
             - names:
-                - 'tasks.otel-collector'
+                - tasks.otel-collector
-              type: 'A'
+              type: A
               port: 8889
 
 processors:

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -2,7 +2,7 @@ version: "2.4"
 
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:22.4.5-alpine
+    image: clickhouse/clickhouse-server:22.8.8-alpine
     container_name: clickhouse
     # ports:
     # - "9000:9000"

deploy/docker/clickhouse-setup/docker-compose-prod.yaml

@@ -1,44 +0,0 @@
-version: "2.4"
-
-services:
-  query-service:
-    image: signoz/query-service:0.11.2
-    container_name: query-service
-    command: ["-config=/root/config/prometheus.yml"]
-    # ports:
-    #   - "6060:6060"     # pprof port
-    #   - "8080:8080"     # query-service port
-    volumes:
-      - ./prometheus.yml:/root/config/prometheus.yml
-      - ../dashboards:/root/config/dashboards
-      - ./data/signoz/:/var/lib/signoz/
-    environment:
-      - ClickHouseUrl=tcp://clickhouse:9000/?database=signoz_traces
-      - ALERTMANAGER_API_PREFIX=http://alertmanager:9093/api/
-      - SIGNOZ_LOCAL_DB_PATH=/var/lib/signoz/signoz.db
-      - DASHBOARDS_PATH=/root/config/dashboards
-      - STORAGE=clickhouse
-      - GODEBUG=netdns=go
-      - TELEMETRY_ENABLED=true
-      - DEPLOYMENT_TYPE=docker-standalone-amd
-    restart: on-failure
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/version"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
-    depends_on:
-      clickhouse:
-        condition: service_healthy
-
-  frontend:
-    image: signoz/frontend:0.11.2
-    container_name: frontend
-    restart: on-failure
-    depends_on:
-      - alertmanager
-      - query-service
-    ports:
-      - "3301:3301"
-    volumes:
-      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "2.4"
 
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:22.4.5-alpine
+    image: clickhouse/clickhouse-server:22.8.8-alpine
     # ports:
     # - "9000:9000"
     # - "8123:8123"

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -55,6 +55,16 @@ receivers:
       disk: {}
       filesystem: {}
       network: {}
+  prometheus:
+    config:
+      global:
+        scrape_interval: 30s
+      scrape_configs:
+        # otel-collector internal metrics
+        - job_name: otel-collector
+          static_configs:
+          - targets:
+            - localhost:8888
 
 processors:
   batch:
@@ -138,8 +148,8 @@ service:
       receivers: [otlp]
       processors: [batch]
       exporters: [clickhousemetricswrite]
-    metrics/hostmetrics:
+    metrics/generic:
-      receivers: [hostmetrics]
+      receivers: [hostmetrics, prometheus]
       processors: [resourcedetection, batch]
       exporters: [clickhousemetricswrite]
     metrics/spanmetrics:

deploy/docker/clickhouse-setup/otel-collector-metrics-config.yaml

@@ -6,20 +6,14 @@ receivers:
   prometheus:
     config:
       scrape_configs:
-        # otel-collector internal metrics
-        - job_name: "otel-collector"
-          scrape_interval: 60s
-          static_configs:
-            - targets:
-              - otel-collector:8888
         # otel-collector-metrics internal metrics
-        - job_name: "otel-collector-metrics"
+        - job_name: otel-collector-metrics
           scrape_interval: 60s
           static_configs:
             - targets:
               - localhost:8888
         # SigNoz span metrics
-        - job_name: "signozspanmetrics-collector"
+        - job_name: signozspanmetrics-collector
           scrape_interval: 60s
           static_configs:
             - targets:

pkg/query-service/tests/test-deploy/docker-compose.yaml

@@ -2,7 +2,7 @@ version: "2.4"
 
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:22.4.5-alpine
+    image: clickhouse/clickhouse-server:22.8.8-alpine
     tty: true
     volumes:
       - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml

pkg/query-service/tests/test-deploy/otel-collector-config.yaml

@@ -55,6 +55,17 @@ receivers:
       disk: {}
       filesystem: {}
       network: {}
+  prometheus:
+    config:
+      global:
+        scrape_interval: 30s
+      scrape_configs:
+        # otel-collector internal metrics
+        - job_name: otel-collector
+          scrape_interval: 60s
+          static_configs:
+            - targets:
+              - otel-collector:8888
 
 processors:
   batch:
@@ -126,8 +137,8 @@ service:
       receivers: [otlp]
       processors: [batch]
       exporters: [clickhousemetricswrite]
-    metrics/hostmetrics:
+    metrics/generic:
-      receivers: [hostmetrics]
+      receivers: [hostmetrics, prometheus]
       processors: [resourcedetection, batch]
       exporters: [clickhousemetricswrite]
     metrics/spanmetrics:

pkg/query-service/tests/test-deploy/otel-collector-metrics-config.yaml

@@ -6,20 +6,14 @@ receivers:
   prometheus:
     config:
       scrape_configs:
-        # otel-collector internal metrics
-        - job_name: "otel-collector"
-          scrape_interval: 60s
-          static_configs:
-            - targets:
-              - otel-collector:8888
         # otel-collector-metrics internal metrics
-        - job_name: "otel-collector-metrics"
+        - job_name: otel-collector-metrics
           scrape_interval: 60s
           static_configs:
             - targets:
               - localhost:8888
         # SigNoz span metrics
-        - job_name: "signozspanmetrics-collector"
+        - job_name: signozspanmetrics-collector
           scrape_interval: 60s
           static_configs:
             - targets: