From b6afc9315bbab084565a1e3a512270003e91f7ec Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Wed, 10 Aug 2022 23:00:05 +0530 Subject: [PATCH] clickhouse logs exporter added to deployment file (#1500) * clickhouse logs exporter added to deployment file * updated to latest otel collector --- .../clickhouse-setup/docker-compose.yaml | 6 ++++-- .../otel-collector-config.yaml | 18 +++++++++++++++++ .../clickhouse-setup/docker-compose.yaml | 6 ++++-- .../otel-collector-config.yaml | 20 +++++++++++++++++++ .../tests/test-deploy/docker-compose.yaml | 4 ++-- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 9fbf9e0632..31f2182d66 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -81,14 +81,16 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/otelcontribcol:0.45.1-1.3 + image: signoz-otel-collector:0.55.0-rc.3 command: ["--config=/etc/otel-collector-config.yaml"] + # user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml ports: # - "1777:1777" # pprof extension - "4317:4317" # OTLP gRPC receiver - "4318:4318" # OTLP HTTP receiver + - "4319:4319" # OTLP logs receiver # - "8888:8888" # OtelCollector internal metrics # - "8889:8889" # signoz spanmetrics exposed by the agent # - "9411:9411" # Zipkin port @@ -111,7 +113,7 @@ services: - clickhouse otel-collector-metrics: - image: signoz/otelcontribcol:0.45.1-1.3 + image: signoz-otel-collector:0.55.0-rc.3 command: ["--config=/etc/otel-collector-metrics-config.yaml"] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml index 61292c5781..43e529d836 100644 --- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml @@ -30,6 +30,10 @@ receivers: disk: {} filesystem: {} network: {} + otlp/logs: + protocols: + grpc: + endpoint: 0.0.0.0:4319 processors: batch: @@ -76,6 +80,16 @@ exporters: prometheus: endpoint: 0.0.0.0:8889 # logging: {} + clickhouselogsexporter: + dsn: tcp://clickhouse:9000/ + timeout: 5s + sending_queue: + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s extensions: health_check: @@ -106,3 +120,7 @@ service: metrics/spanmetrics: receivers: [otlp/spanmetrics] exporters: [prometheus] + logs: + receivers: [otlp/logs] + processors: [batch] + exporters: [clickhouselogsexporter] diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 2892cb89a2..1cad876503 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -78,8 +78,9 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/otelcontribcol:0.45.1-1.3 + image: signoz/signoz-otel-collector:0.55.0-rc.3 command: ["--config=/etc/otel-collector-config.yaml"] + # user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml environment: @@ -88,6 +89,7 @@ services: # - "1777:1777" # pprof extension - "4317:4317" # OTLP gRPC receiver - "4318:4318" # OTLP HTTP receiver + - "4319:4319" # OTLP logs receiver # - "8888:8888" # OtelCollector internal metrics # - "8889:8889" # signoz spanmetrics exposed by the agent # - "9411:9411" # Zipkin port @@ -103,7 +105,7 @@ services: condition: service_healthy otel-collector-metrics: - image: signoz/otelcontribcol:0.45.1-1.3 + image: signoz/signoz-otel-collector:0.55.0-rc.3 command: ["--config=/etc/otel-collector-metrics-config.yaml"] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml index 0717cf4c45..265a943e4c 100644 --- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml @@ -31,6 +31,11 @@ receivers: filesystem: {} network: {} + otlp/logs: + protocols: + grpc: + endpoint: 0.0.0.0:4319 + processors: batch: send_batch_size: 10000 @@ -85,6 +90,17 @@ exporters: endpoint: 0.0.0.0:8889 # logging: {} + clickhouselogsexporter: + dsn: tcp://clickhouse:9000/ + timeout: 5s + sending_queue: + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + service: telemetry: metrics: @@ -109,3 +125,7 @@ service: metrics/spanmetrics: receivers: [otlp/spanmetrics] exporters: [prometheus] + logs: + receivers: [otlp/logs] + processors: [batch] + exporters: [clickhouselogsexporter] \ No newline at end of file diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index d6d9b9c1eb..dd1e08f311 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -59,7 +59,7 @@ services: condition: service_healthy otel-collector: - image: signoz/otelcontribcol:0.45.1-1.3 + image: signoz-otel-collector:0.55.0-rc.3 command: ["--config=/etc/otel-collector-config.yaml"] volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml @@ -74,7 +74,7 @@ services: condition: service_healthy otel-collector-metrics: - image: signoz/otelcontribcol:0.45.1-1.3 + image: signoz-otel-collector:0.55.0-rc.3 command: ["--config=/etc/otel-collector-metrics-config.yaml"] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml