From ae3d4fece8523f2fd4da65e95415eeb1da367a05 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Mon, 7 Aug 2023 21:17:53 +0530 Subject: [PATCH] feat: add logspout as default docker logs collector (#3225) --- .../clickhouse-setup/docker-compose.yaml | 57 +++++++++--- .../otel-collector-config.yaml | 78 ++++++++++------ .../clickhouse-setup/docker-compose-core.yaml | 46 +++++++--- .../docker-compose-local.yaml | 15 +++- .../clickhouse-setup/docker-compose.yaml | 76 ++++++++++++---- .../otel-collector-config.yaml | 42 ++++----- .../tests/test-deploy/docker-compose.yaml | 88 ++++++++++++++----- .../test-deploy/otel-collector-config.yaml | 42 ++++----- 8 files changed, 301 insertions(+), 143 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index d273e996bf..967a94020d 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -1,6 +1,7 @@ version: "3.9" -x-clickhouse-defaults: &clickhouse-defaults +x-clickhouse-defaults: + &clickhouse-defaults image: clickhouse/clickhouse-server:22.8.8-alpine tty: true deploy: @@ -16,7 +17,14 @@ x-clickhouse-defaults: &clickhouse-defaults max-file: "3" healthcheck: # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8123/ping" + ] interval: 30s timeout: 5s retries: 3 @@ -26,7 +34,8 @@ x-clickhouse-defaults: &clickhouse-defaults soft: 262144 hard: 262144 -x-clickhouse-depend: &clickhouse-depend +x-clickhouse-depend: + &clickhouse-depend depends_on: - clickhouse # - clickhouse-2 @@ -138,7 +147,7 @@ services: query-service: image: signoz/query-service:0.25.3 - command: ["-config=/root/config/prometheus.yml"] + command: [ "-config=/root/config/prometheus.yml" ] # ports: # - "6060:6060" # pprof port # - "8080:8080" # query-service port @@ -156,7 +165,14 @@ services: - TELEMETRY_ENABLED=true - DEPLOYMENT_TYPE=docker-swarm healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8080/api/v1/health" + ] interval: 30s timeout: 5s retries: 3 @@ -180,7 +196,11 @@ services: otel-collector: image: signoz/signoz-otel-collector:0.79.4 - command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + command: + [ + "--config=/etc/otel-collector-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml @@ -191,8 +211,8 @@ services: - LOW_CARDINAL_EXCEPTION_GROUPING=false ports: # - "1777:1777" # pprof extension - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP receiver + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver # - "8888:8888" # OtelCollector internal metrics # - "8889:8889" # signoz spanmetrics exposed by the agent # - "9411:9411" # Zipkin port @@ -209,7 +229,11 @@ services: otel-collector-metrics: image: signoz/signoz-otel-collector:0.79.4 - command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + command: + [ + "--config=/etc/otel-collector-metrics-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml # ports: @@ -222,9 +246,22 @@ services: condition: on-failure <<: *clickhouse-depend + logspout: + image: "gliderlabs/logspout:v3.2.14" + volumes: + - /etc/hostname:/etc/host_hostname:ro + - /var/run/docker.sock:/var/run/docker.sock + command: syslog+tcp://otel-collector:2255 + depends_on: + - otel-collector + deploy: + mode: global + restart_policy: + condition: on-failure + hotrod: image: jaegertracing/example-hotrod:1.30 - command: ["all"] + command: [ "all" ] environment: - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces logging: diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml index e755936e8b..74df932c0f 100644 --- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml @@ -1,29 +1,21 @@ receivers: - filelog/dockercontainers: - include: [ "/var/lib/docker/containers/*/*.log" ] - start_at: end - include_file_path: true - include_file_name: false + tcplog/docker: + listen_address: "0.0.0.0:2255" operators: - - type: json_parser - id: parser-docker - output: extract_metadata_from_filepath - timestamp: - parse_from: attributes.time - layout: '%Y-%m-%dT%H:%M:%S.%LZ' - - type: regex_parser - id: extract_metadata_from_filepath - regex: '^.*containers/(?P[^_]+)/.*log$' - parse_from: attributes["log.file.path"] - output: parse_body - - type: move - id: parse_body - from: attributes.log - to: body - output: time - - type: remove - id: time - field: attributes.time + - type: regex_parser + regex: '^<([0-9]+)>[0-9]+ (?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P\S+) (?P\S+) [0-9]+ - -( (?P.*))?' + timestamp: + parse_from: attributes.timestamp + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: move + from: attributes["body"] + to: body + - type: remove + field: attributes.timestamp + # please remove names from below if you want to collect logs from them + - type: filter + id: signoz_logs_filter + expr: 'attributes.container_name matches "^signoz_(logspout|frontend|alertmanager|query-service|otel-collector|otel-collector-metrics|clickhouse|zookeeper)"' opencensus: endpoint: 0.0.0.0:55678 otlp/spanmetrics: @@ -69,6 +61,40 @@ receivers: job_name: otel-collector processors: + logstransform/internal: + operators: + - type: trace_parser + if: '"trace_id" in attributes or "span_id" in attributes' + trace_id: + parse_from: attributes.trace_id + span_id: + parse_from: attributes.span_id + output: remove_trace_id + - type: trace_parser + if: '"traceId" in attributes or "spanId" in attributes' + trace_id: + parse_from: attributes.traceId + span_id: + parse_from: attributes.spanId + output: remove_traceId + - id: remove_traceId + type: remove + if: '"traceId" in attributes' + field: attributes.traceId + output: remove_spanId + - id: remove_spanId + type: remove + if: '"spanId" in attributes' + field: attributes.spanId + - id: remove_trace_id + type: remove + if: '"trace_id" in attributes' + field: attributes.trace_id + output: remove_span_id + - id: remove_span_id + type: remove + if: '"span_id" in attributes' + field: attributes.span_id batch: send_batch_size: 10000 send_batch_max_size: 11000 @@ -166,6 +192,6 @@ service: receivers: [otlp/spanmetrics] exporters: [prometheus] logs: - receivers: [otlp, filelog/dockercontainers] - processors: [batch] + receivers: [otlp, tcplog/docker] + processors: [logstransform/internal, batch] exporters: [clickhouselogsexporter] diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml index 5b6caa920f..439ad546ea 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml @@ -3,7 +3,7 @@ version: "2.4" services: clickhouse: image: clickhouse/clickhouse-server:22.8.8-alpine - container_name: clickhouse + container_name: signoz-clickhouse # ports: # - "9000:9000" # - "8123:8123" @@ -20,13 +20,20 @@ services: max-file: "3" healthcheck: # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8123/ping" + ] interval: 30s timeout: 5s retries: 3 alertmanager: - container_name: alertmanager + container_name: signoz-alertmanager image: signoz/alertmanager:0.23.1 volumes: - ./data/alertmanager:/data @@ -40,9 +47,13 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` otel-collector: - container_name: otel-collector + container_name: signoz-otel-collector image: signoz/signoz-otel-collector:0.79.4 - command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + command: + [ + "--config=/etc/otel-collector-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] # user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml @@ -50,8 +61,8 @@ services: - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux ports: # - "1777:1777" # pprof extension - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP receiver + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver # - "8888:8888" # OtelCollector internal metrics # - "8889:8889" # signoz spanmetrics exposed by the agent # - "9411:9411" # Zipkin port @@ -66,9 +77,13 @@ services: condition: service_healthy otel-collector-metrics: - container_name: otel-collector-metrics + container_name: signoz-otel-collector-metrics image: signoz/signoz-otel-collector:0.79.4 - command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + command: + [ + "--config=/etc/otel-collector-metrics-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml # ports: @@ -81,6 +96,17 @@ services: clickhouse: condition: service_healthy + logspout: + image: "gliderlabs/logspout:v3.2.14" + container_name: signoz-logspout + volumes: + - /etc/hostname:/etc/host_hostname:ro + - /var/run/docker.sock:/var/run/docker.sock + command: syslog+tcp://otel-collector:2255 + depends_on: + - otel-collector + restart: on-failure + hotrod: image: jaegertracing/example-hotrod:1.30 container_name: hotrod @@ -88,7 +114,7 @@ services: options: max-size: 50m max-file: "3" - command: ["all"] + command: [ "all" ] environment: - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces diff --git a/deploy/docker/clickhouse-setup/docker-compose-local.yaml b/deploy/docker/clickhouse-setup/docker-compose-local.yaml index b4c7cfca7e..cce46073f3 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-local.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-local.yaml @@ -9,7 +9,7 @@ services: args: LDFLAGS: "" TARGETPLATFORM: "${LOCAL_GOOS}/${LOCAL_GOARCH}" - container_name: query-service + container_name: signoz-query-service environment: - ClickHouseUrl=tcp://clickhouse:9000 - ALERTMANAGER_API_PREFIX=http://alertmanager:9093/api/ @@ -22,13 +22,20 @@ services: - ./prometheus.yml:/root/config/prometheus.yml - ../dashboards:/root/config/dashboards - ./data/signoz/:/var/lib/signoz/ - command: ["-config=/root/config/prometheus.yml"] + command: [ "-config=/root/config/prometheus.yml" ] ports: - "6060:6060" - "8080:8080" restart: on-failure healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8080/api/v1/health" + ] interval: 30s timeout: 5s retries: 3 @@ -43,7 +50,7 @@ services: args: TARGETOS: "${LOCAL_GOOS}" TARGETPLATFORM: "${LOCAL_GOARCH}" - container_name: frontend + container_name: signoz-frontend environment: - FRONTEND_API_ENDPOINT=http://query-service:8080 restart: on-failure diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index bcf6135274..62e809a624 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -1,6 +1,7 @@ version: "2.4" -x-clickhouse-defaults: &clickhouse-defaults +x-clickhouse-defaults: + &clickhouse-defaults restart: on-failure image: clickhouse/clickhouse-server:22.8.8-alpine tty: true @@ -14,7 +15,14 @@ x-clickhouse-defaults: &clickhouse-defaults max-file: "3" healthcheck: # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8123/ping" + ] interval: 30s timeout: 5s retries: 3 @@ -24,7 +32,8 @@ x-clickhouse-defaults: &clickhouse-defaults soft: 262144 hard: 262144 -x-clickhouse-depend: &clickhouse-depend +x-clickhouse-depend: + &clickhouse-depend depends_on: clickhouse: condition: service_healthy @@ -37,7 +46,7 @@ services: zookeeper-1: image: bitnami/zookeeper:3.7.1 - container_name: zookeeper-1 + container_name: signoz-zookeeper-1 hostname: zookeeper-1 user: root ports: @@ -54,7 +63,7 @@ services: # zookeeper-2: # image: bitnami/zookeeper:3.7.0 - # container_name: zookeeper-2 + # container_name: signoz-zookeeper-2 # hostname: zookeeper-2 # user: root # ports: @@ -71,7 +80,7 @@ services: # zookeeper-3: # image: bitnami/zookeeper:3.7.0 - # container_name: zookeeper-3 + # container_name: signoz-zookeeper-3 # hostname: zookeeper-3 # user: root # ports: @@ -88,7 +97,7 @@ services: clickhouse: <<: *clickhouse-defaults - container_name: clickhouse + container_name: signoz-clickhouse hostname: clickhouse ports: - "9000:9000" @@ -105,7 +114,7 @@ services: # clickhouse-2: # <<: *clickhouse-defaults - # container_name: clickhouse-2 + # container_name: signoz-clickhouse-2 # hostname: clickhouse-2 # ports: # - "9001:9000" @@ -120,10 +129,10 @@ services: # - ./data/clickhouse-2/:/var/lib/clickhouse/ # - ./user_scripts:/var/lib/clickhouse/user_scripts/ - + # clickhouse-3: # <<: *clickhouse-defaults - # container_name: clickhouse-3 + # container_name: signoz-clickhouse-3 # hostname: clickhouse-3 # ports: # - "9002:9000" @@ -140,6 +149,7 @@ services: alertmanager: image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.1} + container_name: signoz-alertmanager volumes: - ./data/alertmanager:/data depends_on: @@ -154,8 +164,8 @@ services: query-service: image: signoz/query-service:${DOCKER_TAG:-0.25.3} - container_name: query-service - command: ["-config=/root/config/prometheus.yml"] + container_name: signoz-query-service + command: [ "-config=/root/config/prometheus.yml" ] # ports: # - "6060:6060" # pprof port # - "8080:8080" # query-service port @@ -174,7 +184,14 @@ services: - DEPLOYMENT_TYPE=docker-standalone-amd restart: on-failure healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8080/api/v1/health" + ] interval: 30s timeout: 5s retries: 3 @@ -182,7 +199,7 @@ services: frontend: image: signoz/frontend:${DOCKER_TAG:-0.25.3} - container_name: frontend + container_name: signoz-frontend restart: on-failure depends_on: - alertmanager @@ -194,7 +211,12 @@ services: otel-collector: image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.4} - command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + container_name: signoz-otel-collector + command: + [ + "--config=/etc/otel-collector-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml @@ -205,8 +227,8 @@ services: - LOW_CARDINAL_EXCEPTION_GROUPING=false ports: # - "1777:1777" # pprof extension - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP receiver + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver # - "8888:8888" # OtelCollector internal metrics # - "8889:8889" # signoz spanmetrics exposed by the agent # - "9411:9411" # Zipkin port @@ -220,7 +242,12 @@ services: otel-collector-metrics: image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.4} - command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + container_name: signoz-otel-collector-metrics + command: + [ + "--config=/etc/otel-collector-metrics-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml # ports: @@ -231,6 +258,17 @@ services: restart: on-failure <<: *clickhouse-depend + logspout: + image: "gliderlabs/logspout:v3.2.14" + container_name: signoz-logspout + volumes: + - /etc/hostname:/etc/host_hostname:ro + - /var/run/docker.sock:/var/run/docker.sock + command: syslog+tcp://otel-collector:2255 + depends_on: + - otel-collector + restart: on-failure + hotrod: image: jaegertracing/example-hotrod:1.30 container_name: hotrod @@ -238,7 +276,7 @@ services: options: max-size: 50m max-file: "3" - command: ["all"] + command: [ "all" ] environment: - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml index c331f3a032..1604ae38a1 100644 --- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml @@ -1,29 +1,21 @@ receivers: - filelog/dockercontainers: - include: [ "/var/lib/docker/containers/*/*.log" ] - start_at: end - include_file_path: true - include_file_name: false + tcplog/docker: + listen_address: "0.0.0.0:2255" operators: - - type: json_parser - id: parser-docker - output: extract_metadata_from_filepath - timestamp: - parse_from: attributes.time - layout: '%Y-%m-%dT%H:%M:%S.%LZ' - - type: regex_parser - id: extract_metadata_from_filepath - regex: '^.*containers/(?P[^_]+)/.*log$' - parse_from: attributes["log.file.path"] - output: parse_body - - type: move - id: parse_body - from: attributes.log - to: body - output: time - - type: remove - id: time - field: attributes.time + - type: regex_parser + regex: '^<([0-9]+)>[0-9]+ (?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P\S+) (?P\S+) [0-9]+ - -( (?P.*))?' + timestamp: + parse_from: attributes.timestamp + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: move + from: attributes["body"] + to: body + - type: remove + field: attributes.timestamp + # please remove names from below if you want to collect logs from them + - type: filter + id: signoz_logs_filter + expr: 'attributes.container_name matches "^signoz-(logspout|frontend|alertmanager|query-service|otel-collector|otel-collector-metrics|clickhouse|zookeeper)"' opencensus: endpoint: 0.0.0.0:55678 otlp/spanmetrics: @@ -205,6 +197,6 @@ service: receivers: [otlp/spanmetrics] exporters: [prometheus] logs: - receivers: [otlp, filelog/dockercontainers] + receivers: [otlp, tcplog/docker] processors: [logstransform/internal, batch] exporters: [clickhouselogsexporter] \ No newline at end of file diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index 6a81776933..d1cf43d79c 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -1,6 +1,7 @@ version: "2.4" -x-clickhouse-defaults: &clickhouse-defaults +x-clickhouse-defaults: + &clickhouse-defaults restart: on-failure image: clickhouse/clickhouse-server:22.8.8-alpine tty: true @@ -14,7 +15,14 @@ x-clickhouse-defaults: &clickhouse-defaults max-file: "3" healthcheck: # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8123/ping" + ] interval: 30s timeout: 5s retries: 3 @@ -24,7 +32,8 @@ x-clickhouse-defaults: &clickhouse-defaults soft: 262144 hard: 262144 -x-clickhouse-depends: &clickhouse-depends +x-clickhouse-depends: + &clickhouse-depends depends_on: clickhouse: condition: service_healthy @@ -36,6 +45,7 @@ x-clickhouse-depends: &clickhouse-depends services: zookeeper-1: image: bitnami/zookeeper:3.7.1 + container_name: signoz-zookeeper-1 user: root ports: - "2181:2181" @@ -52,6 +62,7 @@ services: # zookeeper-2: # image: bitnami/zookeeper:3.7.0 + # container_name: signoz-zookeeper-2 # user: root # ports: # - "2182:2181" @@ -67,6 +78,7 @@ services: # zookeeper-3: # image: bitnami/zookeeper:3.7.0 + # container_name: signoz-zookeeper-3 # user: root # ports: # - "2183:2181" @@ -82,7 +94,7 @@ services: clickhouse: <<: *clickhouse-defaults - container_name: clickhouse + container_name: signoz-clickhouse hostname: clickhouse ports: - "9000:9000" @@ -95,10 +107,9 @@ services: # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml - ./data/clickhouse/:/var/lib/clickhouse/ - # clickhouse-2: # <<: *clickhouse-defaults - # container_name: clickhouse-2 + # container_name: signoz-clickhouse-2 # hostname: clickhouse-2 # ports: # - "9001:9000" @@ -113,7 +124,7 @@ services: # clickhouse-3: # <<: *clickhouse-defaults - # container_name: clickhouse-3 + # container_name: signoz-clickhouse-3 # hostname: clickhouse-3 # ports: # - "9002:9000" @@ -128,6 +139,7 @@ services: alertmanager: image: signoz/alertmanager:0.23.1 + container_name: signoz-alertmanager volumes: - ./data/alertmanager:/data depends_on: @@ -138,12 +150,12 @@ services: - --queryService.url=http://query-service:8085 - --storage.path=/data -# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` + # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: image: signoz/query-service:latest - container_name: query-service - command: ["-config=/root/config/prometheus.yml"] + container_name: signoz-query-service + command: [ "-config=/root/config/prometheus.yml" ] # ports: # - "6060:6060" # pprof port # - "8080:8080" # query-service port @@ -162,7 +174,14 @@ services: - DEPLOYMENT_TYPE=docker-standalone-amd restart: on-failure healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] + test: + [ + "CMD", + "wget", + "--spider", + "-q", + "localhost:8080/api/v1/health" + ] interval: 30s timeout: 5s retries: 3 @@ -170,7 +189,12 @@ services: otel-collector: image: signoz/signoz-otel-collector:0.79.4 - command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + container_name: signoz-otel-collector + command: + [ + "--config=/etc/otel-collector-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml @@ -181,8 +205,8 @@ services: - LOW_CARDINAL_EXCEPTION_GROUPING=false ports: # - "1777:1777" # pprof extension - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP receiver + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP HTTP receiver # - "8888:8888" # OtelCollector internal metrics # - "8889:8889" # signoz spanmetrics exposed by the agent # - "9411:9411" # Zipkin port @@ -196,7 +220,12 @@ services: otel-collector-metrics: image: signoz/signoz-otel-collector:0.79.4 - command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] + container_name: signoz-otel-collector-metrics + command: + [ + "--config=/etc/otel-collector-metrics-config.yaml", + "--feature-gates=-pkg.translator.prometheus.NormalizeName" + ] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml # ports: @@ -207,16 +236,27 @@ services: restart: on-failure <<: *clickhouse-depends + logspout: + image: "gliderlabs/logspout:v3.2.14" + container_name: signoz-logspout + volumes: + - /etc/hostname:/etc/host_hostname:ro + - /var/run/docker.sock:/var/run/docker.sock + command: syslog+tcp://otel-collector:2255 + depends_on: + - otel-collector + restart: on-failure + hotrod: - image: jaegertracing/example-hotrod:1.30 - container_name: hotrod - logging: - options: - max-size: 50m - max-file: "3" - command: ["all"] - environment: - - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces + image: jaegertracing/example-hotrod:1.30 + container_name: hotrod + logging: + options: + max-size: 50m + max-file: "3" + command: [ "all" ] + environment: + - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces load-hotrod: image: "signoz/locust:1.2.3" diff --git a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml index b4aab981e2..cc839e737f 100644 --- a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml +++ b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml @@ -1,29 +1,21 @@ receivers: - filelog/dockercontainers: - include: [ "/var/lib/docker/containers/*/*.log" ] - start_at: end - include_file_path: true - include_file_name: false + tcplog/docker: + listen_address: "0.0.0.0:2255" operators: - - type: json_parser - id: parser-docker - output: extract_metadata_from_filepath - timestamp: - parse_from: attributes.time - layout: '%Y-%m-%dT%H:%M:%S.%LZ' - - type: regex_parser - id: extract_metadata_from_filepath - regex: '^.*containers/(?P[^_]+)/.*log$' - parse_from: attributes["log.file.path"] - output: parse_body - - type: move - id: parse_body - from: attributes.log - to: body - output: time - - type: remove - id: time - field: attributes.time + - type: regex_parser + regex: '^<([0-9]+)>[0-9]+ (?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P\S+) (?P\S+) [0-9]+ - -( (?P.*))?' + timestamp: + parse_from: attributes.timestamp + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: move + from: attributes["body"] + to: body + - type: remove + field: attributes.timestamp + # please remove names from below if you want to collect logs from them + - type: filter + id: signoz_logs_filter + expr: 'attributes.container_name matches "^signoz-(logspout|frontend|alertmanager|query-service|otel-collector|otel-collector-metrics|clickhouse|zookeeper)"' opencensus: endpoint: 0.0.0.0:55678 otlp/spanmetrics: @@ -157,6 +149,6 @@ service: receivers: [otlp/spanmetrics] exporters: [prometheus] logs: - receivers: [otlp, filelog/dockercontainers] + receivers: [otlp, tcplog/docker] processors: [batch] exporters: [clickhouselogsexporter]