From 07dcdb51f7e98cb853f32e065d59082bb6bfb20b Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Wed, 24 Aug 2022 15:11:38 +0530 Subject: [PATCH] =?UTF-8?q?chore:=20=F0=9F=94=A7=20enable=20logs=20capturi?= =?UTF-8?q?ng=20by=20default=20(#1534)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Shahi --- .../clickhouse-setup/docker-compose.yaml | 10 +++---- .../otel-collector-config.yaml | 27 ++++++++++++++++- .../clickhouse-setup/docker-compose.yaml | 3 +- .../otel-collector-config.yaml | 27 ++++++++++++++++- .../tests/test-deploy/docker-compose.yaml | 2 ++ .../test-deploy/otel-collector-config.yaml | 29 +++++++++++++++++++ 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index ac34d7800d..8e069aa679 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -83,9 +83,12 @@ services: otel-collector: image: signoz-otel-collector:0.55.0 command: ["--config=/etc/otel-collector-config.yaml"] - # user: root # required for reading docker container logs + user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + environment: + - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}} ports: # - "1777:1777" # pprof extension - "4317:4317" # OTLP gRPC receiver @@ -98,11 +101,8 @@ services: # - "14268:14268" # Jaeger thrift HTTP # - "55678:55678" # OpenCensus receiver # - "55679:55679" # zPages extension - environment: - - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}} deploy: - mode: replicated - replicas: 3 + mode: global restart_policy: condition: on-failure resources: diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml index c3203e4f77..0a54841d0c 100644 --- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml @@ -1,4 +1,29 @@ receivers: + filelog/dockercontainers: + include: [ "/var/lib/docker/containers/*/*.log" ] + start_at: end + include_file_path: true + include_file_name: false + operators: + - type: json_parser + id: parser-docker + output: extract_metadata_from_filepath + timestamp: + parse_from: attributes.time + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: regex_parser + id: extract_metadata_from_filepath + regex: '^.*containers/(?P[^_]+)/.*log$' + parse_from: attributes["log.file.path"] + output: parse_body + - type: move + id: parse_body + from: attributes.log + to: body + output: time + - type: remove + id: time + field: attributes.time opencensus: endpoint: 0.0.0.0:55678 otlp/spanmetrics: @@ -117,6 +142,6 @@ service: receivers: [otlp/spanmetrics] exporters: [prometheus] logs: - receivers: [otlp] + receivers: [otlp, filelog/dockercontainers] processors: [batch] exporters: [clickhouselogsexporter] diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 3e45f195c8..b28ad35508 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -80,9 +80,10 @@ services: otel-collector: image: signoz/signoz-otel-collector:0.55.0 command: ["--config=/etc/otel-collector-config.yaml"] - # user: root # required for reading docker container logs + user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - /var/lib/docker/containers:/var/lib/docker/containers:ro environment: - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux ports: diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml index c264197352..b05b957e30 100644 --- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml @@ -1,4 +1,29 @@ receivers: + filelog/dockercontainers: + include: [ "/var/lib/docker/containers/*/*.log" ] + start_at: end + include_file_path: true + include_file_name: false + operators: + - type: json_parser + id: parser-docker + output: extract_metadata_from_filepath + timestamp: + parse_from: attributes.time + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: regex_parser + id: extract_metadata_from_filepath + regex: '^.*containers/(?P[^_]+)/.*log$' + parse_from: attributes["log.file.path"] + output: parse_body + - type: move + id: parse_body + from: attributes.log + to: body + output: time + - type: remove + id: time + field: attributes.time opencensus: endpoint: 0.0.0.0:55678 otlp/spanmetrics: @@ -121,6 +146,6 @@ service: receivers: [otlp/spanmetrics] exporters: [prometheus] logs: - receivers: [otlp] + receivers: [otlp, filelog/dockercontainers] processors: [batch] exporters: [clickhouselogsexporter] \ No newline at end of file diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index 8c459c7e5a..4ff378cf6b 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -61,8 +61,10 @@ services: otel-collector: image: signoz-otel-collector:0.55.0 command: ["--config=/etc/otel-collector-config.yaml"] + user: root # required for reading docker container logs volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + - /var/lib/docker/containers:/var/lib/docker/containers:ro environment: - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux ports: diff --git a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml index b343350a34..d11e2793b8 100644 --- a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml +++ b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml @@ -1,4 +1,29 @@ receivers: + filelog/dockercontainers: + include: [ "/var/lib/docker/containers/*/*.log" ] + start_at: end + include_file_path: true + include_file_name: false + operators: + - type: json_parser + id: parser-docker + output: extract_metadata_from_filepath + timestamp: + parse_from: attributes.time + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + - type: regex_parser + id: extract_metadata_from_filepath + regex: '^.*containers/(?P[^_]+)/.*log$' + parse_from: attributes["log.file.path"] + output: parse_body + - type: move + id: parse_body + from: attributes.log + to: body + output: time + - type: remove + id: time + field: attributes.time opencensus: endpoint: 0.0.0.0:55678 otlp/spanmetrics: @@ -108,3 +133,7 @@ service: metrics/spanmetrics: receivers: [otlp/spanmetrics] exporters: [prometheus] + logs: + receivers: [otlp, filelog/dockercontainers] + processors: [batch] + exporters: [clickhouselogsexporter]