chore: 🔧 enable logs capturing by default (#1534)

Signed-off-by: Prashant Shahi <prashant@signoz.io>

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -83,9 +83,12 @@ services:
   otel-collector:
     image: signoz-otel-collector:0.55.0
     command: ["--config=/etc/otel-collector-config.yaml"]
-    # user: root # required for reading docker container logs
+    user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+    environment:
+      - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}}
     ports:
       # - "1777:1777"     # pprof extension
       - "4317:4317"     # OTLP gRPC receiver
@@ -98,11 +101,8 @@ services:
       # - "14268:14268"   # Jaeger thrift HTTP
       # - "55678:55678"   # OpenCensus receiver
       # - "55679:55679"   # zPages extension
-    environment:
-      - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}}
     deploy:
-      mode: replicated
+      mode: global
-      replicas: 3
       restart_policy:
         condition: on-failure
       resources:

deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml

@@ -1,4 +1,29 @@
 receivers:
+  filelog/dockercontainers:
+    include: [  "/var/lib/docker/containers/*/*.log" ]
+    start_at: end
+    include_file_path: true
+    include_file_name: false
+    operators:
+    - type: json_parser
+      id: parser-docker
+      output: extract_metadata_from_filepath
+      timestamp:
+        parse_from: attributes.time
+        layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+    - type: regex_parser
+      id: extract_metadata_from_filepath
+      regex: '^.*containers/(?P<container_id>[^_]+)/.*log$'
+      parse_from: attributes["log.file.path"]
+      output: parse_body
+    - type: move
+      id: parse_body
+      from: attributes.log
+      to: body
+      output: time
+    - type: remove
+      id: time
+      field: attributes.time
   opencensus:
     endpoint: 0.0.0.0:55678
   otlp/spanmetrics:
@@ -117,6 +142,6 @@ service:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]
     logs:
-      receivers: [otlp]
+      receivers: [otlp, filelog/dockercontainers]
       processors: [batch]
       exporters: [clickhouselogsexporter]

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -80,9 +80,10 @@ services:
   otel-collector:
     image: signoz/signoz-otel-collector:0.55.0
     command: ["--config=/etc/otel-collector-config.yaml"]
-    # user: root # required for reading docker container logs
+    user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
     ports:

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -1,4 +1,29 @@
 receivers:
+  filelog/dockercontainers:
+    include: [  "/var/lib/docker/containers/*/*.log" ]
+    start_at: end
+    include_file_path: true
+    include_file_name: false
+    operators:
+    - type: json_parser
+      id: parser-docker
+      output: extract_metadata_from_filepath
+      timestamp:
+        parse_from: attributes.time
+        layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+    - type: regex_parser
+      id: extract_metadata_from_filepath
+      regex: '^.*containers/(?P<container_id>[^_]+)/.*log$'
+      parse_from: attributes["log.file.path"]
+      output: parse_body
+    - type: move
+      id: parse_body
+      from: attributes.log
+      to: body
+      output: time
+    - type: remove
+      id: time
+      field: attributes.time
   opencensus:
     endpoint: 0.0.0.0:55678
   otlp/spanmetrics:
@@ -121,6 +146,6 @@ service:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]
     logs:
-      receivers: [otlp]
+      receivers: [otlp, filelog/dockercontainers]
       processors: [batch]
       exporters: [clickhouselogsexporter]

pkg/query-service/tests/test-deploy/docker-compose.yaml

@@ -61,8 +61,10 @@ services:
   otel-collector:
     image: signoz-otel-collector:0.55.0
     command: ["--config=/etc/otel-collector-config.yaml"]
+    user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
     ports:

pkg/query-service/tests/test-deploy/otel-collector-config.yaml

@@ -1,4 +1,29 @@
 receivers:
+  filelog/dockercontainers:
+    include: [  "/var/lib/docker/containers/*/*.log" ]
+    start_at: end
+    include_file_path: true
+    include_file_name: false
+    operators:
+    - type: json_parser
+      id: parser-docker
+      output: extract_metadata_from_filepath
+      timestamp:
+        parse_from: attributes.time
+        layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+    - type: regex_parser
+      id: extract_metadata_from_filepath
+      regex: '^.*containers/(?P<container_id>[^_]+)/.*log$'
+      parse_from: attributes["log.file.path"]
+      output: parse_body
+    - type: move
+      id: parse_body
+      from: attributes.log
+      to: body
+      output: time
+    - type: remove
+      id: time
+      field: attributes.time
   opencensus:
     endpoint: 0.0.0.0:55678
   otlp/spanmetrics:
@@ -108,3 +133,7 @@ service:
     metrics/spanmetrics:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]
+    logs:
+      receivers: [otlp, filelog/dockercontainers]
+      processors: [batch]
+      exporters: [clickhouselogsexporter]