GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-06-04 11:25:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add logspout as default docker logs collector (#3225)

Browse Source
This commit is contained in:
Nityananda Gohain 2023-08-07 21:17:53 +05:30 committed by GitHub
parent d63ae429bd
commit ae3d4fece8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 301 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
View File

@ -1,6 +1,7 @@
version: "3.9" version: "3.9"
x-clickhouse-defaults: &clickhouse-defaults x-clickhouse-defaults:
&clickhouse-defaults
image: clickhouse/clickhouse-server:22.8.8-alpine image: clickhouse/clickhouse-server:22.8.8-alpine
tty: true tty: true
deploy: deploy:
@ -16,7 +17,14 @@ x-clickhouse-defaults: &clickhouse-defaults
max-file: "3" max-file: "3"
healthcheck: healthcheck:
# "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8123/ping"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -26,7 +34,8 @@ x-clickhouse-defaults: &clickhouse-defaults
soft: 262144 soft: 262144
hard: 262144 hard: 262144
x-clickhouse-depend: &clickhouse-depend x-clickhouse-depend:
&clickhouse-depend
depends_on: depends_on:
- clickhouse - clickhouse
# - clickhouse-2 # - clickhouse-2
@ -138,7 +147,7 @@ services:
query-service: query-service:
image: signoz/query-service:0.25.3 image: signoz/query-service:0.25.3
command: ["-config=/root/config/prometheus.yml"] command: [ "-config=/root/config/prometheus.yml" ]
# ports: # ports:
# - "6060:6060" # pprof port # - "6060:6060" # pprof port
# - "8080:8080" # query-service port # - "8080:8080" # query-service port
@ -156,7 +165,14 @@ services:
- TELEMETRY_ENABLED=true - TELEMETRY_ENABLED=true
- DEPLOYMENT_TYPE=docker-swarm - DEPLOYMENT_TYPE=docker-swarm
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8080/api/v1/health"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -180,7 +196,11 @@ services:
otel-collector: otel-collector:
image: signoz/signoz-otel-collector:0.79.4 image: signoz/signoz-otel-collector:0.79.4
command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] command:
[
"--config=/etc/otel-collector-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
user: root # required for reading docker container logs user: root # required for reading docker container logs
volumes: volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@ -191,8 +211,8 @@ services:
- LOW_CARDINAL_EXCEPTION_GROUPING=false - LOW_CARDINAL_EXCEPTION_GROUPING=false
ports: ports:
# - "1777:1777" # pprof extension # - "1777:1777" # pprof extension
- "4317:4317" # OTLP gRPC receiver - "4317:4317" # OTLP gRPC receiver
- "4318:4318" # OTLP HTTP receiver - "4318:4318" # OTLP HTTP receiver
# - "8888:8888" # OtelCollector internal metrics # - "8888:8888" # OtelCollector internal metrics
# - "8889:8889" # signoz spanmetrics exposed by the agent # - "8889:8889" # signoz spanmetrics exposed by the agent
# - "9411:9411" # Zipkin port # - "9411:9411" # Zipkin port
@ -209,7 +229,11 @@ services:
otel-collector-metrics: otel-collector-metrics:
image: signoz/signoz-otel-collector:0.79.4 image: signoz/signoz-otel-collector:0.79.4
command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] command:
[
"--config=/etc/otel-collector-metrics-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
volumes: volumes:
- ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
# ports: # ports:
@ -222,9 +246,22 @@ services:
condition: on-failure condition: on-failure
<<: *clickhouse-depend <<: *clickhouse-depend
logspout:
image: "gliderlabs/logspout:v3.2.14"
volumes:
- /etc/hostname:/etc/host_hostname:ro
- /var/run/docker.sock:/var/run/docker.sock
command: syslog+tcp://otel-collector:2255
depends_on:
- otel-collector
deploy:
mode: global
restart_policy:
condition: on-failure
hotrod: hotrod:
image: jaegertracing/example-hotrod:1.30 image: jaegertracing/example-hotrod:1.30
command: ["all"] command: [ "all" ]
environment: environment:
- JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces
logging: logging:

78
deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
View File

@ -1,29 +1,21 @@
receivers: receivers:
filelog/dockercontainers: tcplog/docker:
include: [ "/var/lib/docker/containers/*/*.log" ] listen_address: "0.0.0.0:2255"
start_at: end
include_file_path: true
include_file_name: false
operators: operators:
- type: json_parser - type: regex_parser
id: parser-docker regex: '^<([0-9]+)>[0-9]+ (?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P<container_id>\S+) (?P<container_name>\S+) [0-9]+ - -( (?P<body>.*))?'
output: extract_metadata_from_filepath timestamp:
timestamp: parse_from: attributes.timestamp
parse_from: attributes.time layout: '%Y-%m-%dT%H:%M:%S.%LZ'
layout: '%Y-%m-%dT%H:%M:%S.%LZ' - type: move
- type: regex_parser from: attributes["body"]
id: extract_metadata_from_filepath to: body
regex: '^.*containers/(?P<container_id>[^_]+)/.*log$' - type: remove
parse_from: attributes["log.file.path"] field: attributes.timestamp
output: parse_body # please remove names from below if you want to collect logs from them
- type: move - type: filter
id: parse_body id: signoz_logs_filter
from: attributes.log expr: 'attributes.container_name matches "^signoz_(logspout|frontend|alertmanager|query-service|otel-collector|otel-collector-metrics|clickhouse|zookeeper)"'
to: body
output: time
- type: remove
id: time
field: attributes.time
opencensus: opencensus:
endpoint: 0.0.0.0:55678 endpoint: 0.0.0.0:55678
otlp/spanmetrics: otlp/spanmetrics:
@ -69,6 +61,40 @@ receivers:
job_name: otel-collector job_name: otel-collector
processors: processors:
logstransform/internal:
operators:
- type: trace_parser
if: '"trace_id" in attributes or "span_id" in attributes'
trace_id:
parse_from: attributes.trace_id
span_id:
parse_from: attributes.span_id
output: remove_trace_id
- type: trace_parser
if: '"traceId" in attributes or "spanId" in attributes'
trace_id:
parse_from: attributes.traceId
span_id:
parse_from: attributes.spanId
output: remove_traceId
- id: remove_traceId
type: remove
if: '"traceId" in attributes'
field: attributes.traceId
output: remove_spanId
- id: remove_spanId
type: remove
if: '"spanId" in attributes'
field: attributes.spanId
- id: remove_trace_id
type: remove
if: '"trace_id" in attributes'
field: attributes.trace_id
output: remove_span_id
- id: remove_span_id
type: remove
if: '"span_id" in attributes'
field: attributes.span_id
batch: batch:
send_batch_size: 10000 send_batch_size: 10000
send_batch_max_size: 11000 send_batch_max_size: 11000
@ -166,6 +192,6 @@ service:
receivers: [otlp/spanmetrics] receivers: [otlp/spanmetrics]
exporters: [prometheus] exporters: [prometheus]
logs: logs:
receivers: [otlp, filelog/dockercontainers] receivers: [otlp, tcplog/docker]
processors: [batch] processors: [logstransform/internal, batch]
exporters: [clickhouselogsexporter] exporters: [clickhouselogsexporter]

46
deploy/docker/clickhouse-setup/docker-compose-core.yaml
View File

@ -3,7 +3,7 @@ version: "2.4"
services: services:
clickhouse: clickhouse:
image: clickhouse/clickhouse-server:22.8.8-alpine image: clickhouse/clickhouse-server:22.8.8-alpine
container_name: clickhouse container_name: signoz-clickhouse
# ports: # ports:
# - "9000:9000" # - "9000:9000"
# - "8123:8123" # - "8123:8123"
@ -20,13 +20,20 @@ services:
max-file: "3" max-file: "3"
healthcheck: healthcheck:
# "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8123/ping"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
alertmanager: alertmanager:
container_name: alertmanager container_name: signoz-alertmanager
image: signoz/alertmanager:0.23.1 image: signoz/alertmanager:0.23.1
volumes: volumes:
- ./data/alertmanager:/data - ./data/alertmanager:/data
@ -40,9 +47,13 @@ services:
# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
otel-collector: otel-collector:
container_name: otel-collector container_name: signoz-otel-collector
image: signoz/signoz-otel-collector:0.79.4 image: signoz/signoz-otel-collector:0.79.4
command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] command:
[
"--config=/etc/otel-collector-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
# user: root # required for reading docker container logs # user: root # required for reading docker container logs
volumes: volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@ -50,8 +61,8 @@ services:
- OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
ports: ports:
# - "1777:1777" # pprof extension # - "1777:1777" # pprof extension
- "4317:4317" # OTLP gRPC receiver - "4317:4317" # OTLP gRPC receiver
- "4318:4318" # OTLP HTTP receiver - "4318:4318" # OTLP HTTP receiver
# - "8888:8888" # OtelCollector internal metrics # - "8888:8888" # OtelCollector internal metrics
# - "8889:8889" # signoz spanmetrics exposed by the agent # - "8889:8889" # signoz spanmetrics exposed by the agent
# - "9411:9411" # Zipkin port # - "9411:9411" # Zipkin port
@ -66,9 +77,13 @@ services:
condition: service_healthy condition: service_healthy
otel-collector-metrics: otel-collector-metrics:
container_name: otel-collector-metrics container_name: signoz-otel-collector-metrics
image: signoz/signoz-otel-collector:0.79.4 image: signoz/signoz-otel-collector:0.79.4
command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] command:
[
"--config=/etc/otel-collector-metrics-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
volumes: volumes:
- ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
# ports: # ports:
@ -81,6 +96,17 @@ services:
clickhouse: clickhouse:
condition: service_healthy condition: service_healthy
logspout:
image: "gliderlabs/logspout:v3.2.14"
container_name: signoz-logspout
volumes:
- /etc/hostname:/etc/host_hostname:ro
- /var/run/docker.sock:/var/run/docker.sock
command: syslog+tcp://otel-collector:2255
depends_on:
- otel-collector
restart: on-failure
hotrod: hotrod:
image: jaegertracing/example-hotrod:1.30 image: jaegertracing/example-hotrod:1.30
container_name: hotrod container_name: hotrod
@ -88,7 +114,7 @@ services:
options: options:
max-size: 50m max-size: 50m
max-file: "3" max-file: "3"
command: ["all"] command: [ "all" ]
environment: environment:
- JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces

15
deploy/docker/clickhouse-setup/docker-compose-local.yaml
View File

@ -9,7 +9,7 @@ services:
args: args:
LDFLAGS: "" LDFLAGS: ""
TARGETPLATFORM: "${LOCAL_GOOS}/${LOCAL_GOARCH}" TARGETPLATFORM: "${LOCAL_GOOS}/${LOCAL_GOARCH}"
container_name: query-service container_name: signoz-query-service
environment: environment:
- ClickHouseUrl=tcp://clickhouse:9000 - ClickHouseUrl=tcp://clickhouse:9000
- ALERTMANAGER_API_PREFIX=http://alertmanager:9093/api/ - ALERTMANAGER_API_PREFIX=http://alertmanager:9093/api/
@ -22,13 +22,20 @@ services:
- ./prometheus.yml:/root/config/prometheus.yml - ./prometheus.yml:/root/config/prometheus.yml
- ../dashboards:/root/config/dashboards - ../dashboards:/root/config/dashboards
- ./data/signoz/:/var/lib/signoz/ - ./data/signoz/:/var/lib/signoz/
command: ["-config=/root/config/prometheus.yml"] command: [ "-config=/root/config/prometheus.yml" ]
ports: ports:
- "6060:6060" - "6060:6060"
- "8080:8080" - "8080:8080"
restart: on-failure restart: on-failure
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8080/api/v1/health"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -43,7 +50,7 @@ services:
args: args:
TARGETOS: "${LOCAL_GOOS}" TARGETOS: "${LOCAL_GOOS}"
TARGETPLATFORM: "${LOCAL_GOARCH}" TARGETPLATFORM: "${LOCAL_GOARCH}"
container_name: frontend container_name: signoz-frontend
environment: environment:
- FRONTEND_API_ENDPOINT=http://query-service:8080 - FRONTEND_API_ENDPOINT=http://query-service:8080
restart: on-failure restart: on-failure

76
deploy/docker/clickhouse-setup/docker-compose.yaml
View File

@ -1,6 +1,7 @@
version: "2.4" version: "2.4"
x-clickhouse-defaults: &clickhouse-defaults x-clickhouse-defaults:
&clickhouse-defaults
restart: on-failure restart: on-failure
image: clickhouse/clickhouse-server:22.8.8-alpine image: clickhouse/clickhouse-server:22.8.8-alpine
tty: true tty: true
@ -14,7 +15,14 @@ x-clickhouse-defaults: &clickhouse-defaults
max-file: "3" max-file: "3"
healthcheck: healthcheck:
# "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8123/ping"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -24,7 +32,8 @@ x-clickhouse-defaults: &clickhouse-defaults
soft: 262144 soft: 262144
hard: 262144 hard: 262144
x-clickhouse-depend: &clickhouse-depend x-clickhouse-depend:
&clickhouse-depend
depends_on: depends_on:
clickhouse: clickhouse:
condition: service_healthy condition: service_healthy
@ -37,7 +46,7 @@ services:
zookeeper-1: zookeeper-1:
image: bitnami/zookeeper:3.7.1 image: bitnami/zookeeper:3.7.1
container_name: zookeeper-1 container_name: signoz-zookeeper-1
hostname: zookeeper-1 hostname: zookeeper-1
user: root user: root
ports: ports:
@ -54,7 +63,7 @@ services:
# zookeeper-2: # zookeeper-2:
# image: bitnami/zookeeper:3.7.0 # image: bitnami/zookeeper:3.7.0
# container_name: zookeeper-2 # container_name: signoz-zookeeper-2
# hostname: zookeeper-2 # hostname: zookeeper-2
# user: root # user: root
# ports: # ports:
@ -71,7 +80,7 @@ services:
# zookeeper-3: # zookeeper-3:
# image: bitnami/zookeeper:3.7.0 # image: bitnami/zookeeper:3.7.0
# container_name: zookeeper-3 # container_name: signoz-zookeeper-3
# hostname: zookeeper-3 # hostname: zookeeper-3
# user: root # user: root
# ports: # ports:
@ -88,7 +97,7 @@ services:
clickhouse: clickhouse:
<<: *clickhouse-defaults <<: *clickhouse-defaults
container_name: clickhouse container_name: signoz-clickhouse
hostname: clickhouse hostname: clickhouse
ports: ports:
- "9000:9000" - "9000:9000"
@ -105,7 +114,7 @@ services:
# clickhouse-2: # clickhouse-2:
# <<: *clickhouse-defaults # <<: *clickhouse-defaults
# container_name: clickhouse-2 # container_name: signoz-clickhouse-2
# hostname: clickhouse-2 # hostname: clickhouse-2
# ports: # ports:
# - "9001:9000" # - "9001:9000"
@ -120,10 +129,10 @@ services:
# - ./data/clickhouse-2/:/var/lib/clickhouse/ # - ./data/clickhouse-2/:/var/lib/clickhouse/
# - ./user_scripts:/var/lib/clickhouse/user_scripts/ # - ./user_scripts:/var/lib/clickhouse/user_scripts/
# clickhouse-3: # clickhouse-3:
# <<: *clickhouse-defaults # <<: *clickhouse-defaults
# container_name: clickhouse-3 # container_name: signoz-clickhouse-3
# hostname: clickhouse-3 # hostname: clickhouse-3
# ports: # ports:
# - "9002:9000" # - "9002:9000"
@ -140,6 +149,7 @@ services:
alertmanager: alertmanager:
image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.1} image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.1}
container_name: signoz-alertmanager
volumes: volumes:
- ./data/alertmanager:/data - ./data/alertmanager:/data
depends_on: depends_on:
@ -154,8 +164,8 @@ services:
query-service: query-service:
image: signoz/query-service:${DOCKER_TAG:-0.25.3} image: signoz/query-service:${DOCKER_TAG:-0.25.3}
container_name: query-service container_name: signoz-query-service
command: ["-config=/root/config/prometheus.yml"] command: [ "-config=/root/config/prometheus.yml" ]
# ports: # ports:
# - "6060:6060" # pprof port # - "6060:6060" # pprof port
# - "8080:8080" # query-service port # - "8080:8080" # query-service port
@ -174,7 +184,14 @@ services:
- DEPLOYMENT_TYPE=docker-standalone-amd - DEPLOYMENT_TYPE=docker-standalone-amd
restart: on-failure restart: on-failure
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8080/api/v1/health"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -182,7 +199,7 @@ services:
frontend: frontend:
image: signoz/frontend:${DOCKER_TAG:-0.25.3} image: signoz/frontend:${DOCKER_TAG:-0.25.3}
container_name: frontend container_name: signoz-frontend
restart: on-failure restart: on-failure
depends_on: depends_on:
- alertmanager - alertmanager
@ -194,7 +211,12 @@ services:
otel-collector: otel-collector:
image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.4} image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.4}
command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] container_name: signoz-otel-collector
command:
[
"--config=/etc/otel-collector-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
user: root # required for reading docker container logs user: root # required for reading docker container logs
volumes: volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@ -205,8 +227,8 @@ services:
- LOW_CARDINAL_EXCEPTION_GROUPING=false - LOW_CARDINAL_EXCEPTION_GROUPING=false
ports: ports:
# - "1777:1777" # pprof extension # - "1777:1777" # pprof extension
- "4317:4317" # OTLP gRPC receiver - "4317:4317" # OTLP gRPC receiver
- "4318:4318" # OTLP HTTP receiver - "4318:4318" # OTLP HTTP receiver
# - "8888:8888" # OtelCollector internal metrics # - "8888:8888" # OtelCollector internal metrics
# - "8889:8889" # signoz spanmetrics exposed by the agent # - "8889:8889" # signoz spanmetrics exposed by the agent
# - "9411:9411" # Zipkin port # - "9411:9411" # Zipkin port
@ -220,7 +242,12 @@ services:
otel-collector-metrics: otel-collector-metrics:
image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.4} image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.79.4}
command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] container_name: signoz-otel-collector-metrics
command:
[
"--config=/etc/otel-collector-metrics-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
volumes: volumes:
- ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
# ports: # ports:
@ -231,6 +258,17 @@ services:
restart: on-failure restart: on-failure
<<: *clickhouse-depend <<: *clickhouse-depend
logspout:
image: "gliderlabs/logspout:v3.2.14"
container_name: signoz-logspout
volumes:
- /etc/hostname:/etc/host_hostname:ro
- /var/run/docker.sock:/var/run/docker.sock
command: syslog+tcp://otel-collector:2255
depends_on:
- otel-collector
restart: on-failure
hotrod: hotrod:
image: jaegertracing/example-hotrod:1.30 image: jaegertracing/example-hotrod:1.30
container_name: hotrod container_name: hotrod
@ -238,7 +276,7 @@ services:
options: options:
max-size: 50m max-size: 50m
max-file: "3" max-file: "3"
command: ["all"] command: [ "all" ]
environment: environment:
- JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces

42
deploy/docker/clickhouse-setup/otel-collector-config.yaml
View File

@ -1,29 +1,21 @@
receivers: receivers:
filelog/dockercontainers: tcplog/docker:
include: [ "/var/lib/docker/containers/*/*.log" ] listen_address: "0.0.0.0:2255"
start_at: end
include_file_path: true
include_file_name: false
operators: operators:
- type: json_parser - type: regex_parser
id: parser-docker regex: '^<([0-9]+)>[0-9]+ (?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P<container_id>\S+) (?P<container_name>\S+) [0-9]+ - -( (?P<body>.*))?'
output: extract_metadata_from_filepath timestamp:
timestamp: parse_from: attributes.timestamp
parse_from: attributes.time layout: '%Y-%m-%dT%H:%M:%S.%LZ'
layout: '%Y-%m-%dT%H:%M:%S.%LZ' - type: move
- type: regex_parser from: attributes["body"]
id: extract_metadata_from_filepath to: body
regex: '^.*containers/(?P<container_id>[^_]+)/.*log$' - type: remove
parse_from: attributes["log.file.path"] field: attributes.timestamp
output: parse_body # please remove names from below if you want to collect logs from them
- type: move - type: filter
id: parse_body id: signoz_logs_filter
from: attributes.log expr: 'attributes.container_name matches "^signoz-(logspout|frontend|alertmanager|query-service|otel-collector|otel-collector-metrics|clickhouse|zookeeper)"'
to: body
output: time
- type: remove
id: time
field: attributes.time
opencensus: opencensus:
endpoint: 0.0.0.0:55678 endpoint: 0.0.0.0:55678
otlp/spanmetrics: otlp/spanmetrics:
@ -205,6 +197,6 @@ service:
receivers: [otlp/spanmetrics] receivers: [otlp/spanmetrics]
exporters: [prometheus] exporters: [prometheus]
logs: logs:
receivers: [otlp, filelog/dockercontainers] receivers: [otlp, tcplog/docker]
processors: [logstransform/internal, batch] processors: [logstransform/internal, batch]
exporters: [clickhouselogsexporter] exporters: [clickhouselogsexporter]

88
pkg/query-service/tests/test-deploy/docker-compose.yaml
View File

@ -1,6 +1,7 @@
version: "2.4" version: "2.4"
x-clickhouse-defaults: &clickhouse-defaults x-clickhouse-defaults:
&clickhouse-defaults
restart: on-failure restart: on-failure
image: clickhouse/clickhouse-server:22.8.8-alpine image: clickhouse/clickhouse-server:22.8.8-alpine
tty: true tty: true
@ -14,7 +15,14 @@ x-clickhouse-defaults: &clickhouse-defaults
max-file: "3" max-file: "3"
healthcheck: healthcheck:
# "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8123/ping"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -24,7 +32,8 @@ x-clickhouse-defaults: &clickhouse-defaults
soft: 262144 soft: 262144
hard: 262144 hard: 262144
x-clickhouse-depends: &clickhouse-depends x-clickhouse-depends:
&clickhouse-depends
depends_on: depends_on:
clickhouse: clickhouse:
condition: service_healthy condition: service_healthy
@ -36,6 +45,7 @@ x-clickhouse-depends: &clickhouse-depends
services: services:
zookeeper-1: zookeeper-1:
image: bitnami/zookeeper:3.7.1 image: bitnami/zookeeper:3.7.1
container_name: signoz-zookeeper-1
user: root user: root
ports: ports:
- "2181:2181" - "2181:2181"
@ -52,6 +62,7 @@ services:
# zookeeper-2: # zookeeper-2:
# image: bitnami/zookeeper:3.7.0 # image: bitnami/zookeeper:3.7.0
# container_name: signoz-zookeeper-2
# user: root # user: root
# ports: # ports:
# - "2182:2181" # - "2182:2181"
@ -67,6 +78,7 @@ services:
# zookeeper-3: # zookeeper-3:
# image: bitnami/zookeeper:3.7.0 # image: bitnami/zookeeper:3.7.0
# container_name: signoz-zookeeper-3
# user: root # user: root
# ports: # ports:
# - "2183:2181" # - "2183:2181"
@ -82,7 +94,7 @@ services:
clickhouse: clickhouse:
<<: *clickhouse-defaults <<: *clickhouse-defaults
container_name: clickhouse container_name: signoz-clickhouse
hostname: clickhouse hostname: clickhouse
ports: ports:
- "9000:9000" - "9000:9000"
@ -95,10 +107,9 @@ services:
# - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
- ./data/clickhouse/:/var/lib/clickhouse/ - ./data/clickhouse/:/var/lib/clickhouse/
# clickhouse-2: # clickhouse-2:
# <<: *clickhouse-defaults # <<: *clickhouse-defaults
# container_name: clickhouse-2 # container_name: signoz-clickhouse-2
# hostname: clickhouse-2 # hostname: clickhouse-2
# ports: # ports:
# - "9001:9000" # - "9001:9000"
@ -113,7 +124,7 @@ services:
# clickhouse-3: # clickhouse-3:
# <<: *clickhouse-defaults # <<: *clickhouse-defaults
# container_name: clickhouse-3 # container_name: signoz-clickhouse-3
# hostname: clickhouse-3 # hostname: clickhouse-3
# ports: # ports:
# - "9002:9000" # - "9002:9000"
@ -128,6 +139,7 @@ services:
alertmanager: alertmanager:
image: signoz/alertmanager:0.23.1 image: signoz/alertmanager:0.23.1
container_name: signoz-alertmanager
volumes: volumes:
- ./data/alertmanager:/data - ./data/alertmanager:/data
depends_on: depends_on:
@ -138,12 +150,12 @@ services:
- --queryService.url=http://query-service:8085 - --queryService.url=http://query-service:8085
- --storage.path=/data - --storage.path=/data
# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
query-service: query-service:
image: signoz/query-service:latest image: signoz/query-service:latest
container_name: query-service container_name: signoz-query-service
command: ["-config=/root/config/prometheus.yml"] command: [ "-config=/root/config/prometheus.yml" ]
# ports: # ports:
# - "6060:6060" # pprof port # - "6060:6060" # pprof port
# - "8080:8080" # query-service port # - "8080:8080" # query-service port
@ -162,7 +174,14 @@ services:
- DEPLOYMENT_TYPE=docker-standalone-amd - DEPLOYMENT_TYPE=docker-standalone-amd
restart: on-failure restart: on-failure
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"] test:
[
"CMD",
"wget",
"--spider",
"-q",
"localhost:8080/api/v1/health"
]
interval: 30s interval: 30s
timeout: 5s timeout: 5s
retries: 3 retries: 3
@ -170,7 +189,12 @@ services:
otel-collector: otel-collector:
image: signoz/signoz-otel-collector:0.79.4 image: signoz/signoz-otel-collector:0.79.4
command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] container_name: signoz-otel-collector
command:
[
"--config=/etc/otel-collector-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
user: root # required for reading docker container logs user: root # required for reading docker container logs
volumes: volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@ -181,8 +205,8 @@ services:
- LOW_CARDINAL_EXCEPTION_GROUPING=false - LOW_CARDINAL_EXCEPTION_GROUPING=false
ports: ports:
# - "1777:1777" # pprof extension # - "1777:1777" # pprof extension
- "4317:4317" # OTLP gRPC receiver - "4317:4317" # OTLP gRPC receiver
- "4318:4318" # OTLP HTTP receiver - "4318:4318" # OTLP HTTP receiver
# - "8888:8888" # OtelCollector internal metrics # - "8888:8888" # OtelCollector internal metrics
# - "8889:8889" # signoz spanmetrics exposed by the agent # - "8889:8889" # signoz spanmetrics exposed by the agent
# - "9411:9411" # Zipkin port # - "9411:9411" # Zipkin port
@ -196,7 +220,12 @@ services:
otel-collector-metrics: otel-collector-metrics:
image: signoz/signoz-otel-collector:0.79.4 image: signoz/signoz-otel-collector:0.79.4
command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"] container_name: signoz-otel-collector-metrics
command:
[
"--config=/etc/otel-collector-metrics-config.yaml",
"--feature-gates=-pkg.translator.prometheus.NormalizeName"
]
volumes: volumes:
- ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
# ports: # ports:
@ -207,16 +236,27 @@ services:
restart: on-failure restart: on-failure
<<: *clickhouse-depends <<: *clickhouse-depends
logspout:
image: "gliderlabs/logspout:v3.2.14"
container_name: signoz-logspout
volumes:
- /etc/hostname:/etc/host_hostname:ro
- /var/run/docker.sock:/var/run/docker.sock
command: syslog+tcp://otel-collector:2255
depends_on:
- otel-collector
restart: on-failure
hotrod: hotrod:
image: jaegertracing/example-hotrod:1.30 image: jaegertracing/example-hotrod:1.30
container_name: hotrod container_name: hotrod
logging: logging:
options: options:
max-size: 50m max-size: 50m
max-file: "3" max-file: "3"
command: ["all"] command: [ "all" ]
environment: environment:
- JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces
load-hotrod: load-hotrod:
image: "signoz/locust:1.2.3" image: "signoz/locust:1.2.3"

42
pkg/query-service/tests/test-deploy/otel-collector-config.yaml
View File

@ -1,29 +1,21 @@
receivers: receivers:
filelog/dockercontainers: tcplog/docker:
include: [ "/var/lib/docker/containers/*/*.log" ] listen_address: "0.0.0.0:2255"
start_at: end
include_file_path: true
include_file_name: false
operators: operators:
- type: json_parser - type: regex_parser
id: parser-docker regex: '^<([0-9]+)>[0-9]+ (?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P<container_id>\S+) (?P<container_name>\S+) [0-9]+ - -( (?P<body>.*))?'
output: extract_metadata_from_filepath timestamp:
timestamp: parse_from: attributes.timestamp
parse_from: attributes.time layout: '%Y-%m-%dT%H:%M:%S.%LZ'
layout: '%Y-%m-%dT%H:%M:%S.%LZ' - type: move
- type: regex_parser from: attributes["body"]
id: extract_metadata_from_filepath to: body
regex: '^.*containers/(?P<container_id>[^_]+)/.*log$' - type: remove
parse_from: attributes["log.file.path"] field: attributes.timestamp
output: parse_body # please remove names from below if you want to collect logs from them
- type: move - type: filter
id: parse_body id: signoz_logs_filter
from: attributes.log expr: 'attributes.container_name matches "^signoz-(logspout|frontend|alertmanager|query-service|otel-collector|otel-collector-metrics|clickhouse|zookeeper)"'
to: body
output: time
- type: remove
id: time
field: attributes.time
opencensus: opencensus:
endpoint: 0.0.0.0:55678 endpoint: 0.0.0.0:55678
otlp/spanmetrics: otlp/spanmetrics:
@ -157,6 +149,6 @@ service:
receivers: [otlp/spanmetrics] receivers: [otlp/spanmetrics]
exporters: [prometheus] exporters: [prometheus]
logs: logs:
receivers: [otlp, filelog/dockercontainers] receivers: [otlp, tcplog/docker]
processors: [batch] processors: [batch]
exporters: [clickhouselogsexporter] exporters: [clickhouselogsexporter]