diff --git a/deploy/docker/docker-compose.yaml b/deploy/docker/docker-compose.yaml new file mode 100644 index 0000000000..927c87d533 --- /dev/null +++ b/deploy/docker/docker-compose.yaml @@ -0,0 +1,254 @@ +version: "2.4" + +volumes: + metadata_data: {} + middle_var: {} + historical_var: {} + broker_var: {} + coordinator_var: {} + router_var: {} + +# If able to connect to kafka but not able to write to topic otlp_spans look into below link +# https://github.com/wurstmeister/kafka-docker/issues/409#issuecomment-428346707 + +services: + + zookeeper: + image: bitnami/zookeeper:3.6.2-debian-10-r100 + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + + kafka: + # image: wurstmeister/kafka + image: bitnami/kafka:2.7.0-debian-10-r1 + ports: + - "9092:9092" + hostname: kafka + environment: + KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_TOPICS: 'otlp_spans:1:1,flattened_spans:1:1' + + healthcheck: + # test: ["CMD", "kafka-topics.sh", "--create", "--topic", "otlp_spans", "--zookeeper", "zookeeper:2181"] + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 30s + timeout: 10s + retries: 4 + depends_on: + - zookeeper + + postgres: + container_name: postgres + image: postgres:latest + volumes: + - metadata_data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=FoolishPassword + - POSTGRES_USER=druid + - POSTGRES_DB=druid + + coordinator: + image: apache/druid:0.20.0 + container_name: coordinator + volumes: + - ./storage:/opt/data + - coordinator_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + ports: + - "8081:8081" + command: + - coordinator + env_file: + - environment + + broker: + image: apache/druid:0.20.0 + container_name: broker + volumes: + - broker_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8082:8082" + command: + - broker + env_file: + - environment + + historical: + image: apache/druid:0.20.0 + container_name: historical + volumes: + - ./storage:/opt/data + - historical_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8083:8083" + command: + - historical + env_file: + - environment + + middlemanager: + image: apache/druid:0.20.0 + container_name: middlemanager + volumes: + - ./storage:/opt/data + - middle_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8091:8091" + command: + - middleManager + env_file: + - environment + + router: + image: apache/druid:0.20.0 + container_name: router + volumes: + - router_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8888:8888" + command: + - router + env_file: + - environment + + + flatten-processor: + image: signoz/flattener-processor:0.1.1 + container_name: flattener-processor + + depends_on: + - kafka + - otel-collector + ports: + - "8000:8000" + + environment: + - KAFKA_BROKER=kafka:9092 + - KAFKA_INPUT_TOPIC=otlp_spans + - KAFKA_OUTPUT_TOPIC=flattened_spans + + + query-service: + image: signoz/query-service:0.1.3 + container_name: query-service + + depends_on: + - router + ports: + - "8080:8080" + + environment: + - DruidClientUrl=http://router:8888 + - DruidDatasource=flattened_spans + - POSTHOG_API_KEY=H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w + + + frontend: + image: signoz/frontend:0.1.7 + container_name: frontend + + depends_on: + - query-service + links: + - "query-service" + ports: + - "3000:3000" + volumes: + - ./nginx-config.conf:/etc/nginx/conf.d/default.conf + + create-supervisor: + image: theithollow/hollowapp-blog:curl + container_name: create-supervisor + command: + - /bin/sh + - -c + - "curl -X POST -H 'Content-Type: application/json' -d @/app/supervisor-spec.json http://router:8888/druid/indexer/v1/supervisor" + + depends_on: + - router + volumes: + - ./druid-jobs/supervisor-spec.json:/app/supervisor-spec.json + + + set-retention: + image: theithollow/hollowapp-blog:curl + container_name: set-retention + command: + - /bin/sh + - -c + - "curl -X POST -H 'Content-Type: application/json' -d @/app/retention-spec.json http://router:8888/druid/coordinator/v1/rules/flattened_spans" + + depends_on: + - router + volumes: + - ./druid-jobs/retention-spec.json:/app/retention-spec.json + + otel-collector: + image: otel/opentelemetry-collector-dev:latest + command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1777:1777" # pprof extension + - "8887:8888" # Prometheus metrics exposed by the agent + - "14268:14268" # Jaeger receiver + - "55678" # OpenCensus receiver + - "55679:55679" # zpages extension + - "13133" # health_check + depends_on: + kafka: + condition: service_healthy + + + hotrod: + image: jaegertracing/example-hotrod:latest + container_name: hotrod + ports: + - "9000:8080" + command: ["all"] + environment: + - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces + + + load-hotrod: + image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" + container_name: load-hotrod + hostname: load-hotrod + ports: + - "8089:8089" + environment: + ATTACKED_HOST: http://hotrod:8080 + LOCUST_MODE: standalone + NO_PROXY: standalone + TASK_DELAY_FROM: 5 + TASK_DELAY_TO: 30 + QUIET_MODE: "${QUIET_MODE:-false}" + LOCUST_OPTS: "--headless -u 10 -r 1" + volumes: + - ./locust-scripts:/locust + diff --git a/deploy/docker/druid-jobs/retention-spec.json b/deploy/docker/druid-jobs/retention-spec.json new file mode 100644 index 0000000000..b4b0ebb426 --- /dev/null +++ b/deploy/docker/druid-jobs/retention-spec.json @@ -0,0 +1 @@ +[{"period":"P3D","includeFuture":true,"tieredReplicants":{"_default_tier":1},"type":"loadByPeriod"},{"type":"dropForever"}] \ No newline at end of file diff --git a/deploy/docker/druid-jobs/supervisor-spec.json b/deploy/docker/druid-jobs/supervisor-spec.json new file mode 100644 index 0000000000..33fd45b0a3 --- /dev/null +++ b/deploy/docker/druid-jobs/supervisor-spec.json @@ -0,0 +1,53 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "flattened_spans", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "StartTimeUnixNano", + "format": "nano" + }, + "dimensionsSpec": { + "dimensions": [ + "TraceId", + "SpanId", + "ParentSpanId", + "Name", + "ServiceName", + "References", + "Tags", + "TagsKeys", + "TagsValues", + { "name": "DurationNano", "type": "Long" }, + { "name": "Kind", "type": "int" } + ] + } + } + }, + "metricsSpec" : [ + { "type": "quantilesDoublesSketch", "name": "QuantileDuration", "fieldName": "DurationNano" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": true + }, + "ioConfig": { + "topic": "flattened_spans", + "replicas": 1, + "taskDuration": "PT20M", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafka:9092" + } + } + } diff --git a/deploy/docker/environment b/deploy/docker/environment new file mode 100644 index 0000000000..08b12a1bc0 --- /dev/null +++ b/deploy/docker/environment @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=1g +DRUID_XMS=1g +DRUID_MAXNEWSIZE=250m +DRUID_NEWSIZE=250m +DRUID_MAXDIRECTMEMORYSIZE=6172m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456 + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=2 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/locust-scripts/locustfile.py b/deploy/docker/locust-scripts/locustfile.py new file mode 100644 index 0000000000..0b518208cd --- /dev/null +++ b/deploy/docker/locust-scripts/locustfile.py @@ -0,0 +1,16 @@ +from locust import HttpUser, task, between +class UserTasks(HttpUser): + wait_time = between(5, 15) + + @task + def rachel(self): + self.client.get("/dispatch?customer=123&nonse=0.6308392664170006") + @task + def trom(self): + self.client.get("/dispatch?customer=392&nonse=0.015296363321630757") + @task + def japanese(self): + self.client.get("/dispatch?customer=731&nonse=0.8022286220408668") + @task + def coffee(self): + self.client.get("/dispatch?customer=567&nonse=0.0022220379420636593") diff --git a/deploy/docker/nginx-config.conf b/deploy/docker/nginx-config.conf new file mode 100644 index 0000000000..fa936b0732 --- /dev/null +++ b/deploy/docker/nginx-config.conf @@ -0,0 +1,20 @@ +server { + listen 3000; + server_name _; + location / { + root /usr/share/nginx/html; + index index.html index.htm; + try_files $uri $uri/ /index.html; + } + location /api { + proxy_pass http://query-service:8080/api; + + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file diff --git a/deploy/docker/otel-collector-config.yaml b/deploy/docker/otel-collector-config.yaml new file mode 100644 index 0000000000..d337607df2 --- /dev/null +++ b/deploy/docker/otel-collector-config.yaml @@ -0,0 +1,34 @@ +receivers: + jaeger: + protocols: + grpc: + thrift_http: +processors: + batch: + memory_limiter: + # Same as --mem-ballast-size-mib CLI argument + ballast_size_mib: 683 + # 80% of maximum memory up to 2G + limit_mib: 1500 + # 25% of limit up to 2G + spike_limit_mib: 512 + check_interval: 5s + queued_retry: + num_workers: 4 + queue_size: 100 + retry_on_failure: true +extensions: + health_check: {} + zpages: {} +exporters: + kafka: + brokers: + - kafka:9092 + protocol_version: 2.0.0 +service: + extensions: [health_check, zpages] + pipelines: + traces: + receivers: [jaeger] + processors: [memory_limiter, batch, queued_retry] + exporters: [kafka] \ No newline at end of file