added docker compose

deploy/docker/docker-compose.yaml

@@ -0,0 +1,254 @@
+version: "2.4"
+
+volumes:
+  metadata_data: {}
+  middle_var: {}
+  historical_var: {}
+  broker_var: {}
+  coordinator_var: {}
+  router_var: {}
+
+# If able to connect to kafka but not able to write to topic otlp_spans look into below link
+# https://github.com/wurstmeister/kafka-docker/issues/409#issuecomment-428346707
+
+services:
+
+  zookeeper:
+    image: bitnami/zookeeper:3.6.2-debian-10-r100
+    ports:
+      - "2181:2181"
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+
+
+  kafka:
+    # image: wurstmeister/kafka
+    image: bitnami/kafka:2.7.0-debian-10-r1
+    ports:
+      - "9092:9092"
+    hostname: kafka
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      ALLOW_PLAINTEXT_LISTENER: 'yes'
+      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TOPICS: 'otlp_spans:1:1,flattened_spans:1:1'
+
+    healthcheck:
+      # test: ["CMD", "kafka-topics.sh", "--create", "--topic", "otlp_spans", "--zookeeper", "zookeeper:2181"]
+      test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
+      interval: 30s
+      timeout: 10s
+      retries: 4
+    depends_on:
+      - zookeeper
+
+  postgres:
+    container_name: postgres
+    image: postgres:latest
+    volumes:
+      - metadata_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+
+  coordinator:
+    image: apache/druid:0.20.0
+    container_name: coordinator
+    volumes:
+      - ./storage:/opt/data
+      - coordinator_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+    ports:
+      - "8081:8081"
+    command:
+      - coordinator
+    env_file:
+      - environment
+
+  broker:
+    image: apache/druid:0.20.0
+    container_name: broker
+    volumes:
+      - broker_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8082:8082"
+    command:
+      - broker
+    env_file:
+      - environment
+
+  historical:
+    image: apache/druid:0.20.0
+    container_name: historical
+    volumes:
+      - ./storage:/opt/data
+      - historical_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8083:8083"
+    command:
+      - historical
+    env_file:
+      - environment
+
+  middlemanager:
+    image: apache/druid:0.20.0
+    container_name: middlemanager
+    volumes:
+      - ./storage:/opt/data
+      - middle_var:/opt/druid/var
+    depends_on: 
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8091:8091"
+    command:
+      - middleManager
+    env_file:
+      - environment
+
+  router:
+    image: apache/druid:0.20.0
+    container_name: router
+    volumes:
+      - router_var:/opt/druid/var
+    depends_on:
+      - zookeeper
+      - postgres
+      - coordinator
+    ports:
+      - "8888:8888"
+    command:
+      - router
+    env_file:
+      - environment
+
+
+  flatten-processor:
+    image: signoz/flattener-processor:0.1.1
+    container_name: flattener-processor
+
+    depends_on:
+      - kafka
+      - otel-collector
+    ports:
+      - "8000:8000"
+
+    environment:
+      - KAFKA_BROKER=kafka:9092
+      - KAFKA_INPUT_TOPIC=otlp_spans
+      - KAFKA_OUTPUT_TOPIC=flattened_spans
+    
+
+  query-service:
+    image: signoz/query-service:0.1.3
+    container_name: query-service
+
+    depends_on:
+      - router
+    ports:
+      - "8080:8080"
+
+    environment:
+      - DruidClientUrl=http://router:8888
+      - DruidDatasource=flattened_spans
+      - POSTHOG_API_KEY=H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w
+
+    
+  frontend:
+    image: signoz/frontend:0.1.7
+    container_name: frontend
+
+    depends_on:
+      - query-service
+    links:
+      - "query-service"
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./nginx-config.conf:/etc/nginx/conf.d/default.conf
+
+  create-supervisor:
+    image: theithollow/hollowapp-blog:curl
+    container_name: create-supervisor
+    command: 
+    - /bin/sh 
+    - -c 
+    - "curl -X POST -H 'Content-Type: application/json' -d @/app/supervisor-spec.json http://router:8888/druid/indexer/v1/supervisor"
+
+    depends_on:
+      - router
+    volumes:
+      - ./druid-jobs/supervisor-spec.json:/app/supervisor-spec.json
+
+
+  set-retention:
+    image: theithollow/hollowapp-blog:curl
+    container_name: set-retention
+    command: 
+    - /bin/sh 
+    - -c 
+    - "curl -X POST -H 'Content-Type: application/json' -d @/app/retention-spec.json http://router:8888/druid/coordinator/v1/rules/flattened_spans"
+    
+    depends_on:
+      - router
+    volumes:
+      - ./druid-jobs/retention-spec.json:/app/retention-spec.json
+
+  otel-collector:
+    image: otel/opentelemetry-collector-dev:latest
+    command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "1777:1777"   # pprof extension
+      - "8887:8888"   # Prometheus metrics exposed by the agent
+      - "14268:14268"       # Jaeger receiver
+      - "55678"       # OpenCensus receiver
+      - "55679:55679" # zpages extension
+      - "13133"       # health_check
+    depends_on: 
+      kafka:
+        condition: service_healthy
+
+
+  hotrod:
+    image: jaegertracing/example-hotrod:latest
+    container_name: hotrod
+    ports: 
+    - "9000:8080"
+    command: ["all"]
+    environment:
+    - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces
+
+
+  load-hotrod:
+    image: "grubykarol/locust:1.2.3-python3.9-alpine3.12"
+    container_name: load-hotrod
+    hostname: load-hotrod
+    ports:
+     - "8089:8089"
+    environment:
+      ATTACKED_HOST: http://hotrod:8080
+      LOCUST_MODE: standalone
+      NO_PROXY: standalone
+      TASK_DELAY_FROM: 5
+      TASK_DELAY_TO: 30
+      QUIET_MODE: "${QUIET_MODE:-false}"
+      LOCUST_OPTS: "--headless -u 10 -r 1"
+    volumes:
+      - ./locust-scripts:/locust
+

deploy/docker/druid-jobs/retention-spec.json

@@ -0,0 +1 @@
+[{"period":"P3D","includeFuture":true,"tieredReplicants":{"_default_tier":1},"type":"loadByPeriod"},{"type":"dropForever"}]

deploy/docker/druid-jobs/supervisor-spec.json

@@ -0,0 +1,53 @@
+{
+    "type": "kafka",
+    "dataSchema": {
+      "dataSource": "flattened_spans",
+      "parser": {
+        "type": "string",
+        "parseSpec": {
+          "format": "json",
+          "timestampSpec": {
+            "column": "StartTimeUnixNano",
+            "format": "nano"
+          },
+          "dimensionsSpec": {
+            "dimensions": [
+              "TraceId",
+              "SpanId",
+              "ParentSpanId",
+              "Name",
+              "ServiceName",
+              "References",
+              "Tags",
+              "TagsKeys",
+              "TagsValues",
+              { "name": "DurationNano", "type": "Long" },
+              { "name": "Kind", "type": "int" }
+            ]
+          }
+        }
+      },
+      "metricsSpec" : [
+          { "type": "quantilesDoublesSketch", "name": "QuantileDuration", "fieldName": "DurationNano" }
+      ],
+      "granularitySpec": {
+        "type": "uniform",
+        "segmentGranularity": "DAY",
+        "queryGranularity": "NONE",
+        "rollup": false
+      }
+    },
+    "tuningConfig": {
+      "type": "kafka",
+      "reportParseExceptions": true
+    },
+    "ioConfig": {
+      "topic": "flattened_spans",
+      "replicas": 1,
+      "taskDuration": "PT20M",
+      "completionTimeout": "PT30M",
+      "consumerProperties": {
+        "bootstrap.servers": "kafka:9092"
+      }
+    }
+  }

deploy/docker/environment

@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+DRUID_XMX=1g
+DRUID_XMS=1g
+DRUID_MAXNEWSIZE=250m
+DRUID_NEWSIZE=250m
+DRUID_MAXDIRECTMEMORYSIZE=6172m
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"]
+
+druid_zk_service_host=zookeeper
+
+druid_metadata_storage_host=
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/data/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/data/indexing-logs
+
+druid_processing_numThreads=2
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

deploy/docker/locust-scripts/locustfile.py

@@ -0,0 +1,16 @@
+from locust import HttpUser, task, between
+class UserTasks(HttpUser):
+    wait_time = between(5, 15)
+
+    @task
+    def rachel(self):
+        self.client.get("/dispatch?customer=123&nonse=0.6308392664170006")
+    @task
+    def trom(self):
+        self.client.get("/dispatch?customer=392&nonse=0.015296363321630757")
+    @task
+    def japanese(self):
+        self.client.get("/dispatch?customer=731&nonse=0.8022286220408668")
+    @task
+    def coffee(self):
+        self.client.get("/dispatch?customer=567&nonse=0.0022220379420636593")

deploy/docker/nginx-config.conf

@@ -0,0 +1,20 @@
+server {
+    listen       3000;
+    server_name  _;
+    location / {
+    root   /usr/share/nginx/html;
+    index  index.html index.htm;
+    try_files $uri $uri/ /index.html;
+    }
+    location /api {
+    proxy_pass http://query-service:8080/api;
+    
+    }
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+    root   /usr/share/nginx/html;
+    }
+}

deploy/docker/otel-collector-config.yaml

@@ -0,0 +1,34 @@
+receivers:
+  jaeger:
+    protocols:
+      grpc:
+      thrift_http:
+processors:
+  batch:
+  memory_limiter:
+    # Same as --mem-ballast-size-mib CLI argument
+    ballast_size_mib: 683
+    # 80% of maximum memory up to 2G
+    limit_mib: 1500
+    # 25% of limit up to 2G
+    spike_limit_mib: 512
+    check_interval: 5s
+  queued_retry:
+    num_workers: 4
+    queue_size: 100
+    retry_on_failure: true
+extensions:
+  health_check: {}
+  zpages: {}
+exporters:
+  kafka:
+    brokers:
+      - kafka:9092
+    protocol_version: 2.0.0
+service:
+  extensions: [health_check, zpages]
+  pipelines:
+    traces:
+      receivers: [jaeger]
+      processors: [memory_limiter, batch, queued_retry]
+      exporters: [kafka]