diff --git a/deploy/docker/docker-compose-tiny.yaml b/deploy/docker/docker-compose-tiny.yaml new file mode 100644 index 0000000000..312f75d04e --- /dev/null +++ b/deploy/docker/docker-compose-tiny.yaml @@ -0,0 +1,256 @@ +version: "2.4" + +volumes: + metadata_data: {} + middle_var: {} + historical_var: {} + broker_var: {} + coordinator_var: {} + router_var: {} + +# If able to connect to kafka but not able to write to topic otlp_spans look into below link +# https://github.com/wurstmeister/kafka-docker/issues/409#issuecomment-428346707 + +services: + + zookeeper: + image: bitnami/zookeeper:3.6.2-debian-10-r100 + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + + kafka: + # image: wurstmeister/kafka + image: bitnami/kafka:2.7.0-debian-10-r1 + ports: + - "9092:9092" + hostname: kafka + environment: + KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_TOPICS: 'otlp_spans:1:1,flattened_spans:1:1' + + healthcheck: + # test: ["CMD", "kafka-topics.sh", "--create", "--topic", "otlp_spans", "--zookeeper", "zookeeper:2181"] + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 30s + timeout: 10s + retries: 4 + depends_on: + - zookeeper + + postgres: + container_name: postgres + image: postgres:latest + volumes: + - metadata_data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=FoolishPassword + - POSTGRES_USER=druid + - POSTGRES_DB=druid + + coordinator: + image: apache/druid:0.20.0 + container_name: coordinator + volumes: + - ./storage:/opt/data + - coordinator_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + ports: + - "8081:8081" + command: + - coordinator + env_file: + - environment_nano/coordinator + + broker: + image: apache/druid:0.20.0 + container_name: broker + volumes: + - broker_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8082:8082" + command: + - broker + env_file: + - environment_nano/broker + + historical: + image: apache/druid:0.20.0 + container_name: historical + volumes: + - ./storage:/opt/data + - historical_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8083:8083" + command: + - historical + env_file: + - environment_nano/historical + + middlemanager: + image: apache/druid:0.20.0 + container_name: middlemanager + volumes: + - ./storage:/opt/data + - middle_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8091:8091" + command: + - middleManager + env_file: + - environment_nano/middlemanager + + router: + image: apache/druid:0.20.0 + container_name: router + volumes: + - router_var:/opt/druid/var + depends_on: + - zookeeper + - postgres + - coordinator + ports: + - "8888:8888" + command: + - router + env_file: + - environment_nano/router + + flatten-processor: + image: signoz/flattener-processor:0.1.1 + container_name: flattener-processor + + depends_on: + - kafka + - otel-collector + ports: + - "8000:8000" + + environment: + - KAFKA_BROKER=kafka:9092 + - KAFKA_INPUT_TOPIC=otlp_spans + - KAFKA_OUTPUT_TOPIC=flattened_spans + + + query-service: + image: signoz/query-service:0.1.3 + container_name: query-service + + depends_on: + - router + ports: + - "8080:8080" + + environment: + - DruidClientUrl=http://router:8888 + - DruidDatasource=flattened_spans + - POSTHOG_API_KEY=H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w + + + frontend: + image: signoz/frontend:0.1.7 + container_name: frontend + + depends_on: + - query-service + links: + - "query-service" + ports: + - "3000:3000" + volumes: + - ./nginx-config.conf:/etc/nginx/conf.d/default.conf + + create-supervisor: + image: theithollow/hollowapp-blog:curl + container_name: create-supervisor + command: + - /bin/sh + - -c + - "curl -X POST -H 'Content-Type: application/json' -d @/app/supervisor-spec.json http://router:8888/druid/indexer/v1/supervisor" + + depends_on: + - router + restart: on-failure:6 + + volumes: + - ./druid-jobs/supervisor-spec.json:/app/supervisor-spec.json + + + set-retention: + image: theithollow/hollowapp-blog:curl + container_name: set-retention + command: + - /bin/sh + - -c + - "curl -X POST -H 'Content-Type: application/json' -d @/app/retention-spec.json http://router:8888/druid/coordinator/v1/rules/flattened_spans" + + depends_on: + - router + restart: on-failure:6 + volumes: + - ./druid-jobs/retention-spec.json:/app/retention-spec.json + + otel-collector: + image: otel/opentelemetry-collector-dev:latest + command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1777:1777" # pprof extension + - "8887:8888" # Prometheus metrics exposed by the agent + - "14268:14268" # Jaeger receiver + - "55678" # OpenCensus receiver + - "55681:55681" # OTLP HTTP receiver + - "55679:55679" # zpages extension + - "13133" # health_check + depends_on: + kafka: + condition: service_healthy + + + hotrod: + image: jaegertracing/example-hotrod:latest + container_name: hotrod + ports: + - "9000:8080" + command: ["all"] + environment: + - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces + + load-hotrod: + image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" + container_name: load-hotrod + hostname: load-hotrod + ports: + - "8089:8089" + environment: + ATTACKED_HOST: http://hotrod:8080 + LOCUST_MODE: standalone + NO_PROXY: standalone + TASK_DELAY_FROM: 5 + TASK_DELAY_TO: 30 + QUIET_MODE: "${QUIET_MODE:-false}" + LOCUST_OPTS: "--headless -u 10 -r 1" + volumes: + - ./locust-scripts:/locust + diff --git a/deploy/docker/environment b/deploy/docker/environment index 08b12a1bc0..4ccc70dab7 100644 --- a/deploy/docker/environment +++ b/deploy/docker/environment @@ -22,7 +22,7 @@ DRUID_XMX=1g DRUID_XMS=1g DRUID_MAXNEWSIZE=250m DRUID_NEWSIZE=250m -DRUID_MAXDIRECTMEMORYSIZE=6172m +DRUID_MAXDIRECTMEMORYSIZE=3072m druid_emitter_logging_logLevel=debug diff --git a/deploy/docker/environment_nano/broker b/deploy/docker/environment_nano/broker new file mode 100644 index 0000000000..6a45c747a9 --- /dev/null +++ b/deploy/docker/environment_nano/broker @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=512m +DRUID_XMS=512m +DRUID_MAXNEWSIZE=256m +DRUID_NEWSIZE=256m +DRUID_MAXDIRECTMEMORYSIZE=400m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 +druid_processing_buffer_sizeBytes=50MiB + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_nano/coordinator b/deploy/docker/environment_nano/coordinator new file mode 100644 index 0000000000..25ec3847ba --- /dev/null +++ b/deploy/docker/environment_nano/coordinator @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=64m +DRUID_XMS=64m +DRUID_MAXNEWSIZE=256m +DRUID_NEWSIZE=256m +DRUID_MAXDIRECTMEMORYSIZE=400m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xms64m", "-Xmx64m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_nano/historical b/deploy/docker/environment_nano/historical new file mode 100644 index 0000000000..6a45c747a9 --- /dev/null +++ b/deploy/docker/environment_nano/historical @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=512m +DRUID_XMS=512m +DRUID_MAXNEWSIZE=256m +DRUID_NEWSIZE=256m +DRUID_MAXDIRECTMEMORYSIZE=400m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 +druid_processing_buffer_sizeBytes=50MiB + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_nano/middlemanager b/deploy/docker/environment_nano/middlemanager new file mode 100644 index 0000000000..d170371f0e --- /dev/null +++ b/deploy/docker/environment_nano/middlemanager @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=64m +DRUID_XMS=64m +DRUID_MAXNEWSIZE=256m +DRUID_NEWSIZE=256m +DRUID_MAXDIRECTMEMORYSIZE=400m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_nano/router b/deploy/docker/environment_nano/router new file mode 100644 index 0000000000..b37dabeed8 --- /dev/null +++ b/deploy/docker/environment_nano/router @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Java tuning +DRUID_XMX=64m +DRUID_XMS=64m +DRUID_MAXNEWSIZE=256m +DRUID_NEWSIZE=256m +DRUID_MAXDIRECTMEMORYSIZE=128m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host= +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=FoolishPassword + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xms64m", "-Xmx64m", "-XX:MaxDirectMemorySize=128m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=1 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J= \ No newline at end of file