From 9d53fc6055e7988ba28960a7d927c2827fd08a8d Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Thu, 1 Apr 2021 20:11:33 +0530 Subject: [PATCH] adding separate s3 configs --- .../docker/docker-compose-tiny-with-s3.yaml | 264 ------------------ deploy/docker/docker-compose-tiny.yaml | 16 +- deploy/docker/environment_tiny/common | 34 ++- deploy/docker/environment_tiny_local/broker | 53 ---- .../docker/environment_tiny_local/coordinator | 52 ---- .../docker/environment_tiny_local/historical | 53 ---- .../environment_tiny_local/middlemanager | 52 ---- deploy/docker/environment_tiny_local/router | 52 ---- 8 files changed, 27 insertions(+), 549 deletions(-) delete mode 100644 deploy/docker/docker-compose-tiny-with-s3.yaml delete mode 100644 deploy/docker/environment_tiny_local/broker delete mode 100644 deploy/docker/environment_tiny_local/coordinator delete mode 100644 deploy/docker/environment_tiny_local/historical delete mode 100644 deploy/docker/environment_tiny_local/middlemanager delete mode 100644 deploy/docker/environment_tiny_local/router diff --git a/deploy/docker/docker-compose-tiny-with-s3.yaml b/deploy/docker/docker-compose-tiny-with-s3.yaml deleted file mode 100644 index f0b9801686..0000000000 --- a/deploy/docker/docker-compose-tiny-with-s3.yaml +++ /dev/null @@ -1,264 +0,0 @@ -version: "2.4" - -volumes: - metadata_data: {} - middle_var: {} - historical_var: {} - broker_var: {} - coordinator_var: {} - router_var: {} - -# If able to connect to kafka but not able to write to topic otlp_spans look into below link -# https://github.com/wurstmeister/kafka-docker/issues/409#issuecomment-428346707 - -services: - - zookeeper: - image: bitnami/zookeeper:3.6.2-debian-10-r100 - ports: - - "2181:2181" - environment: - - ALLOW_ANONYMOUS_LOGIN=yes - - - kafka: - # image: wurstmeister/kafka - image: bitnami/kafka:2.7.0-debian-10-r1 - ports: - - "9092:9092" - hostname: kafka - environment: - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_ADVERTISED_PORT: 9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true' - KAFKA_TOPICS: 'otlp_spans:1:1,flattened_spans:1:1' - - healthcheck: - # test: ["CMD", "kafka-topics.sh", "--create", "--topic", "otlp_spans", "--zookeeper", "zookeeper:2181"] - test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] - interval: 30s - timeout: 10s - retries: 10 - depends_on: - - zookeeper - - postgres: - container_name: postgres - image: postgres:latest - volumes: - - metadata_data:/var/lib/postgresql/data - environment: - - POSTGRES_PASSWORD=FoolishPassword - - POSTGRES_USER=druid - - POSTGRES_DB=druid - - coordinator: - image: apache/druid:0.20.0 - container_name: coordinator - volumes: - - ./storage:/opt/druid/deepStorage - - coordinator_var:/opt/druid/data - depends_on: - - zookeeper - - postgres - ports: - - "8081:8081" - command: - - coordinator - env_file: - - environment_tiny_s3/coordinator - - environment_tiny_s3/s3_common - - broker: - image: apache/druid:0.20.0 - container_name: broker - volumes: - - broker_var:/opt/druid/data - depends_on: - - zookeeper - - postgres - - coordinator - ports: - - "8082:8082" - command: - - broker - env_file: - - environment_tiny_s3/broker - - environment_tiny_s3/s3_common - - historical: - image: apache/druid:0.20.0 - container_name: historical - volumes: - - ./storage:/opt/druid/deepStorage - - historical_var:/opt/druid/data - depends_on: - - zookeeper - - postgres - - coordinator - ports: - - "8083:8083" - command: - - historical - env_file: - - environment_tiny_s3/historical - - environment_tiny_s3/s3_common - - middlemanager: - image: apache/druid:0.20.0 - container_name: middlemanager - volumes: - - ./storage:/opt/druid/deepStorage - - middle_var:/opt/druid/data - depends_on: - - zookeeper - - postgres - - coordinator - ports: - - "8091:8091" - command: - - middleManager - env_file: - - environment_tiny_s3/middlemanager - - environment_tiny_s3/s3_common - - router: - image: apache/druid:0.20.0 - container_name: router - volumes: - - router_var:/opt/druid/data - depends_on: - - zookeeper - - postgres - - coordinator - ports: - - "8888:8888" - command: - - router - env_file: - - environment_tiny_s3/router - - environment_tiny_s3/s3_common - - flatten-processor: - image: signoz/flattener-processor:0.1.1 - container_name: flattener-processor - - depends_on: - - kafka - - otel-collector - ports: - - "8000:8000" - - environment: - - KAFKA_BROKER=kafka:9092 - - KAFKA_INPUT_TOPIC=otlp_spans - - KAFKA_OUTPUT_TOPIC=flattened_spans - - - query-service: - image: signoz.docker.scarf.sh/signoz/query-service:0.1.4 - container_name: query-service - - depends_on: - - router - ports: - - "8080:8080" - - environment: - - DruidClientUrl=http://router:8888 - - DruidDatasource=flattened_spans - - POSTHOG_API_KEY=H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w - - - frontend: - image: signoz/frontend:0.2.0 - container_name: frontend - - depends_on: - - query-service - links: - - "query-service" - ports: - - "3000:3000" - volumes: - - ./nginx-config.conf:/etc/nginx/conf.d/default.conf - - create-supervisor: - image: theithollow/hollowapp-blog:curl - container_name: create-supervisor - command: - - /bin/sh - - -c - - "curl -X POST -H 'Content-Type: application/json' -d @/app/supervisor-spec.json http://router:8888/druid/indexer/v1/supervisor" - - depends_on: - - router - restart: on-failure:6 - - volumes: - - ./druid-jobs/supervisor-spec.json:/app/supervisor-spec.json - - - set-retention: - image: theithollow/hollowapp-blog:curl - container_name: set-retention - command: - - /bin/sh - - -c - - "curl -X POST -H 'Content-Type: application/json' -d @/app/retention-spec.json http://router:8888/druid/coordinator/v1/rules/flattened_spans" - - depends_on: - - router - restart: on-failure:6 - volumes: - - ./druid-jobs/retention-spec.json:/app/retention-spec.json - - otel-collector: - image: otel/opentelemetry-collector:0.18.0 - command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"] - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ports: - - "1777:1777" # pprof extension - - "8887:8888" # Prometheus metrics exposed by the agent - - "14268:14268" # Jaeger receiver - - "55678" # OpenCensus receiver - - "55680:55680" # OTLP HTTP/2.0 legacy port - - "55681:55681" # OTLP HTTP/1.0 receiver - - "4317:4317" # OTLP GRPC receiver - - "55679:55679" # zpages extension - - "13133" # health_check - depends_on: - kafka: - condition: service_healthy - - - hotrod: - image: jaegertracing/example-hotrod:latest - container_name: hotrod - ports: - - "9000:8080" - command: ["all"] - environment: - - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - - - load-hotrod: - image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" - container_name: load-hotrod - hostname: load-hotrod - ports: - - "8089:8089" - environment: - ATTACKED_HOST: http://hotrod:8080 - LOCUST_MODE: standalone - NO_PROXY: standalone - TASK_DELAY_FROM: 5 - TASK_DELAY_TO: 30 - QUIET_MODE: "${QUIET_MODE:-false}" - LOCUST_OPTS: "--headless -u 10 -r 1" - volumes: - - ./locust-scripts:/locust - diff --git a/deploy/docker/docker-compose-tiny.yaml b/deploy/docker/docker-compose-tiny.yaml index a9183e5a56..04d1535919 100644 --- a/deploy/docker/docker-compose-tiny.yaml +++ b/deploy/docker/docker-compose-tiny.yaml @@ -58,8 +58,8 @@ services: image: apache/druid:0.20.0 container_name: coordinator volumes: - - ./storage:/opt/druid/deepStorage - - coordinator_var:/opt/druid/data + - ./storage:/opt/data + - coordinator_var:/opt/druid/var depends_on: - zookeeper - postgres @@ -75,7 +75,7 @@ services: image: apache/druid:0.20.0 container_name: broker volumes: - - broker_var:/opt/druid/data + - broker_var:/opt/druid/var depends_on: - zookeeper - postgres @@ -92,8 +92,8 @@ services: image: apache/druid:0.20.0 container_name: historical volumes: - - ./storage:/opt/druid/deepStorage - - historical_var:/opt/druid/data + - ./storage:/opt/data + - historical_var:/opt/druid/var depends_on: - zookeeper - postgres @@ -110,8 +110,8 @@ services: image: apache/druid:0.20.0 container_name: middlemanager volumes: - - ./storage:/opt/druid/deepStorage - - middle_var:/opt/druid/data + - ./storage:/opt/data + - middle_var:/opt/druid/var depends_on: - zookeeper - postgres @@ -128,7 +128,7 @@ services: image: apache/druid:0.20.0 container_name: router volumes: - - router_var:/opt/druid/data + - router_var:/opt/druid/var depends_on: - zookeeper - postgres diff --git a/deploy/docker/environment_tiny/common b/deploy/docker/environment_tiny/common index ef36b55932..07dd771ffe 100644 --- a/deploy/docker/environment_tiny/common +++ b/deploy/docker/environment_tiny/common @@ -1,22 +1,26 @@ -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-s3-extensions"] +# For S3 storage + +# druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-s3-extensions"] -druid_storage_type=s3 -druid_storage_bucket=solvzy-test3 -druid_storage_baseKey=druid/segments +# druid_storage_type=s3 +# druid_storage_bucket=signoz +# druid_storage_baseKey=druid/segments -AWS_ACCESS_KEY_ID=AKIARKCF5OX3CMI3XRXC -AWS_SECRET_ACCESS_KEY=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 -AWS_REGION=us-east-2 +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# AWS_REGION=us-east-2 -druid_indexer_logs_type=s3 -druid_indexer_logs_s3Bucket=solvzy-test3 -druid_indexer_logs_s3Prefix=druid/indexing-logs +# druid_indexer_logs_type=s3 +# druid_indexer_logs_s3Bucket=signoz +# druid_indexer_logs_s3Prefix=druid/indexing-logs +# ----------------------------------------------------------- +# For local storage +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] -# druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] -# druid_storage_type=local -# druid_storage_storageDirectory=/opt/druid/deepStorage -# druid_indexer_logs_type=file -# druid_indexer_logs_directory=/opt/druid/data/indexing-logs +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs diff --git a/deploy/docker/environment_tiny_local/broker b/deploy/docker/environment_tiny_local/broker deleted file mode 100644 index 1c22ac36a7..0000000000 --- a/deploy/docker/environment_tiny_local/broker +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Java tuning -DRUID_XMX=512m -DRUID_XMS=512m -DRUID_MAXNEWSIZE=256m -DRUID_NEWSIZE=256m -DRUID_MAXDIRECTMEMORYSIZE=400m - -druid_emitter_logging_logLevel=debug - -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -druid_zk_service_host=zookeeper - -druid_metadata_storage_host= -druid_metadata_storage_type=postgresql -druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=druid -druid_metadata_storage_connector_password=FoolishPassword - -druid_coordinator_balancer_strategy=cachingCost - -druid_indexer_runner_javaOptsArray=["-server", "-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 -druid_processing_buffer_sizeBytes=50MiB - -druid_storage_type=local -druid_storage_storageDirectory=/opt/druid/deepStorage -druid_indexer_logs_type=file -druid_indexer_logs_directory=/opt/druid/data/indexing-logs - -druid_processing_numThreads=1 -druid_processing_numMergeBuffers=2 - -DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_tiny_local/coordinator b/deploy/docker/environment_tiny_local/coordinator deleted file mode 100644 index 476e6dcdef..0000000000 --- a/deploy/docker/environment_tiny_local/coordinator +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Java tuning -DRUID_XMX=64m -DRUID_XMS=64m -DRUID_MAXNEWSIZE=256m -DRUID_NEWSIZE=256m -DRUID_MAXDIRECTMEMORYSIZE=400m - -druid_emitter_logging_logLevel=debug - -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -druid_zk_service_host=zookeeper - -druid_metadata_storage_host= -druid_metadata_storage_type=postgresql -druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=druid -druid_metadata_storage_connector_password=FoolishPassword - -druid_coordinator_balancer_strategy=cachingCost - -druid_indexer_runner_javaOptsArray=["-server", "-Xms64m", "-Xmx64m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 - -druid_storage_type=local -druid_storage_storageDirectory=/opt/druid/deepStorage -druid_indexer_logs_type=file -druid_indexer_logs_directory=/opt/druid/data/indexing-logs - -druid_processing_numThreads=1 -druid_processing_numMergeBuffers=2 - -DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_tiny_local/historical b/deploy/docker/environment_tiny_local/historical deleted file mode 100644 index 1c22ac36a7..0000000000 --- a/deploy/docker/environment_tiny_local/historical +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Java tuning -DRUID_XMX=512m -DRUID_XMS=512m -DRUID_MAXNEWSIZE=256m -DRUID_NEWSIZE=256m -DRUID_MAXDIRECTMEMORYSIZE=400m - -druid_emitter_logging_logLevel=debug - -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -druid_zk_service_host=zookeeper - -druid_metadata_storage_host= -druid_metadata_storage_type=postgresql -druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=druid -druid_metadata_storage_connector_password=FoolishPassword - -druid_coordinator_balancer_strategy=cachingCost - -druid_indexer_runner_javaOptsArray=["-server", "-Xms512m", "-Xmx512m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 -druid_processing_buffer_sizeBytes=50MiB - -druid_storage_type=local -druid_storage_storageDirectory=/opt/druid/deepStorage -druid_indexer_logs_type=file -druid_indexer_logs_directory=/opt/druid/data/indexing-logs - -druid_processing_numThreads=1 -druid_processing_numMergeBuffers=2 - -DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_tiny_local/middlemanager b/deploy/docker/environment_tiny_local/middlemanager deleted file mode 100644 index 9b244a4a7b..0000000000 --- a/deploy/docker/environment_tiny_local/middlemanager +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Java tuning -DRUID_XMX=64m -DRUID_XMS=64m -DRUID_MAXNEWSIZE=256m -DRUID_NEWSIZE=256m -DRUID_MAXDIRECTMEMORYSIZE=400m - -druid_emitter_logging_logLevel=debug - -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -druid_zk_service_host=zookeeper - -druid_metadata_storage_host= -druid_metadata_storage_type=postgresql -druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=druid -druid_metadata_storage_connector_password=FoolishPassword - -druid_coordinator_balancer_strategy=cachingCost - -druid_indexer_runner_javaOptsArray=["-server", "-Xms256m", "-Xmx256m", "-XX:MaxDirectMemorySize=400m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 - -druid_storage_type=local -druid_storage_storageDirectory=/opt/druid/deepStorage -druid_indexer_logs_type=file -druid_indexer_logs_directory=/opt/druid/data/indexing-logs - -druid_processing_numThreads=1 -druid_processing_numMergeBuffers=2 - -DRUID_LOG4J= \ No newline at end of file diff --git a/deploy/docker/environment_tiny_local/router b/deploy/docker/environment_tiny_local/router deleted file mode 100644 index 32ce129cbd..0000000000 --- a/deploy/docker/environment_tiny_local/router +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Java tuning -DRUID_XMX=64m -DRUID_XMS=64m -DRUID_MAXNEWSIZE=256m -DRUID_NEWSIZE=256m -DRUID_MAXDIRECTMEMORYSIZE=128m - -druid_emitter_logging_logLevel=debug - -druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] - -druid_zk_service_host=zookeeper - -druid_metadata_storage_host= -druid_metadata_storage_type=postgresql -druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres:5432/druid -druid_metadata_storage_connector_user=druid -druid_metadata_storage_connector_password=FoolishPassword - -druid_coordinator_balancer_strategy=cachingCost - -druid_indexer_runner_javaOptsArray=["-server", "-Xms64m", "-Xmx64m", "-XX:MaxDirectMemorySize=128m", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] -druid_indexer_fork_property_druid_processing_buffer_sizeBytes=25000000 - -druid_storage_type=local -druid_storage_storageDirectory=/opt/druid/deepStorage -druid_indexer_logs_type=file -druid_indexer_logs_directory=/opt/druid/data/indexing-logs - -druid_processing_numThreads=1 -druid_processing_numMergeBuffers=2 - -DRUID_LOG4J= \ No newline at end of file