From 362f264baecb308355985a9a81e6c6b319e723c8 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Thu, 3 Jun 2021 20:54:41 +0530 Subject: [PATCH] Installation changes for docker (#149) * installation steps WIP * changing install.sh * fixes * fixes * fixes * handled enter key press in setup_type * fixes * fixes * fixes Co-authored-by: Ankit Anand --- .../clickhouse-setup/clickhouse-config.xml | 517 ++++++++++++++++++ .../clickhouse-setup/docker-compose.yaml | 97 ++++ .../docker-entrypoint-initdb.d/init-db.sql | 27 + .../otel-collector-config.yaml | 39 ++ .../{ => common}/locust-scripts/locustfile.py | 0 deploy/docker/{ => common}/nginx-config.conf | 0 .../docker-compose-tiny.yaml | 4 +- .../docker-compose.yaml | 0 .../druid-jobs/retention-spec.json | 0 .../druid-jobs/supervisor-spec.json | 0 .../environment_small/broker | 0 .../environment_small/coordinator | 0 .../environment_small/historical | 0 .../environment_small/middlemanager | 0 .../environment_small/router | 0 .../environment_tiny/broker | 0 .../environment_tiny/common | 0 .../environment_tiny/common_test | 26 + .../environment_tiny/coordinator | 0 .../environment_tiny/historical | 0 .../environment_tiny/middlemanager | 0 .../environment_tiny/router | 0 .../otel-collector-config.yaml | 0 deploy/install.k8s.sh | 256 --------- deploy/install.sh | 168 +++--- 25 files changed, 809 insertions(+), 325 deletions(-) create mode 100644 deploy/docker/clickhouse-setup/clickhouse-config.xml create mode 100644 deploy/docker/clickhouse-setup/docker-compose.yaml create mode 100644 deploy/docker/clickhouse-setup/docker-entrypoint-initdb.d/init-db.sql create mode 100644 deploy/docker/clickhouse-setup/otel-collector-config.yaml rename deploy/docker/{ => common}/locust-scripts/locustfile.py (100%) rename deploy/docker/{ => common}/nginx-config.conf (100%) rename deploy/docker/{ => druid-kafka-setup}/docker-compose-tiny.yaml (98%) rename deploy/docker/{ => druid-kafka-setup}/docker-compose.yaml (100%) rename deploy/docker/{ => druid-kafka-setup}/druid-jobs/retention-spec.json (100%) rename deploy/docker/{ => druid-kafka-setup}/druid-jobs/supervisor-spec.json (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_small/broker (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_small/coordinator (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_small/historical (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_small/middlemanager (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_small/router (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_tiny/broker (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_tiny/common (100%) create mode 100644 deploy/docker/druid-kafka-setup/environment_tiny/common_test rename deploy/docker/{ => druid-kafka-setup}/environment_tiny/coordinator (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_tiny/historical (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_tiny/middlemanager (100%) rename deploy/docker/{ => druid-kafka-setup}/environment_tiny/router (100%) rename deploy/docker/{ => druid-kafka-setup}/otel-collector-config.yaml (100%) delete mode 100644 deploy/install.k8s.sh diff --git a/deploy/docker/clickhouse-setup/clickhouse-config.xml b/deploy/docker/clickhouse-setup/clickhouse-config.xml new file mode 100644 index 0000000000..23898ef5e7 --- /dev/null +++ b/deploy/docker/clickhouse-setup/clickhouse-config.xml @@ -0,0 +1,517 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + +9009 + + + + + +:: + + + + + + +4096 +3 + + +100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + users.xml + + + default + + + default + + + + + + + + + + + + + + localhost + 9000 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + ^carbon\. + any + + 0 + 60 + + + 7776000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^collectd\. + any + + 0 + 10 + + + 43200 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^high\. + any + + 0 + 10 + + + 172800 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^medium\. + any + + 0 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^low\. + any + + 0 + 600 + + + 15552000 + 1800 + + + 31536000 + 3600 + + + 63072000 + 21600 + + + 126144000 + 43200 + + + 252288000 + 86400 + + + 315360000 + 604800 + + + + + any + + 0 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + + /var/lib/clickhouse/format_schemas/ + diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml new file mode 100644 index 0000000000..bd71734bf3 --- /dev/null +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -0,0 +1,97 @@ +version: "2.4" + +services: + clickhouse: + image: yandex/clickhouse-server + expose: + - 8123 + - 9000 + ports: + - 9001:9000 + - 8123:8123 + volumes: + - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml + - ./docker-entrypoint-initdb.d/init-db.sql:/docker-entrypoint-initdb.d/init-db.sql + healthcheck: + # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" + test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + interval: 30s + timeout: 5s + retries: 3 + + query-service: + image: signoz/query-service:0.3.0 + container_name: query-service + + ports: + - "8080:8080" + + environment: + - ClickHouseUrl=tcp://clickhouse:9000 + - STORAGE=clickhouse + - POSTHOG_API_KEY=H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w + + depends_on: + clickhouse: + condition: service_healthy + + frontend: + image: signoz/frontend:0.3.0 + container_name: frontend + + depends_on: + - query-service + links: + - "query-service" + ports: + - "3000:3000" + volumes: + - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf + + + otel-collector: + image: signoz/otelcol:latest + command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1777:1777" # pprof extension + - "8887:8888" # Prometheus metrics exposed by the agent + - "14268:14268" # Jaeger receiver + - "55678" # OpenCensus receiver + - "55680:55680" # OTLP HTTP/2.0 legacy port + - "55681:55681" # OTLP HTTP/1.0 receiver + - "4317:4317" # OTLP GRPC receiver + - "55679:55679" # zpages extension + - "13133" # health_check + + depends_on: + clickhouse: + condition: service_healthy + + hotrod: + image: jaegertracing/example-hotrod:latest + container_name: hotrod + ports: + - "9000:8080" + command: ["all"] + environment: + - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces + + + load-hotrod: + image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" + container_name: load-hotrod + hostname: load-hotrod + ports: + - "8089:8089" + environment: + ATTACKED_HOST: http://hotrod:8080 + LOCUST_MODE: standalone + NO_PROXY: standalone + TASK_DELAY_FROM: 5 + TASK_DELAY_TO: 30 + QUIET_MODE: "${QUIET_MODE:-false}" + LOCUST_OPTS: "--headless -u 10 -r 1" + volumes: + - ../common/locust-scripts:/locust \ No newline at end of file diff --git a/deploy/docker/clickhouse-setup/docker-entrypoint-initdb.d/init-db.sql b/deploy/docker/clickhouse-setup/docker-entrypoint-initdb.d/init-db.sql new file mode 100644 index 0000000000..5aa7c0f7b2 --- /dev/null +++ b/deploy/docker/clickhouse-setup/docker-entrypoint-initdb.d/init-db.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS signoz_index ( + timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), + traceID String CODEC(ZSTD(1)), + spanID String CODEC(ZSTD(1)), + parentSpanID String CODEC(ZSTD(1)), + serviceName LowCardinality(String) CODEC(ZSTD(1)), + name LowCardinality(String) CODEC(ZSTD(1)), + kind Int32 CODEC(ZSTD(1)), + durationNano UInt64 CODEC(ZSTD(1)), + tags Array(String) CODEC(ZSTD(1)), + tagsKeys Array(String) CODEC(ZSTD(1)), + tagsValues Array(String) CODEC(ZSTD(1)), + statusCode Int64 CODEC(ZSTD(1)), + references String CODEC(ZSTD(1)), + externalHttpMethod Nullable(String) CODEC(ZSTD(1)), + externalHttpUrl Nullable(String) CODEC(ZSTD(1)), + component Nullable(String) CODEC(ZSTD(1)), + dbSystem Nullable(String) CODEC(ZSTD(1)), + dbName Nullable(String) CODEC(ZSTD(1)), + dbOperation Nullable(String) CODEC(ZSTD(1)), + peerService Nullable(String) CODEC(ZSTD(1)), + INDEX idx_tagsKeys tagsKeys TYPE bloom_filter(0.01) GRANULARITY 64, + INDEX idx_tagsValues tagsValues TYPE bloom_filter(0.01) GRANULARITY 64, + INDEX idx_duration durationNano TYPE minmax GRANULARITY 1 +) ENGINE MergeTree() +PARTITION BY toDate(timestamp) +ORDER BY (serviceName, -toUnixTimestamp(timestamp)) \ No newline at end of file diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml new file mode 100644 index 0000000000..f2134d98ee --- /dev/null +++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml @@ -0,0 +1,39 @@ +receivers: + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: +processors: + batch: + send_batch_size: 1000 + timeout: 10s + memory_limiter: + # Same as --mem-ballast-size-mib CLI argument + ballast_size_mib: 683 + # 80% of maximum memory up to 2G + limit_mib: 1500 + # 25% of limit up to 2G + spike_limit_mib: 512 + check_interval: 5s + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + health_check: {} + zpages: {} +exporters: + clickhouse: + datasource: tcp://clickhouse:9000 + +service: + extensions: [health_check, zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [batch] + exporters: [clickhouse] \ No newline at end of file diff --git a/deploy/docker/locust-scripts/locustfile.py b/deploy/docker/common/locust-scripts/locustfile.py similarity index 100% rename from deploy/docker/locust-scripts/locustfile.py rename to deploy/docker/common/locust-scripts/locustfile.py diff --git a/deploy/docker/nginx-config.conf b/deploy/docker/common/nginx-config.conf similarity index 100% rename from deploy/docker/nginx-config.conf rename to deploy/docker/common/nginx-config.conf diff --git a/deploy/docker/docker-compose-tiny.yaml b/deploy/docker/druid-kafka-setup/docker-compose-tiny.yaml similarity index 98% rename from deploy/docker/docker-compose-tiny.yaml rename to deploy/docker/druid-kafka-setup/docker-compose-tiny.yaml index 51b547996c..0aef8e385a 100644 --- a/deploy/docker/docker-compose-tiny.yaml +++ b/deploy/docker/druid-kafka-setup/docker-compose-tiny.yaml @@ -192,7 +192,7 @@ services: ports: - "3000:3000" volumes: - - ./nginx-config.conf:/etc/nginx/conf.d/default.conf + - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf create-supervisor: image: theithollow/hollowapp-blog:curl @@ -269,5 +269,5 @@ services: QUIET_MODE: "${QUIET_MODE:-false}" LOCUST_OPTS: "--headless -u 10 -r 1" volumes: - - ./locust-scripts:/locust + - ../common/locust-scripts:/locust diff --git a/deploy/docker/docker-compose.yaml b/deploy/docker/druid-kafka-setup/docker-compose.yaml similarity index 100% rename from deploy/docker/docker-compose.yaml rename to deploy/docker/druid-kafka-setup/docker-compose.yaml diff --git a/deploy/docker/druid-jobs/retention-spec.json b/deploy/docker/druid-kafka-setup/druid-jobs/retention-spec.json similarity index 100% rename from deploy/docker/druid-jobs/retention-spec.json rename to deploy/docker/druid-kafka-setup/druid-jobs/retention-spec.json diff --git a/deploy/docker/druid-jobs/supervisor-spec.json b/deploy/docker/druid-kafka-setup/druid-jobs/supervisor-spec.json similarity index 100% rename from deploy/docker/druid-jobs/supervisor-spec.json rename to deploy/docker/druid-kafka-setup/druid-jobs/supervisor-spec.json diff --git a/deploy/docker/environment_small/broker b/deploy/docker/druid-kafka-setup/environment_small/broker similarity index 100% rename from deploy/docker/environment_small/broker rename to deploy/docker/druid-kafka-setup/environment_small/broker diff --git a/deploy/docker/environment_small/coordinator b/deploy/docker/druid-kafka-setup/environment_small/coordinator similarity index 100% rename from deploy/docker/environment_small/coordinator rename to deploy/docker/druid-kafka-setup/environment_small/coordinator diff --git a/deploy/docker/environment_small/historical b/deploy/docker/druid-kafka-setup/environment_small/historical similarity index 100% rename from deploy/docker/environment_small/historical rename to deploy/docker/druid-kafka-setup/environment_small/historical diff --git a/deploy/docker/environment_small/middlemanager b/deploy/docker/druid-kafka-setup/environment_small/middlemanager similarity index 100% rename from deploy/docker/environment_small/middlemanager rename to deploy/docker/druid-kafka-setup/environment_small/middlemanager diff --git a/deploy/docker/environment_small/router b/deploy/docker/druid-kafka-setup/environment_small/router similarity index 100% rename from deploy/docker/environment_small/router rename to deploy/docker/druid-kafka-setup/environment_small/router diff --git a/deploy/docker/environment_tiny/broker b/deploy/docker/druid-kafka-setup/environment_tiny/broker similarity index 100% rename from deploy/docker/environment_tiny/broker rename to deploy/docker/druid-kafka-setup/environment_tiny/broker diff --git a/deploy/docker/environment_tiny/common b/deploy/docker/druid-kafka-setup/environment_tiny/common similarity index 100% rename from deploy/docker/environment_tiny/common rename to deploy/docker/druid-kafka-setup/environment_tiny/common diff --git a/deploy/docker/druid-kafka-setup/environment_tiny/common_test b/deploy/docker/druid-kafka-setup/environment_tiny/common_test new file mode 100644 index 0000000000..b5d8534f22 --- /dev/null +++ b/deploy/docker/druid-kafka-setup/environment_tiny/common_test @@ -0,0 +1,26 @@ +# For S3 storage + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-s3-extensions"] + + +druid_storage_type=s3 +druid_storage_bucket=solvzy-test3 +druid_storage_baseKey=druid/segments + +AWS_ACCESS_KEY_ID=AKIARKCF5OX3CMI3XRXC +AWS_SECRET_ACCESS_KEY=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 +AWS_REGION=us-east-2 + +druid_indexer_logs_type=s3 +druid_indexer_logs_s3Bucket=solvzy-test3 +druid_indexer_logs_s3Prefix=druid/indexing-logs + +# ----------------------------------------------------------- +# For local storage +# druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service"] + +# druid_storage_type=local +# druid_storage_storageDirectory=/opt/data/segments +# druid_indexer_logs_type=file +# druid_indexer_logs_directory=/opt/data/indexing-logs + diff --git a/deploy/docker/environment_tiny/coordinator b/deploy/docker/druid-kafka-setup/environment_tiny/coordinator similarity index 100% rename from deploy/docker/environment_tiny/coordinator rename to deploy/docker/druid-kafka-setup/environment_tiny/coordinator diff --git a/deploy/docker/environment_tiny/historical b/deploy/docker/druid-kafka-setup/environment_tiny/historical similarity index 100% rename from deploy/docker/environment_tiny/historical rename to deploy/docker/druid-kafka-setup/environment_tiny/historical diff --git a/deploy/docker/environment_tiny/middlemanager b/deploy/docker/druid-kafka-setup/environment_tiny/middlemanager similarity index 100% rename from deploy/docker/environment_tiny/middlemanager rename to deploy/docker/druid-kafka-setup/environment_tiny/middlemanager diff --git a/deploy/docker/environment_tiny/router b/deploy/docker/druid-kafka-setup/environment_tiny/router similarity index 100% rename from deploy/docker/environment_tiny/router rename to deploy/docker/druid-kafka-setup/environment_tiny/router diff --git a/deploy/docker/otel-collector-config.yaml b/deploy/docker/druid-kafka-setup/otel-collector-config.yaml similarity index 100% rename from deploy/docker/otel-collector-config.yaml rename to deploy/docker/druid-kafka-setup/otel-collector-config.yaml diff --git a/deploy/install.k8s.sh b/deploy/install.k8s.sh deleted file mode 100644 index 062cc8f274..0000000000 --- a/deploy/install.k8s.sh +++ /dev/null @@ -1,256 +0,0 @@ -#!/bin/bash - -set -o errexit - -is_command_present() { - type "$1" >/dev/null 2>&1 -} - -is_mac() { - [[ $OSTYPE == darwin* ]] -} - - -check_k8s_setup() { - echo "Checking your k8s setup status" - if ! is_command_present kubectl; then - echo "Please install kubectl on your machine" - exit 1 - else - - if ! is_command_present jq; then - install_jq - fi - clusters=`kubectl config view -o json | jq -r '."current-context"'` - if [[ ! -n $clusters ]]; then - echo "Please setup a k8s cluster & config kubectl to connect to it" - exit 1 - fi - k8s_minor_version=`kubectl version --short -o json | jq ."serverVersion.minor" | sed 's/[^0-9]*//g'` - # if [[ $k8s_minor_version < 18 ]]; then - # echo "+++++++++++ ERROR ++++++++++++++++++++++" - # echo "SigNoz deployments require Kubernetes >= v1.18. Found version: v1.$k8s_minor_version" - # echo "+++++++++++ ++++++++++++++++++++++++++++" - # exit 1 - # fi; - fi -} - -install_jq(){ - if [ $package_manager == "brew" ]; then - brew install jq - elif [ $package_manager == "yum" ]; then - yum_cmd="sudo yum --assumeyes --quiet" - $yum_cmd install jq - else - apt_cmd="sudo apt-get --yes --quiet" - $apt_cmd update - $apt_cmd install jq - fi -} - - -check_os() { - if is_mac; then - package_manager="brew" - desired_os=1 - os="Mac" - return - fi - - os_name="$(cat /etc/*-release | awk -F= '$1 == "NAME" { gsub(/"/, ""); print $2; exit }')" - - case "$os_name" in - Ubuntu*) - desired_os=1 - os="ubuntu" - package_manager="apt-get" - ;; - Debian*) - desired_os=1 - os="debian" - package_manager="apt-get" - ;; - Red\ Hat*) - desired_os=1 - os="red hat" - package_manager="yum" - ;; - CentOS*) - desired_os=1 - os="centos" - package_manager="yum" - ;; - *) - desired_os=0 - os="Not Found" - esac -} - - -echo_contact_support() { - echo "Please contact with your OS details and version${1:-.}" -} - -bye() { # Prints a friendly good bye message and exits the script. - set +o errexit - echo "Please share your email to receive support with the installation" - read -rp 'Email: ' email - - while [[ $email == "" ]] - do - read -rp 'Email: ' email - done - - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Support", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "email": "'"$email"'", "platform": "k8s", "k8s_minor_version": "'"$k8s_minor_version"'" } }' - URL="https://app.posthog.com/capture" - HEADER="Content-Type: application/json" - - - if has_curl; then - curl -sfL -d "$DATA" --header "$HEADER" "$URL" > /dev/null 2>&1 - elif has_wget; then - wget -q --post-data="$DATA" --header="$HEADER" "$URL" > /dev/null 2>&1 - fi - - echo -e "\nExiting for now. Bye! \U1F44B\n" - exit 1 -} - -deploy_app() { - kubectl apply -f "$install_dir/config-template" - kubectl apply -f "$install_dir" -} - -wait_for_application_start() { - local timeout=$1 - address=$custom_domain - if [[ "$ssl_enable" == "true" ]]; then - protocol="https" - else - protocol="http" - fi - # The while loop is important because for-loops don't work for dynamic values - while [[ $timeout -gt 0 ]]; do - if [[ $address == "" || $address == null ]]; then - address=`kubectl get ingress appsmith-ingress -o json | jq -r '.status.loadBalancer.ingress[0].ip'` - fi - status_code="$(curl -s -o /dev/null -w "%{http_code}" $protocol://$address/api/v1 || true)" - if [[ status_code -eq 401 ]]; then - break - else - echo -ne "Waiting for all containers to start. This check will timeout in $timeout seconds...\r\c" - fi - ((timeout--)) - sleep 1 - done - - echo "" -} - - -echo -e "šŸ‘‹ Thank you for trying out SigNoz! " -echo "" - - -# Checking OS and assigning package manager -desired_os=0 -os="" -echo -e "šŸ•µļø Detecting your OS" -check_os -SIGNOZ_INSTALLATION_ID=$(curl -s 'https://api64.ipify.org') - -# Run bye if failure happens -trap bye EXIT - -DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Started", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "platform": "k8s", "k8s_minor_version": "'"$k8s_minor_version"'" } }' - -URL="https://app.posthog.com/capture" -HEADER="Content-Type: application/json" - -if has_curl; then - curl -sfL -d "$DATA" --header "$HEADER" "$URL" > /dev/null 2>&1 -elif has_wget; then - wget -q --post-data="$DATA" --header="$HEADER" "$URL" > /dev/null 2>&1 -fi - -# Check for kubernetes setup -check_k8s_setup - -echo "" -echo "Deploy Appmisth on your cluster" -echo "" - -deploy_app - -wait_for_application_start 60 - - -if [[ $status_code -ne 200 ]]; then - echo "+++++++++++ ERROR ++++++++++++++++++++++" - echo "The containers didn't seem to start correctly. Please run the following command to check containers that may have errored out:" - echo "" - echo -e "sudo docker-compose -f docker/docker-compose-tiny.yaml ps -a" - echo "Please read our troubleshooting guide https://signoz.io/docs/deployment/docker#troubleshooting" - echo "or reach us on SigNoz for support https://join.slack.com/t/signoz-community/shared_invite/zt-lrjknbbp-J_mI13rlw8pGF4EWBnorJA" - echo "++++++++++++++++++++++++++++++++++++++++" - - SUPERVISORS="$(curl -so - http://localhost:8888/druid/indexer/v1/supervisor)" - - DATASOURCES="$(curl -so - http://localhost:8888/druid/coordinator/v1/datasources)" - - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error - Checks", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "platform": "k8s", "error": "Containers not started", "SUPERVISORS": '"$SUPERVISORS"', "DATASOURCES": '"$DATASOURCES"' } }' - - URL="https://app.posthog.com/capture" - HEADER="Content-Type: application/json" - - if has_curl; then - curl -sfL -d "$DATA" --header "$HEADER" "$URL" > /dev/null 2>&1 - elif has_wget; then - wget -q --post-data="$DATA" --header="$HEADER" "$URL" > /dev/null 2>&1 - fi - - exit 1 - -else - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Success", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'"} }' - URL="https://app.posthog.com/capture" - HEADER="Content-Type: application/json" - - if has_curl; then - curl -sfL -d "$DATA" --header "$HEADER" "$URL" > /dev/null 2>&1 - elif has_wget; then - wget -q --post-data="$DATA" --header="$HEADER" "$URL" > /dev/null 2>&1 - fi - echo "++++++++++++++++++ SUCCESS ++++++++++++++++++++++" - echo "Your installation is complete!" - echo "" - echo "Your frontend is running on 'http://localhost:3000'." - - echo "" - echo "+++++++++++++++++++++++++++++++++++++++++++++++++" - echo "" - echo "Need help Getting Started?" - echo "Join us on Slack https://join.slack.com/t/signoz-community/shared_invite/zt-lrjknbbp-J_mI13rlw8pGF4EWBnorJA" - echo "" - echo "Please share your email to receive support & updates about SigNoz!" - read -rp 'Email: ' email - - while [[ $email == "" ]] - do - read -rp 'Email: ' email - done - - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Identify Successful Installation", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "email": "'"$email"'", "platform": "k8s" } }' - URL="https://app.posthog.com/capture" - HEADER="Content-Type: application/json" - - if has_curl; then - curl -sfL -d "$DATA" --header "$HEADER" "$URL" > /dev/null 2>&1 - elif has_wget; then - wget -q --post-data="$DATA" --header="$HEADER" "$URL" > /dev/null 2>&1 - fi - -fi - -echo -e "\nThank you!\n" diff --git a/deploy/install.sh b/deploy/install.sh index 54008266be..4c086dcd98 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -2,6 +2,16 @@ set -o errexit +# Regular Colors +Black='\033[0;30m' # Black +Red='\[\e[0;31m\]' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Blue='\033[0;34m' # Blue +Purple='\033[0;35m' # Purple +Cyan='\033[0;36m' # Cyan +White='\033[0;37m' # White +NC='\033[0m' # No Color is_command_present() { type "$1" >/dev/null 2>&1 @@ -88,7 +98,7 @@ check_os() { # The script should error out in case they aren't available check_ports_occupied() { local port_check_output - local ports_pattern="80|443" + local ports_pattern="80|3000|8080" if is_mac; then port_check_output="$(netstat -anp tcp | awk '$6 == "LISTEN" && $4 ~ /^.*\.('"$ports_pattern"')$/')" @@ -192,7 +202,7 @@ install_docker_compose() { echo "" fi else - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Docker Compose not found" } }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Docker Compose not found", "setup_type": "'"$setup_type"'" } }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -212,8 +222,7 @@ install_docker_compose() { start_docker() { echo "Starting Docker ..." - if [ $os == "Mac" ] - then + if [ $os = "Mac" ]; then open --background -a Docker && while ! docker system info > /dev/null 2>&1; do sleep 1; done else if ! sudo systemctl is-active docker.service > /dev/null; then @@ -231,16 +240,17 @@ wait_for_containers_start() { if [[ status_code -eq 200 ]]; then break else - SUPERVISORS="$(curl -so - http://localhost:8888/druid/indexer/v1/supervisor)" - LEN_SUPERVISORS="${#SUPERVISORS}" + if [ $setup_type == 'druid' ]; then + SUPERVISORS="$(curl -so - http://localhost:8888/druid/indexer/v1/supervisor)" + LEN_SUPERVISORS="${#SUPERVISORS}" - if [[ LEN_SUPERVISORS -ne 19 && $timeout -eq 50 ]];then - echo "No Supervisors found... Re-applying docker compose\n" - sudo docker-compose -f ./docker/docker-compose-tiny.yaml up -d + if [[ LEN_SUPERVISORS -ne 19 && $timeout -eq 50 ]];then + echo -e "\nšŸŸ  Supervisors taking time to start ā³ ... let's wait for some more time ā±ļø\n\n" + sudo docker-compose -f ./docker/druid-kafka-setup/docker-compose-tiny.yaml up -d + fi fi - - echo -ne "Waiting for all containers to start. This check will timeout in $timeout seconds...\r\c" + echo -ne "Waiting for all containers to start. This check will timeout in $timeout seconds ...\r\c" fi ((timeout--)) sleep 1 @@ -253,14 +263,18 @@ bye() { # Prints a friendly good bye message and exits the script. if [ "$?" -ne 0 ]; then set +o errexit - echo "The containers didn't seem to start correctly. Please run the following command to check containers that may have errored out:" + echo "šŸ”“ The containers didn't seem to start correctly. Please run the following command to check containers that may have errored out:" echo "" - echo -e "sudo docker-compose -f docker/docker-compose-tiny.yaml ps -a" + if [ $setup_type == 'clickhouse' ]; then + echo -e "sudo docker-compose -f docker/clickhouse-setup/docker-compose.yaml ps -a" + else + echo -e "sudo docker-compose -f docker/druid-kafka-setup/docker-compose-tiny.yaml ps -a" + fi # echo "Please read our troubleshooting guide https://signoz.io/docs/deployment/docker#troubleshooting" echo "or reach us on SigNoz for support https://join.slack.com/t/signoz-community/shared_invite/zt-lrjknbbp-J_mI13rlw8pGF4EWBnorJA" echo "++++++++++++++++++++++++++++++++++++++++" - echo "Please share your email to receive support with the installation" + echo -e "\nšŸ“Ø Please share your email to receive support with the installation" read -rp 'Email: ' email while [[ $email == "" ]] @@ -268,7 +282,7 @@ bye() { # Prints a friendly good bye message and exits the script. read -rp 'Email: ' email done - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Support", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "email": "'"$email"'" } }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Support", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "email": "'"$email"'", "setup_type": "'"$setup_type"'" } }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -294,17 +308,39 @@ echo "" # Checking OS and assigning package manager desired_os=0 os="" -echo -e "šŸ•µļø Detecting your OS" +echo -e "Detecting your OS ..." check_os - SIGNOZ_INSTALLATION_ID=$(curl -s 'https://api64.ipify.org') +echo "" + +echo -e "šŸ‘‰ ${RED}Two ways to go forward\n" +echo -e "${RED}1) ClickHouse as database (recommended for low memory usage)\n" +echo -e "${RED}2) Kafka + Druid setup to handle scale (recommended for production use)\n" +read -p "āš™ļø Enter your preference (1/2):" choice_setup + +while [[ $choice_setup == "" || ( $choice_setup != "1" && $choice_setup != "2" ) ]] +do + # echo $choice_setup + echo -e "\nāŒ ${CYAN}Please enter either 1 or 2" + read -rp "āš™ļø Enter your preference (1/2): " choice_setup + # echo $choice_setup +done + +if [ $choice_setup == "1" ];then + setup_type='clickhouse' +else + setup_type='druid' +fi + +echo -e "\nāœ… ${CYAN}You have chosen: ${setup_type} setup\n" + # Run bye if failure happens trap bye EXIT -DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Started", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'" } }' +DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Started", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "setup_type": "'"$setup_type"'" } }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -316,7 +352,7 @@ fi if [[ $desired_os -eq 0 ]];then - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "OS Not Supported" } }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "OS Not Supported", "setup_type": "'"$setup_type"'" } }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -340,7 +376,7 @@ if ! is_command_present docker; then echo "Docker Desktop must be installed manually on Mac OS to proceed. Docker can only be installed automatically on Ubuntu / openSUSE / SLES / Redhat / Cent OS" echo "https://docs.docker.com/docker-for-mac/install/" echo "++++++++++++++++++++++++++++++++++++++++++++++++" - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Docker not installed" } }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Docker not installed", "setup_type": "'"$setup_type"'" } }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -358,43 +394,59 @@ if ! is_command_present docker-compose; then install_docker_compose fi -# if ! is_command_present docker-compose; then -# install_docker_machine -# docker-machine create -d virtualbox --virtualbox-memory 3584 signoz - -# fi start_docker +# sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml up -d --remove-orphans || true + + echo "" -echo "Pulling the latest container images for SigNoz. To run as sudo it will ask for system password." -sudo docker-compose -f ./docker/docker-compose-tiny.yaml pull +echo -e "\nšŸŸ” Pulling the latest container images for SigNoz. To run as sudo it may ask for system password\n" +if [ $setup_type == 'clickhouse' ]; then + sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml pull +else + sudo docker-compose -f ./docker/druid-kafka-setup/docker-compose-tiny.yaml pull +fi + + echo "" -echo "Starting the SigNoz containers. It may take a few minute ..." +echo "šŸŸ” Starting the SigNoz containers. It may take a few minutes ..." echo # The docker-compose command does some nasty stuff for the `--detach` functionality. So we add a `|| true` so that the # script doesn't exit because this command looks like it failed to do it's thing. -sudo docker-compose -f ./docker/docker-compose-tiny.yaml up --detach --remove-orphans || true +if [ $setup_type == 'clickhouse' ]; then + sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml up --detach --remove-orphans || true +else + sudo docker-compose -f ./docker/druid-kafka-setup/docker-compose-tiny.yaml up --detach --remove-orphans || true +fi wait_for_containers_start 60 echo "" if [[ $status_code -ne 200 ]]; then echo "+++++++++++ ERROR ++++++++++++++++++++++" - echo "The containers didn't seem to start correctly. Please run the following command to check containers that may have errored out:" + echo "šŸ”“ The containers didn't seem to start correctly. Please run the following command to check containers that may have errored out:" echo "" - echo -e "sudo docker-compose -f docker/docker-compose-tiny.yaml ps -a" + if [ $setup_type == 'clickhouse' ]; then + echo -e "sudo docker-compose -f docker/clickhouse-setup/docker-compose.yaml ps -a" + else + echo -e "sudo docker-compose -f docker/druid-kafka-setup/docker-compose-tiny.yaml ps -a" + fi echo "Please read our troubleshooting guide https://signoz.io/docs/deployment/docker/#troubleshooting-of-common-issues" echo "or reach us on SigNoz for support https://join.slack.com/t/signoz-community/shared_invite/zt-lrjknbbp-J_mI13rlw8pGF4EWBnorJA" echo "++++++++++++++++++++++++++++++++++++++++" - SUPERVISORS="$(curl -so - http://localhost:8888/druid/indexer/v1/supervisor)" + if [ $setup_type == 'clickhouse' ]; then + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error - Checks", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Containers not started", "data": "some_checks", "setup_type": "'"$setup_type"'" } }' + else + SUPERVISORS="$(curl -so - http://localhost:8888/druid/indexer/v1/supervisor)" - DATASOURCES="$(curl -so - http://localhost:8888/druid/coordinator/v1/datasources)" + DATASOURCES="$(curl -so - http://localhost:8888/druid/coordinator/v1/datasources)" - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error - Checks", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Containers not started", "SUPERVISORS": '"$SUPERVISORS"', "DATASOURCES": '"$DATASOURCES"' } }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Error - Checks", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "error": "Containers not started", "SUPERVISORS": '"$SUPERVISORS"', "DATASOURCES": '"$DATASOURCES"', "setup_type": "'"$setup_type"'" } }' + fi URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -408,7 +460,7 @@ if [[ $status_code -ne 200 ]]; then exit 1 else - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Success", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'"} }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Installation Success", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'"}, "setup_type": "'"$setup_type"'" }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -419,18 +471,24 @@ else fi echo "++++++++++++++++++ SUCCESS ++++++++++++++++++++++" echo "" - echo "Your installation is complete!" + echo "šŸŸ¢ Your installation is complete!" echo "" - echo "Your frontend is running on 'http://localhost:3000'." + echo -e "šŸŸ¢ Your frontend is running on http://localhost:3000" echo "" - echo "To bring down SigNoz and clean volumes : sudo docker-compose -f docker/docker-compose-tiny.yaml down -v" + + if [ $setup_type == 'clickhouse' ]; then + echo "ā„¹ļø To bring down SigNoz and clean volumes : sudo docker-compose -f docker/clickhouse-setup/docker-compose.yaml down -v" + else + echo "ā„¹ļø To bring down SigNoz and clean volumes : sudo docker-compose -f docker/druid-kafka-setup/docker-compose-tiny.yaml down -v" + fi + echo "" echo "+++++++++++++++++++++++++++++++++++++++++++++++++" echo "" - echo "Need help Getting Started?" - echo "Join us on Slack https://join.slack.com/t/signoz-community/shared_invite/zt-lrjknbbp-J_mI13rlw8pGF4EWBnorJA" + echo "šŸ‘‰ Need help Getting Started?" + echo -e "Join us on Slack https://join.slack.com/t/signoz-community/shared_invite/zt-lrjknbbp-J_mI13rlw8pGF4EWBnorJA" echo "" - echo "Please share your email to receive support & updates about SigNoz!" + echo -e "\nšŸ“Ø Please share your email to receive support & updates about SigNoz!" read -rp 'Email: ' email while [[ $email == "" ]] @@ -438,7 +496,7 @@ else read -rp 'Email: ' email done - DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Identify Successful Installation", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "email": "'"$email"'" } }' + DATA='{ "api_key": "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w", "type": "capture", "event": "Identify Successful Installation", "distinct_id": "'"$SIGNOZ_INSTALLATION_ID"'", "properties": { "os": "'"$os"'", "email": "'"$email"'", "setup_type": "'"$setup_type"'" } }' URL="https://app.posthog.com/capture" HEADER="Content-Type: application/json" @@ -450,28 +508,4 @@ else fi -echo -e "\nThank you!\n" - - -##### Changing default memory limit of docker ############ -# # Check if memory is less and Confirm to increase size of docker machine -# # https://github.com/docker/machine/releases -# # On OS X - -# $ curl -L https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-`uname -s`-`uname -m` >/usr/local/bin/docker-machine && \ -# chmod +x /usr/local/bin/docker-machine -# # On Linux - -# $ curl -L https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-`uname -s`-`uname -m` >/tmp/docker-machine && -# chmod +x /tmp/docker-machine && -# sudo cp /tmp/docker-machine /usr/local/bin/docker-machine - -# VBoxManage list vms -# docker-machine stop -# VBoxManage modifyvm default --cpus 2 -# VBoxManage modifyvm default --memory 4096 -# docker-machine start - -# VBoxManage showvminfo default | grep Memory -# VBoxManage showvminfo default | grep CPU - +echo -e "\nšŸ™ Thank you!\n" \ No newline at end of file