build(docker): Two compose files for arm and amd (#638)

* build(docker): 🔨 Two compose files for arm and amd

* refactor(docker): ⚰️ remove env file from install script

* refactor: ⚰️ remove .gitkeep files from data folder

* chore(build): ⚰️ remove env files and update contributing docs

Signed-off-by: Prashant Shahi <prashant@signoz.io>

* build: ♻️ use two compose files in Makefile

Signed-off-by: Prashant Shahi <prashant@signoz.io>

* chore(docker): 🚚 revert back to using same dir and pin image tag

* Revert "chore: Add migration file path in otel collector config (#628)"

This reverts commit 8467d6a00c1322a2b6af3c474fde9131b8bad14b.

Signed-off-by: Prashant Shahi <prashant@signoz.io>

CONTRIBUTING.md

@@ -21,8 +21,8 @@ Need to update [https://github.com/SigNoz/signoz/tree/main/frontend](https://git
 - comment out frontend service section at `deploy/docker/clickhouse-setup/docker-compose.yaml#L38`
 - run `cd deploy` to move to deploy directory
 - Install signoz locally without the frontend
-  - If you are using x86_64 processors (All Intel/AMD processors) run `sudo docker-compose --env-file ./docker/clickhouse-setup/env/x86_64.env -f docker/clickhouse-setup/docker-compose.yaml up -d`
+  - If you are using x86_64 processors (All Intel/AMD processors) run `sudo docker-compose -f docker/clickhouse-setup/docker-compose.yaml up -d`
-  - If you are on arm64 processors (Apple M1 Macbooks) run `sudo docker-compose --env-file ./docker/clickhouse-setup/env/arm64.env -f docker/clickhouse-setup/docker-compose.yaml up -d`
+  - If you are on arm64 processors (Apple M1 Macbooks) run `sudo docker-compose -f docker/clickhouse-setup/docker-compose.arm.yaml up -d`
 - `cd ../frontend` and change baseURL to `http://localhost:8080` in file `src/constants/env.ts`
 - `yarn install`
 - `yarn dev`

Makefile

@@ -91,7 +91,7 @@ dev-setup:
 	@echo "------------------"
 
 run-x86:
-	@sudo docker-compose --env-file ./deploy/docker/clickhouse-setup/env/x86_64.env -f ./deploy/docker/clickhouse-setup/docker-compose.yaml up -d
+	@sudo docker-compose -f ./deploy/docker/clickhouse-setup/docker-compose.yaml up -d
 
 run-arm:
-	@sudo docker-compose --env-file ./deploy/docker/clickhouse-setup/env/arm64.env -f ./deploy/docker/clickhouse-setup/docker-compose.yaml up -d
+	@sudo docker-compose -f ./deploy/docker/clickhouse-setup/docker-compose.arm.yaml up -d

deploy/docker/clickhouse-setup/docker-compose.arm.yaml

@@ -0,0 +1,119 @@
+version: "2.4"
+
+services:
+  clickhouse:
+    image: altinity/clickhouse-server:21.12.3.32.altinitydev.arm
+    expose:
+      - 8123
+      - 9000
+    ports:
+      - 9001:9000
+      - 8123:8123
+    volumes:
+      - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+      - ./data/clickhouse/:/var/lib/clickhouse/
+    healthcheck:
+      # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
+      test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+
+  alertmanager:
+    image: signoz/alertmanager:0.5.0
+    volumes:
+      - ./alertmanager.yml:/prometheus/alertmanager.yml
+      - ./data/alertmanager:/data
+    command:
+      - '--config.file=/prometheus/alertmanager.yml'
+      - '--storage.path=/data'
+    ports:
+      - 9093:9093
+
+  query-service:
+    image: signoz/query-service:0.5.4
+    container_name: query-service
+    command: ["-config=/root/config/prometheus.yml"]
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./prometheus.yml:/root/config/prometheus.yml
+      - ../dashboards:/root/config/dashboards
+      - ./data/signoz/:/var/lib/signoz/
+    environment:
+      - ClickHouseUrl=tcp://clickhouse:9000
+      - STORAGE=clickhouse
+      - GODEBUG=netdns=go
+      - TELEMETRY_ENABLED=true
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+
+  frontend:
+    image: signoz/frontend:0.5.4
+    container_name: frontend
+    depends_on:
+      - query-service
+    links:
+      - "query-service"
+    ports:
+      - "3000:3000"
+    volumes:
+      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
+
+  otel-collector:
+    image: signoz/otelcontribcol:0.4.3
+    command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "1777:1777"   # pprof extension
+      - "8887:8888"   # Prometheus metrics exposed by the agent
+      - "14268:14268"       # Jaeger receiver
+      - "55678"       # OpenCensus receiver
+      - "55680:55680"       # OTLP HTTP/2.0 legacy port
+      - "55681:55681"       # OTLP HTTP/1.0 receiver
+      - "4317:4317"       # OTLP GRPC receiver
+      - "55679:55679" # zpages extension
+      - "13133"       # health_check
+      - "8889:8889"   # prometheus exporter
+    mem_limit: 2000m
+    restart: always
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+
+  otel-collector-metrics:
+    image: signoz/otelcontribcol:0.4.3
+    command: ["--config=/etc/otel-collector-metrics-config.yaml", "--mem-ballast-size-mib=683"]
+    volumes:
+      - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+
+  hotrod:
+    image: jaegertracing/example-hotrod:1.30
+    container_name: hotrod
+    ports:
+    - "9000:8080"
+    command: ["all"]
+    environment:
+    - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces
+
+  load-hotrod:
+    image: "grubykarol/locust:1.2.3-python3.9-alpine3.12"
+    container_name: load-hotrod
+    hostname: load-hotrod
+    ports:
+     - "8089:8089"
+    environment:
+      ATTACKED_HOST: http://hotrod:8080
+      LOCUST_MODE: standalone
+      NO_PROXY: standalone
+      TASK_DELAY_FROM: 5
+      TASK_DELAY_TO: 30
+      QUIET_MODE: "${QUIET_MODE:-false}"
+      LOCUST_OPTS: "--headless -u 10 -r 1"
+    volumes:
+      - ../common/locust-scripts:/locust

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -2,23 +2,23 @@ version: "2.4"
 
 services:
   clickhouse:
-      image: ${clickhouse_image}
+    image: yandex/clickhouse-server:21.12.3.32
-      expose:
+    expose:
-        - 8123
+      - 8123
-        - 9000
+      - 9000
-      ports:
+    ports:
-        - 9001:9000
+      - 9001:9000
-        - 8123:8123
+      - 8123:8123
-      volumes:
+    volumes:
-        - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
-        - ./data/clickhouse/:/var/lib/clickhouse/
+      - ./data/clickhouse/:/var/lib/clickhouse/
 
-      healthcheck:
+    healthcheck:
-        # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
+      # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
-        test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
+      test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
-        interval: 30s
+      interval: 30s
-        timeout: 5s
+      timeout: 5s
-        retries: 3
+      retries: 3
 
   alertmanager:
     image: signoz/alertmanager:0.5.0
@@ -41,7 +41,6 @@ services:
       - ./prometheus.yml:/root/config/prometheus.yml
       - ../dashboards:/root/config/dashboards
       - ./data/signoz/:/var/lib/signoz/
-
     environment:
       - ClickHouseUrl=tcp://clickhouse:9000
       - STORAGE=clickhouse
@@ -55,7 +54,6 @@ services:
   frontend:
     image: signoz/frontend:0.5.4
     container_name: frontend
-
     depends_on:
       - query-service
     links:
@@ -66,7 +64,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/otelcontribcol:0.4.2
+    image: signoz/otelcontribcol:0.4.3
     command: ["--config=/etc/otel-collector-config.yaml", "--mem-ballast-size-mib=683"]
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@@ -81,7 +79,6 @@ services:
       - "55679:55679" # zpages extension
       - "13133"       # health_check
       - "8889:8889"   # prometheus exporter
-    
     mem_limit: 2000m
     restart: always
     depends_on:
@@ -89,7 +86,7 @@ services:
         condition: service_healthy
 
   otel-collector-metrics:
-    image: signoz/otelcontribcol:0.4.2
+    image: signoz/otelcontribcol:0.4.3
     command: ["--config=/etc/otel-collector-metrics-config.yaml", "--mem-ballast-size-mib=683"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
@@ -98,7 +95,7 @@ services:
         condition: service_healthy
 
   hotrod:
-    image: jaegertracing/example-hotrod:latest
+    image: jaegertracing/example-hotrod:1.30
     container_name: hotrod
     ports:
     - "9000:8080"

deploy/docker/clickhouse-setup/env/arm64.env

@@ -1 +0,0 @@
-clickhouse_image=altinity/clickhouse-server:21.8.12.1.testingarm

deploy/docker/clickhouse-setup/env/x86_64.env

@@ -1 +0,0 @@
-clickhouse_image=yandex/clickhouse-server:21.10

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -45,7 +45,6 @@ extensions:
 exporters:
   clickhouse:
     datasource: tcp://clickhouse:9000
-    migrations: /migrations
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/?database=signoz_metrics
     resource_to_telemetry_conversion:

deploy/install.sh

@@ -272,9 +272,9 @@ bye() {  # Prints a friendly good bye message and exits the script.
         echo ""
         if [ $setup_type == 'clickhouse' ]; then
             if is_arm64; then
-                echo -e "sudo docker-compose --env-file ./docker/clickhouse-setup/env/arm64.env -f docker/clickhouse-setup/docker-compose.yaml ps -a"
+                echo -e "sudo docker-compose -f docker/clickhouse-setup/docker-compose.arm.yaml ps -a"
             else
-                echo -e "sudo docker-compose --env-file ./docker/clickhouse-setup/env/x86_64.env -f docker/clickhouse-setup/docker-compose.yaml ps -a"
+                echo -e "sudo docker-compose -f docker/clickhouse-setup/docker-compose.yaml ps -a"
             fi
         else   
             echo -e "sudo docker-compose -f docker/druid-kafka-setup/docker-compose-tiny.yaml ps -a"
@@ -417,9 +417,9 @@ echo ""
 echo -e "\n🟡 Pulling the latest container images for SigNoz. To run as sudo it may ask for system password\n"
 if [ $setup_type == 'clickhouse' ]; then
     if is_arm64; then
-        sudo docker-compose --env-file ./docker/clickhouse-setup/env/arm64.env -f ./docker/clickhouse-setup/docker-compose.yaml pull
+        sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.arm.yaml pull
     else
-        sudo docker-compose --env-file ./docker/clickhouse-setup/env/x86_64.env -f ./docker/clickhouse-setup/docker-compose.yaml pull
+        sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml pull
     fi
 else
     sudo docker-compose -f ./docker/druid-kafka-setup/docker-compose-tiny.yaml pull
@@ -433,9 +433,9 @@ echo
 # script doesn't exit because this command looks like it failed to do it's thing.
 if [ $setup_type == 'clickhouse' ]; then
     if is_arm64; then
-        sudo docker-compose --env-file ./docker/clickhouse-setup/env/arm64.env -f ./docker/clickhouse-setup/docker-compose.yaml up --detach --remove-orphans || true
+        sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.arm.yaml up --detach --remove-orphans || true
     else
-        sudo docker-compose --env-file ./docker/clickhouse-setup/env/x86_64.env -f ./docker/clickhouse-setup/docker-compose.yaml up --detach --remove-orphans || true
+        sudo docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml up --detach --remove-orphans || true
     fi
 else
     sudo docker-compose -f ./docker/druid-kafka-setup/docker-compose-tiny.yaml up --detach --remove-orphans || true
@@ -497,9 +497,9 @@ else
 
     if [ $setup_type == 'clickhouse' ]; then
         if is_arm64; then
-            echo "ℹ️  To bring down SigNoz and clean volumes : sudo docker-compose --env-file ./docker/clickhouse-setup/env/arm64.env -f docker/clickhouse-setup/docker-compose.yaml down -v"
+            echo "ℹ️  To bring down SigNoz and clean volumes : sudo docker-compose -f docker/clickhouse-setup/docker-compose.arm.yaml down -v"
         else
-            echo "ℹ️  To bring down SigNoz and clean volumes : sudo docker-compose --env-file ./docker/clickhouse-setup/env/x86_64.env -f docker/clickhouse-setup/docker-compose.yaml down -v"
+            echo "ℹ️  To bring down SigNoz and clean volumes : sudo docker-compose -f docker/clickhouse-setup/docker-compose.yaml down -v"
         fi
     else
         echo "ℹ️  To bring down SigNoz and clean volumes : sudo docker-compose -f docker/druid-kafka-setup/docker-compose-tiny.yaml down -v"