From da1fd4b0cdba99b9ab77c9150f13a03d35ad04f9 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Tue, 3 Jan 2023 22:28:48 +0530 Subject: [PATCH] ci(deployments): workflows for staging and testing deployments and related changes (#1933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(Makefile): ⚡️ remove no-cache from all docker build commands * chore(Makefile): 🔧 update target name * feat(docker-standalone): ✨ introduce tag environment variables for easy custom deployments * ci(deployments): 👷 workflows for staging and testing deployments * ci(deployments): 👷 pass DEV_BUILD env to remote host --- .github/workflows/remove-label.yaml | 2 +- .github/workflows/staging-deployment.yaml | 37 ++++++++++++++++++ .github/workflows/testing-deployment.yaml | 39 +++++++++++++++++++ Makefile | 25 ++++++------ .../clickhouse-setup/docker-compose.yaml | 10 ++--- 5 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/staging-deployment.yaml create mode 100644 .github/workflows/testing-deployment.yaml diff --git a/.github/workflows/remove-label.yaml b/.github/workflows/remove-label.yaml index 6881b9eb77..418475ea23 100644 --- a/.github/workflows/remove-label.yaml +++ b/.github/workflows/remove-label.yaml @@ -11,6 +11,6 @@ jobs: - name: Remove label uses: buildsville/add-remove-label@v1 with: - label: ok-to-test + label: ok-to-test,testing-deploy type: remove token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/staging-deployment.yaml b/.github/workflows/staging-deployment.yaml new file mode 100644 index 0000000000..257b875f72 --- /dev/null +++ b/.github/workflows/staging-deployment.yaml @@ -0,0 +1,37 @@ +name: staging-deployment +# Trigger deployment only on push to develop branch +on: + push: + branches: + - develop +jobs: + deploy: + name: Deploy latest develop branch to staging + runs-on: ubuntu-latest + environment: staging + steps: + - name: Executing remote ssh commands using ssh key + uses: appleboy/ssh-action@v0.1.6 + env: + DOCKER_TAG: develop + GITHUB_SHA: ${{ github.sha }} + with: + host: ${{ secrets.HOST_DNS }} + username: ${{ secrets.USERNAME }} + key: ${{ secrets.EC2_SSH_KEY }} + envs: DOCKER_TAG,GITHUB_SHA + command_timeout: 60m + script: | + echo "DOCKER_TAG: ${DOCKER_TAG}" + echo "GITHUB_SHA: ${GITHUB_SHA}" + docker system prune --force + cd ~/signoz + git status + git add . + git stash push -m "stashed on $(date --iso-8601=seconds)" + git fetch origin + git checkout develop + git pull + make build-ee-query-service-amd64 + make build-frontend-amd64 + make run-signoz \ No newline at end of file diff --git a/.github/workflows/testing-deployment.yaml b/.github/workflows/testing-deployment.yaml new file mode 100644 index 0000000000..4377520134 --- /dev/null +++ b/.github/workflows/testing-deployment.yaml @@ -0,0 +1,39 @@ +name: testing-deployment +# Trigger deployment only on testing-deploy label on pull request +on: + pull_request: + types: [labeled] +jobs: + deploy: + name: Deploy PR branch to testing + runs-on: ubuntu-latest + environment: testing + if: ${{ github.event.label.name == 'testing-deploy' }} + steps: + - name: Executing remote ssh commands using ssh key + uses: appleboy/ssh-action@v0.1.6 + env: + GITHUB_BRANCH: ${{ github.head_ref || github.ref_name }} + GITHUB_SHA: ${{ github.sha }} + DEV_BUILD: "1" + with: + host: ${{ secrets.HOST_DNS }} + username: ${{ secrets.USERNAME }} + key: ${{ secrets.EC2_SSH_KEY }} + envs: GITHUB_BRANCH,GITHUB_SHA,DEV_BUILD + command_timeout: 60m + script: | + echo "GITHUB_BRANCH: ${GITHUB_BRANCH}" + echo "GITHUB_SHA: ${GITHUB_SHA}" + export DOCKER_TAG="${GITHUB_SHA:0:7}" # needed for child process to access it + docker system prune --force + cd ~/signoz + git status + git add . + git stash push -m "stashed on $(date --iso-8601=seconds)" + git fetch origin + git checkout ${GITHUB_BRANCH} + git pull + make build-ee-query-service-amd64 + make build-frontend-amd64 + make run-signoz \ No newline at end of file diff --git a/Makefile b/Makefile index d745ec9807..a651c84103 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ build-frontend-amd64: @echo "--> Building frontend docker image for amd64" @echo "------------------" @cd $(FRONTEND_DIRECTORY) && \ - docker build --file Dockerfile --no-cache -t $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) \ + docker build --file Dockerfile -t $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) \ --build-arg TARGETPLATFORM="linux/amd64" . # Step to build and push docker image of frontend(used in push pipeline) @@ -54,7 +54,7 @@ build-push-frontend: @echo "--> Building and pushing frontend docker image" @echo "------------------" @cd $(FRONTEND_DIRECTORY) && \ - docker buildx build --file Dockerfile --progress plane --no-cache --push --platform linux/amd64 \ + docker buildx build --file Dockerfile --progress plane --push --platform linux/amd64 \ --tag $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) . # Steps to build and push docker image of query service @@ -65,7 +65,7 @@ build-query-service-amd64: @echo "--> Building query-service docker image for amd64" @echo "------------------" @docker build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile \ - --no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \ + -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \ --build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="$(LD_FLAGS)" . # Step to build and push docker image of query in amd64 and arm64 (used in push pipeline) @@ -73,7 +73,7 @@ build-push-query-service: @echo "------------------" @echo "--> Building and pushing query-service docker image" @echo "------------------" - @docker buildx build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile --progress plane --no-cache \ + @docker buildx build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile --progress plane \ --push --platform linux/arm64,linux/amd64 --build-arg LD_FLAGS="$(LD_FLAGS)" \ --tag $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) . @@ -84,11 +84,11 @@ build-ee-query-service-amd64: @echo "------------------" @if [ $(DEV_BUILD) != "" ]; then \ docker build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \ - --no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \ + -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \ --build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="${LD_FLAGS} ${DEV_LD_FLAGS}" .; \ else \ docker build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \ - --no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \ + -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \ --build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="$(LD_FLAGS)" .; \ fi @@ -98,7 +98,7 @@ build-push-ee-query-service: @echo "--> Building and pushing query-service docker image" @echo "------------------" @docker buildx build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \ - --progress plane --no-cache --push --platform linux/arm64,linux/amd64 \ + --progress plane --push --platform linux/arm64,linux/amd64 \ --build-arg LD_FLAGS="$(LD_FLAGS)" --tag $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) . dev-setup: @@ -119,16 +119,19 @@ down-local: $(STANDALONE_DIRECTORY)/docker-compose-core.yaml -f $(STANDALONE_DIRECTORY)/docker-compose-local.yaml \ down -v -run-x86: +pull-signoz: + @docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml pull + +run-signoz: @docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml up --build -d -down-x86: +down-signoz: @docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml down -v clear-standalone-data: @docker run --rm -v "$(PWD)/$(STANDALONE_DIRECTORY)/data:/pwd" busybox \ - sh -c "cd /pwd && rm -rf alertmanager/* clickhous*/* signoz/* zookeeper-*/*" + sh -c "cd /pwd && rm -rf alertmanager/* clickhouse*/* signoz/* zookeeper-*/*" clear-swarm-data: @docker run --rm -v "$(PWD)/$(SWARM_DIRECTORY)/data:/pwd" busybox \ - sh -c "cd /pwd && rm -rf alertmanager/* clickhous*/* signoz/* zookeeper-*/*" + sh -c "cd /pwd && rm -rf alertmanager/* clickhouse*/* signoz/* zookeeper-*/*" diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 55f9d210f7..c8e4328e83 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -132,7 +132,7 @@ services: # - ./data/clickhouse-3/:/var/lib/clickhouse/ alertmanager: - image: signoz/alertmanager:0.23.0-0.2 + image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.0-0.2} volumes: - ./data/alertmanager:/data depends_on: @@ -146,7 +146,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: - image: signoz/query-service:0.13.0 + image: signoz/query-service:${DOCKER_TAG:-0.13.0} container_name: query-service command: ["-config=/root/config/prometheus.yml"] # ports: @@ -174,7 +174,7 @@ services: <<: *clickhouse-depend frontend: - image: signoz/frontend:0.13.0 + image: signoz/frontend:${DOCKER_TAG:-0.13.0} container_name: frontend restart: on-failure depends_on: @@ -186,7 +186,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/signoz-otel-collector:0.66.1 + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.66.1} command: ["--config=/etc/otel-collector-config.yaml"] user: root # required for reading docker container logs volumes: @@ -211,7 +211,7 @@ services: <<: *clickhouse-depend otel-collector-metrics: - image: signoz/signoz-otel-collector:0.66.1 + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.66.1} command: ["--config=/etc/otel-collector-metrics-config.yaml"] volumes: - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml