Feat/staging (#7585)

### Summary

- Non-production build workflow using Primus
- Staging CD: new staging app and dev staging deployments
- cleanup used docker resources in stagingapp/testingapp machines

---------

Signed-off-by: Prashant Shahi <prashant@signoz.io>

.github/workflows/build-community.yaml

@@ -2,10 +2,9 @@ name: build-community
 
 on:
   push:
-    branches:
-      - main
     tags:
-      - v*
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+'
 
 defaults:
   run:
@@ -19,7 +18,6 @@ jobs:
   prepare:
     runs-on: ubuntu-latest
     outputs:
-      docker_providers: ${{ steps.set-docker-providers.outputs.providers }}
       version: ${{ steps.build-info.outputs.version }}
       hash: ${{ steps.build-info.outputs.hash }}
       time: ${{ steps.build-info.outputs.time }}
@@ -38,7 +36,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: signoz/primus
-          ref: ${{ inputs.PRIMUS_REF }}
+          ref: main
           path: .primus
           token: ${{ steps.token.outputs.token }}
       - name: build-info
@@ -47,14 +45,6 @@ jobs:
           echo "hash=$($MAKE info-commit-short)" >> $GITHUB_OUTPUT
           echo "time=$($MAKE info-timestamp)" >> $GITHUB_OUTPUT
           echo "branch=$($MAKE info-branch)" >> $GITHUB_OUTPUT
-      - name: set-docker-providers
-        id: set-docker-providers
-        run: |
-          if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ || ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then
-            echo "providers=dockerhub gcp" >> $GITHUB_OUTPUT
-          else
-            echo "providers=gcp" >> $GITHUB_OUTPUT
-          fi
   js-build:
     uses: signoz/primus.workflows/.github/workflows/js-build.yaml@main
     needs: prepare
@@ -88,4 +78,4 @@ jobs:
       DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}'
       DOCKER_DOCKERFILE_PATH: ./pkg/query-service/Dockerfile.multi-arch
       DOCKER_MANIFEST: true
-      DOCKER_PROVIDERS: ${{ needs.prepare.outputs.docker_providers }}
+      DOCKER_PROVIDERS: dockerhub

.github/workflows/build-enterprise.yaml

@@ -2,8 +2,6 @@ name: build-enterprise
 
 on:
   push:
-    branches:
-      - main
     tags:
       - v*
 
@@ -38,7 +36,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: signoz/primus
-          ref: ${{ inputs.PRIMUS_REF }}
+          ref: main
           path: .primus
           token: ${{ steps.token.outputs.token }}
       - name: build-info
@@ -75,7 +73,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: frontend/.env
-          key: enterprise-dotenv-${{ github.sha }}
+          key: dotenv-${{ github.sha }}
   js-build:
     uses: signoz/primus.workflows/.github/workflows/js-build.yaml@main
     needs: prepare
@@ -83,9 +81,9 @@ jobs:
     with:
       PRIMUS_REF: main
       JS_SRC: frontend
-      JS_INPUT_ARTIFACT_CACHE_KEY: enterprise-dotenv-${{ github.sha }}
+      JS_INPUT_ARTIFACT_CACHE_KEY: dotenv-${{ github.sha }}
       JS_INPUT_ARTIFACT_PATH: frontend/.env
-      JS_OUTPUT_ARTIFACT_CACHE_KEY: enterprise-jsbuild-${{ github.sha }}
+      JS_OUTPUT_ARTIFACT_CACHE_KEY: jsbuild-${{ github.sha }}
       JS_OUTPUT_ARTIFACT_PATH: frontend/build
       DOCKER_BUILD: false
       DOCKER_MANIFEST: false
@@ -95,7 +93,7 @@ jobs:
     secrets: inherit
     with:
       PRIMUS_REF: main
-      GO_INPUT_ARTIFACT_CACHE_KEY: enterprise-jsbuild-${{ github.sha }}
+      GO_INPUT_ARTIFACT_CACHE_KEY: jsbuild-${{ github.sha }}
       GO_INPUT_ARTIFACT_PATH: frontend/build
       GO_BUILD_CONTEXT: ./ee/query-service
       GO_BUILD_FLAGS: >-

.github/workflows/build-staging.yaml

@@ -0,0 +1,131 @@
+name: build-staging
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [labeled]
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  PRIMUS_HOME: .primus
+  MAKE: make --no-print-directory --makefile=.primus/src/make/main.mk
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.label.name, 'staging:') || github.event.ref == 'refs/heads/main' }}
+    outputs:
+      version: ${{ steps.build-info.outputs.version }}
+      hash: ${{ steps.build-info.outputs.hash }}
+      time: ${{ steps.build-info.outputs.time }}
+      branch: ${{ steps.build-info.outputs.branch }}
+      deployment: ${{ steps.build-info.outputs.deployment }}
+    steps:
+      - name: self-checkout
+        uses: actions/checkout@v4
+      - id: token
+        name: github-token-gen
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.PRIMUS_APP_ID }}
+          private-key: ${{ secrets.PRIMUS_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+      - name: primus-checkout
+        uses: actions/checkout@v4
+        with:
+          repository: signoz/primus
+          ref: main
+          path: .primus
+          token: ${{ steps.token.outputs.token }}
+      - name: build-info
+        id: build-info
+        run: |
+          echo "version=$($MAKE info-version)" >> $GITHUB_OUTPUT
+          echo "hash=$($MAKE info-commit-short)" >> $GITHUB_OUTPUT
+          echo "time=$($MAKE info-timestamp)" >> $GITHUB_OUTPUT
+          echo "branch=$($MAKE info-branch)" >> $GITHUB_OUTPUT
+
+          staging_label="${{ github.event.label.name }}"
+          if [[ "${staging_label}" == "staging:"* ]]; then
+              deployment=${staging_label#"staging:"}
+          elif [[ "${{ github.event.ref }}" == "refs/heads/main" ]]; then
+              deployment="staging"
+          else
+              echo "error: not able to determine deployment - please verify the PR label or the branch"
+              exit 1
+          fi
+          echo "deployment=${deployment}" >> $GITHUB_OUTPUT
+      - name: create-dotenv
+        run: |
+          mkdir -p frontend
+          echo 'CI=1' > frontend/.env
+          echo 'INTERCOM_APP_ID="${{ secrets.INTERCOM_APP_ID }}"' >> frontend/.env
+          echo 'SEGMENT_ID="${{ secrets.SEGMENT_ID }}"' >> frontend/.env
+          echo 'SENTRY_AUTH_TOKEN="${{ secrets.SENTRY_AUTH_TOKEN }}"' >> frontend/.env
+          echo 'SENTRY_ORG="${{ secrets.SENTRY_ORG }}"' >> frontend/.env
+          echo 'SENTRY_PROJECT_ID="${{ secrets.SENTRY_PROJECT_ID }}"' >> frontend/.env
+          echo 'SENTRY_DSN="${{ secrets.SENTRY_DSN }}"' >> frontend/.env
+          echo 'TUNNEL_URL="${{ secrets.TUNNEL_URL }}"' >> frontend/.env
+          echo 'TUNNEL_DOMAIN="${{ secrets.TUNNEL_DOMAIN }}"' >> frontend/.env
+          echo 'POSTHOG_KEY="${{ secrets.POSTHOG_KEY }}"' >> frontend/.env
+          echo 'CUSTOMERIO_ID="${{ secrets.CUSTOMERIO_ID }}"' >> frontend/.env
+          echo 'CUSTOMERIO_SITE_ID="${{ secrets.CUSTOMERIO_SITE_ID }}"' >> frontend/.env
+      - name: cache-dotenv
+        uses: actions/cache@v4
+        with:
+          path: frontend/.env
+          key: dotenv-${{ github.sha }}
+  js-build:
+    uses: signoz/primus.workflows/.github/workflows/js-build.yaml@main
+    needs: prepare
+    secrets: inherit
+    with:
+      PRIMUS_REF: main
+      JS_SRC: frontend
+      JS_INPUT_ARTIFACT_CACHE_KEY: dotenv-${{ github.sha }}
+      JS_INPUT_ARTIFACT_PATH: frontend/.env
+      JS_OUTPUT_ARTIFACT_CACHE_KEY: jsbuild-${{ github.sha }}
+      JS_OUTPUT_ARTIFACT_PATH: frontend/build
+      DOCKER_BUILD: false
+      DOCKER_MANIFEST: false
+  go-build:
+    uses: signoz/primus.workflows/.github/workflows/go-build.yaml@main
+    needs: [prepare, js-build]
+    secrets: inherit
+    with:
+      PRIMUS_REF: main
+      GO_INPUT_ARTIFACT_CACHE_KEY: jsbuild-${{ github.sha }}
+      GO_INPUT_ARTIFACT_PATH: frontend/build
+      GO_BUILD_CONTEXT: ./ee/query-service
+      GO_BUILD_FLAGS: >-
+        -tags timetzdata
+        -ldflags='-linkmode external -extldflags \"-static\" -s -w
+        -X github.com/SigNoz/signoz/pkg/version.version=${{ needs.prepare.outputs.version }}
+        -X github.com/SigNoz/signoz/pkg/version.variant=enterprise
+        -X github.com/SigNoz/signoz/pkg/version.hash=${{ needs.prepare.outputs.hash }}
+        -X github.com/SigNoz/signoz/pkg/version.time=${{ needs.prepare.outputs.time }}
+        -X github.com/SigNoz/signoz/pkg/version.branch=${{ needs.prepare.outputs.branch }}
+        -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.staging.signoz.cloud
+        -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.staging.signoz.cloud/api/v1'
+      GO_CGO_ENABLED: 1
+      DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}'
+      DOCKER_DOCKERFILE_PATH: ./ee/query-service/Dockerfile.multi-arch
+      DOCKER_MANIFEST: true
+      DOCKER_PROVIDERS: gcp
+  staging:
+    if: ${{ contains(github.event.label.name, 'staging:') || github.event.ref == 'refs/heads/main' }}
+    uses: signoz/primus.workflows/.github/workflows/github-trigger.yaml@main
+    secrets: inherit
+    needs: [prepare, go-build]
+    with:
+      PRIMUS_REF: main
+      GITHUB_ENVIRONMENT: staging
+      GITHUB_SILENT: true
+      GITHUB_REPOSITORY_NAME: charts-saas-v3-staging
+      GITHUB_EVENT_NAME: releaser
+      GITHUB_EVENT_PAYLOAD: "{\"deployment\": \"${{ needs.prepare.outputs.deployment }}\", \"signoz_version\": \"${{ needs.prepare.outputs.version }}\"}"

.github/workflows/staging-deployment.yaml

@@ -36,12 +36,17 @@ jobs:
           echo "GITHUB_BRANCH: ${GITHUB_BRANCH}"
           echo "GITHUB_SHA: ${GITHUB_SHA}"
           export VERSION="${GITHUB_SHA:0:7}" # needed for child process to access it
-          export OTELCOL_TAG="main"
           export PATH="/usr/local/go/bin/:$PATH" # needed for Golang to work
           export KAFKA_SPAN_EVAL="true"
-          docker system prune --force
+          docker system prune --force --all
-          docker pull signoz/signoz-otel-collector:main
+          OTELCOL_TAG=$(curl -s https://api.github.com/repos/SigNoz/signoz-otel-collector/releases/latest | jq -r '.tag_name // "not-found"')
-          docker pull signoz/signoz-schema-migrator:main
+          if [[ "${OTELCOL_TAG}" == "not-found" ]]; then
+            echo "warning: unable to determine latest SigNoz OtelCollector release tag, skipping latest otelcol deployment"
+          else
+            export OTELCOL_TAG=${OTELCOL_TAG}
+            docker pull signoz/signoz-otel-collector:${OTELCOL_TAG}
+            docker pull signoz/signoz-schema-migrator:${OTELCOL_TAG}
+          fi
           cd ~/signoz
           git status
           git add .

.github/workflows/testing-deployment.yaml

@@ -38,7 +38,7 @@ jobs:
           export VERSION="${GITHUB_SHA:0:7}" # needed for child process to access it
           export DEV_BUILD="1"
           export PATH="/usr/local/go/bin/:$PATH" # needed for Golang to work
-          docker system prune --force
+          docker system prune --force --all
           cd ~/signoz
           git status
           git add .