Merge branch 'develop' of https://github.com/SigNoz/signoz into develop

.github/workflows/push.yaml

@@ -11,6 +11,41 @@ on:
 jobs:
 
   image-build-and-push-query-service:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: benjlevesque/short-sha@v1.2
+        id: short-sha
+      - name: Get branch name
+        id: branch-name
+        uses: tj-actions/branch-names@v5.1
+      - name: Set docker tag environment
+        run: |
+          if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then
+            tag="${{ steps.branch-name.outputs.tag }}"
+            tag="${tag:1}"
+            echo "DOCKER_TAG=${tag}-oss" >> $GITHUB_ENV
+          elif [ '${{ steps.branch-name.outputs.current_branch }}' == 'main' ]; then
+            echo "DOCKER_TAG=latest-oss" >> $GITHUB_ENV
+          else
+            echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}-oss" >> $GITHUB_ENV
+          fi
+      - name: Build and push docker image
+        run: make build-push-query-service
+
+  image-build-and-push-ee-query-service:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -43,7 +78,7 @@ jobs:
             echo "DOCKER_TAG=${{ steps.branch-name.outputs.current_branch }}" >> $GITHUB_ENV
           fi
       - name: Build and push docker image
-        run: make build-push-query-service
+        run: make build-push-ee-query-service
 
   image-build-and-push-frontend:
     runs-on: ubuntu-latest

ee/query-service/usage/manager.go

@@ -139,6 +139,8 @@ func (lm *Manager) UsageExporter(ctx context.Context) {
 			lm.CollectAndStoreUsage(ctx)
 		case <-uploadTicker.C:
 			lm.UploadUsage(ctx)
+			// remove the old snapshots
+			lm.repository.DropOldSnapshots(ctx)
 		}
 	}
 }

ee/query-service/usage/repository/repository.go

@@ -17,6 +17,7 @@ import (
 
 const (
 	MaxFailedSyncCount = 9 // a snapshot will be ignored if the max failed count is greater than or equal to 9
+	SnapShotLife       = 3 * 24 * time.Hour
 )
 
 // Repository is usage Repository which stores usage snapshot in a secured DB
@@ -111,6 +112,18 @@ func (r *Repository) GetSnapshotsNotSynced(ctx context.Context) ([]*model.Usage,
 	return snapshots, nil
 }
 
+func (r *Repository) DropOldSnapshots(ctx context.Context) error {
+	query := `delete from usage where created_at <= $1`
+
+	_, err := r.db.ExecContext(ctx, query, time.Now().Add(-(SnapShotLife)))
+	if err != nil {
+		zap.S().Errorf("failed to remove old snapshots from db: %v", zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
 // CheckSnapshotGtCreatedAt checks if there is any snapshot greater than the provided timestamp
 func (r *Repository) CheckSnapshotGtCreatedAt(ctx context.Context, ts time.Time) (bool, error) {
 

go.mod

@@ -145,4 +145,4 @@ require (
 	k8s.io/client-go v8.0.0+incompatible // indirect
 )
 
-replace github.com/prometheus/prometheus => github.com/SigNoz/prometheus v1.9.73
+replace github.com/prometheus/prometheus => github.com/SigNoz/prometheus v1.9.74

go.sum

@@ -57,8 +57,8 @@ github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb h1:bneLSKPf9YUSFmafKx32bynV6QrzViL/s+ZDvQxH1E4=
 github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb/go.mod h1:JznGDNg9x1cujDKa22RaQOimOvvEfy3nxzDGd8XDgmA=
-github.com/SigNoz/prometheus v1.9.73 h1:f6PjQrJGoCot9iJp/tWoKwgy0HTIqicYave4K3fT9ro=
-github.com/SigNoz/prometheus v1.9.73/go.mod h1:Y4J9tGDmacMC+EcOTp+EIAn2C1sN+9kE+idyVKadiVM=
+github.com/SigNoz/prometheus v1.9.74 h1:/AcKVZ80Cg4FQ/quMDLO4Ejyeb3KDjgUKveiN+OXVL8=
+github.com/SigNoz/prometheus v1.9.74/go.mod h1:Y4J9tGDmacMC+EcOTp+EIAn2C1sN+9kE+idyVKadiVM=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=