Merge pull request #6625 from SigNoz/release/v0.62.x

Release/v0.62.x

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -146,7 +146,7 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.61.0
+    image: signoz/query-service:0.62.0
     command:
       [
         "-config=/root/config/prometheus.yml",
@@ -186,7 +186,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:0.61.0
+    image: signoz/frontend:0.62.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -199,7 +199,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.15
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -237,7 +237,7 @@ services:
       - query-service
 
   otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.111.14
+      image: signoz/signoz-schema-migrator:0.111.15
       deploy:
         restart_policy:
           condition: on-failure

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -69,10 +69,12 @@ services:
       - --storage.path=/data
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
     container_name: otel-migrator
     command:
+      - "sync" 
       - "--dsn=tcp://clickhouse:9000"
+      - "--up=" 
     depends_on:
       clickhouse:
         condition: service_healthy
@@ -84,7 +86,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.15
     command:
       [
         "--config=/etc/otel-collector-config.yaml",

deploy/docker/clickhouse-setup/docker-compose-minimal.yaml

@@ -162,7 +162,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.62.0}
     container_name: signoz-query-service
     command:
       [
@@ -201,7 +201,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.62.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -213,7 +213,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator-sync:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
     container_name: otel-migrator-sync
     command:
       - "sync"
@@ -228,7 +228,7 @@ services:
       #   condition: service_healthy
 
   otel-collector-migrator-async:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
     container_name: otel-migrator-async
     command:
       - "async"
@@ -245,7 +245,7 @@ services:
       #   condition: service_healthy
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.15}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/docker-compose.testing.yaml

@@ -167,7 +167,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.62.0}
     container_name: signoz-query-service
     command:
       [
@@ -208,7 +208,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.62.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -220,7 +220,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -234,7 +234,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.15}
     container_name: signoz-otel-collector
     command:
       [

ee/query-service/app/api/traces.go

@@ -2,31 +2,32 @@ package api
 
 import (
 	"net/http"
+
+	"go.signoz.io/signoz/ee/query-service/app/db"
+	"go.signoz.io/signoz/ee/query-service/model"
+	baseapp "go.signoz.io/signoz/pkg/query-service/app"
+	basemodel "go.signoz.io/signoz/pkg/query-service/model"
+	"go.uber.org/zap"
 )
 
 func (ah *APIHandler) searchTraces(w http.ResponseWriter, r *http.Request) {
 
+	if !ah.CheckFeature(basemodel.SmartTraceDetail) {
+		zap.L().Info("SmartTraceDetail feature is not enabled in this plan")
 		ah.APIHandler.SearchTraces(w, r)
 		return
+	}
+	searchTracesParams, err := baseapp.ParseSearchTracesParams(r)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params")
+		return
+	}
 
-	// This is commented since this will be taken care by new trace API
+	result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm)
+	if ah.HandleError(w, err, http.StatusBadRequest) {
+		return
+	}
 
-	// if !ah.CheckFeature(basemodel.SmartTraceDetail) {
+	ah.WriteJSON(w, r, result)
-	// 	zap.L().Info("SmartTraceDetail feature is not enabled in this plan")
-	// 	ah.APIHandler.SearchTraces(w, r)
-	// 	return
-	// }
-	// searchTracesParams, err := baseapp.ParseSearchTracesParams(r)
-	// if err != nil {
-	// 	RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params")
-	// 	return
-	// }
-
-	// result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm)
-	// if ah.HandleError(w, err, http.StatusBadRequest) {
-	// 	return
-	// }
-
-	// ah.WriteJSON(w, r, result)
 
 }

frontend/src/container/CreateAlertRule/defaults.ts

@@ -117,7 +117,7 @@ export const logAlertDefaults: AlertDef = {
 			chQueries: {
 				A: {
 					name: 'A',
-					query: `select \ntoStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 MINUTE) AS interval, \ntoFloat64(count()) as value \nFROM signoz_logs.distributed_logs_v2  \nWHERE timestamp BETWEEN {{.start_timestamp_nano}} AND {{.end_timestamp_nano}}  \nGROUP BY interval;\n\n-- available variables:\n-- \t{{.start_timestamp_nano}}\n-- \t{{.end_timestamp_nano}}\n\n-- required columns (or alias):\n-- \tvalue\n-- \tinterval`,
+					query: `select \ntoStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 MINUTE) AS interval, \ntoFloat64(count()) as value \nFROM signoz_logs.distributed_logs_v2  \nWHERE timestamp BETWEEN {{.start_timestamp_nano}} AND {{.end_timestamp_nano}} \nAND ts_bucket_start BETWEEN {{.start_timestamp}} - 1800 AND {{.end_timestamp}} \nGROUP BY interval;\n\n-- Please check docs here https://signoz.io/docs/userguide/logs_clickhouse_queries/\n\n-- available variables:\n-- \t{{.start_timestamp_nano}}\n-- \t{{.end_timestamp_nano}}\n-- \t{{.start_timestamp}}\n-- \t{{.end_timestamp}}\n\n-- required columns (or alias):\n-- \tvalue\n-- \tinterval`,
 					legend: '',
 					disabled: false,
 				},
@@ -149,7 +149,7 @@ export const traceAlertDefaults: AlertDef = {
 			chQueries: {
 				A: {
 					name: 'A',
-					query: `SELECT \n\ttoStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, \n\tstringTagMap['peer.service'] AS op_name, \n\ttoFloat64(avg(durationNano)) AS value \nFROM signoz_traces.distributed_signoz_index_v2  \nWHERE stringTagMap['peer.service']!='' \nAND timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} \nGROUP BY (op_name, interval);\n\n-- available variables:\n-- \t{{.start_datetime}}\n-- \t{{.end_datetime}}\n\n-- required column alias:\n-- \tvalue\n-- \tinterval`,
+					query: `SELECT \n\ttoStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, \n\tresource_string_service$$name AS \`service.name\`, \n\ttoFloat64(avg(duration_nano)) AS value \nFROM signoz_traces.distributed_signoz_index_v3  \nWHERE resource_string_service$$name !='' \nAND timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} \nAND ts_bucket_start BETWEEN {{.start_timestamp}} - 1800 AND {{.end_timestamp}} \nGROUP BY (\`service.name\`, interval);\n\n-- Please check docs here https://signoz.io/docs/userguide/writing-clickhouse-traces-query/\n\n-- available variables:\n-- \t{{.start_datetime}}\n-- \t{{.end_datetime}}\n-- \t{{.start_timestamp}}\n-- \t{{.end_timestamp}}\n\n-- required column alias:\n-- \tvalue\n-- \tinterval`,
 					legend: '',
 					disabled: false,
 				},

frontend/src/container/MetricsApplication/Tabs/Overview.tsx

@@ -145,7 +145,9 @@ function Application(): JSX.Element {
 		[servicename, topLevelOperations],
 	);
 
-	const operationPerSecWidget = getWidgetQueryBuilder({
+	const operationPerSecWidget = useMemo(
+		() =>
+			getWidgetQueryBuilder({
 				query: {
 					queryType: EQueryType.QUERY_BUILDER,
 					promql: [],
@@ -161,9 +163,13 @@ function Application(): JSX.Element {
 				panelTypes: PANEL_TYPES.TIME_SERIES,
 				yAxisUnit: 'ops',
 				id: SERVICE_CHART_ID.rps,
-	});
+			}),
+		[servicename, tagFilterItems, topLevelOperationsRoute],
+	);
 
-	const errorPercentageWidget = getWidgetQueryBuilder({
+	const errorPercentageWidget = useMemo(
+		() =>
+			getWidgetQueryBuilder({
 				query: {
 					queryType: EQueryType.QUERY_BUILDER,
 					promql: [],
@@ -179,7 +185,9 @@ function Application(): JSX.Element {
 				panelTypes: PANEL_TYPES.TIME_SERIES,
 				yAxisUnit: '%',
 				id: SERVICE_CHART_ID.errorPercentage,
-	});
+			}),
+		[servicename, tagFilterItems, topLevelOperationsRoute],
+	);
 
 	const stepInterval = useMemo(
 		() =>

frontend/src/container/MetricsApplication/Tabs/Overview/ServiceOverview.tsx

@@ -53,7 +53,9 @@ function ServiceOverview({
 		[isSpanMetricEnable, queries],
 	);
 
-	const latencyWidget = getWidgetQueryBuilder({
+	const latencyWidget = useMemo(
+		() =>
+			getWidgetQueryBuilder({
 				query: {
 					queryType: EQueryType.QUERY_BUILDER,
 					promql: [],
@@ -70,7 +72,9 @@ function ServiceOverview({
 				panelTypes: PANEL_TYPES.TIME_SERIES,
 				yAxisUnit: 'ns',
 				id: SERVICE_CHART_ID.latency,
-	});
+			}),
+		[isSpanMetricEnable, servicename, tagFilterItems, topLevelOperationsRoute],
+	);
 
 	const isQueryEnabled =
 		!topLevelOperationsIsLoading && topLevelOperationsRoute.length > 0;

frontend/src/container/SideNav/SideNav.tsx

@@ -16,7 +16,7 @@ import history from 'lib/history';
 import {
 	AlertTriangle,
 	CheckSquare,
-	RocketIcon,
+	PackagePlus,
 	UserCircle,
 } from 'lucide-react';
 import { MouseEvent, useCallback, useEffect, useMemo, useState } from 'react';
@@ -395,9 +395,9 @@ function SideNav({
 								onClickGetStarted(event);
 							}}
 						>
-							<RocketIcon size={16} />
+							<PackagePlus size={16} />
 
-							<div className="license tag nav-item-label"> Get Started </div>
+							<div className="license tag nav-item-label"> New source </div>
 						</Button>
 					</div>
 				)}

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.25.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.111.14
+	github.com/SigNoz/signoz-otel-collector v0.111.15
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3

go.sum

@@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE=
 github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA=
-github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0=
+github.com/SigNoz/signoz-otel-collector v0.111.15 h1:X2aV/FpMZ9kdN10byKrc2ODF39fAcOo6D68rqUFhxSk=
-github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
+github.com/SigNoz/signoz-otel-collector v0.111.15/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=

pkg/query-service/app/clickhouseReader/reader.go

@@ -1203,7 +1203,9 @@ func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetU
 	return &usageItems, nil
 }
 
-func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams) (*[]model.SearchSpansResult, error) {
+func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams,
+	smartTraceAlgorithm func(payload []model.SearchSpanResponseItem, targetSpanId string,
+		levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) {
 	searchSpansResult := []model.SearchSpansResult{
 		{
 			Columns:   []string{"__time", "SpanId", "TraceId", "ServiceName", "Name", "Kind", "DurationNano", "TagsKeys", "TagsValues", "References", "Events", "HasError", "StatusMessage", "StatusCodeString", "SpanKind"},
@@ -1318,10 +1320,30 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea
 	end = time.Now()
 	zap.L().Debug("getTraceSQLQuery unmarshal took: ", zap.Duration("duration", end.Sub(start)))
 
+	err = r.featureFlags.CheckFeature(model.SmartTraceDetail)
+	smartAlgoEnabled := err == nil
+	if len(searchScanResponses) > params.SpansRenderLimit && smartAlgoEnabled {
+		start = time.Now()
+		searchSpansResult, err = smartTraceAlgorithm(searchSpanResponses, params.SpanID, params.LevelUp, params.LevelDown, params.SpansRenderLimit)
+		if err != nil {
+			return nil, err
+		}
+		end = time.Now()
+		zap.L().Debug("smartTraceAlgo took: ", zap.Duration("duration", end.Sub(start)))
+		userEmail, err := auth.GetEmailFromJwt(ctx)
+		if err == nil {
+			data := map[string]interface{}{
+				"traceSize":        len(searchScanResponses),
+				"spansRenderLimit": params.SpansRenderLimit,
+			}
+			telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LARGE_TRACE_OPENED, data, userEmail, true, false)
+		}
+	} else {
 		for i, item := range searchSpanResponses {
 			spanEvents := item.GetValues()
 			searchSpansResult[0].Events[i] = spanEvents
 		}
+	}
 
 	searchSpansResult[0].StartTimestampMillis = startTime - (durationNano / 1000000)
 	searchSpansResult[0].EndTimestampMillis = endTime + (durationNano / 1000000)
@@ -1334,7 +1356,7 @@ func (r *ClickHouseReader) SearchTraces(ctx context.Context, params *model.Searc
 		levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) {
 
 	if r.useTraceNewSchema {
-		return r.SearchTracesV2(ctx, params)
+		return r.SearchTracesV2(ctx, params, smartTraceAlgorithm)
 	}
 
 	var countSpans uint64

pkg/query-service/app/dashboards/model.go

@@ -289,6 +289,10 @@ func GetDashboard(ctx context.Context, uuid string) (*Dashboard, *model.ApiError
 		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
 	}
 
+	if dashboard.Data["title"] == "Ingestion" && dashboard.Data["description"] != nil {
+		dashboard.Data["description"] = "This dashboard is deprecated. Please use the new Ingestion V2 dashboard. " + dashboard.Data["description"].(string)
+	}
+
 	return &dashboard, nil
 }
 

pkg/query-service/app/http_handler.go

@@ -3141,14 +3141,14 @@ func (aH *APIHandler) getProducerThroughputOverview(
 		Hash: make(map[string]struct{}),
 	}
 
-	queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache)
+	producerQueryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache)
 	if err != nil {
 		zap.L().Error(err.Error())
 		RespondError(w, apiErr, nil)
 		return
 	}
 
-	if err := validateQueryRangeParamsV3(queryRangeParams); err != nil {
+	if err := validateQueryRangeParamsV3(producerQueryRangeParams); err != nil {
 		zap.L().Error(err.Error())
 		RespondError(w, apiErr, nil)
 		return
@@ -3157,7 +3157,7 @@ func (aH *APIHandler) getProducerThroughputOverview(
 	var result []*v3.Result
 	var errQuriesByName map[string]error
 
-	result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), queryRangeParams)
+	result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), producerQueryRangeParams)
 	if err != nil {
 		apiErrObj := &model.ApiError{Typ: model.ErrorBadData, Err: err}
 		RespondError(w, apiErrObj, errQuriesByName)
@@ -3165,21 +3165,21 @@ func (aH *APIHandler) getProducerThroughputOverview(
 	}
 
 	for _, res := range result {
-		for _, list := range res.List {
+		for _, series := range res.Series {
-			serviceName, serviceNameOk := list.Data["service_name"].(*string)
+			serviceName, serviceNameOk := series.Labels["service_name"]
-			topicName, topicNameOk := list.Data["topic"].(*string)
+			topicName, topicNameOk := series.Labels["topic"]
-			params := []string{*serviceName, *topicName}
+			params := []string{serviceName, topicName}
 			hashKey := uniqueIdentifier(params, "#")
 			_, ok := attributeCache.Hash[hashKey]
 			if topicNameOk && serviceNameOk && !ok {
 				attributeCache.Hash[hashKey] = struct{}{}
-				attributeCache.TopicName = append(attributeCache.TopicName, *topicName)
+				attributeCache.TopicName = append(attributeCache.TopicName, topicName)
-				attributeCache.ServiceName = append(attributeCache.ServiceName, *serviceName)
+				attributeCache.ServiceName = append(attributeCache.ServiceName, serviceName)
 			}
 		}
 	}
 
-	queryRangeParams, err = mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-latency", attributeCache)
+	queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-byte-rate", attributeCache)
 	if err != nil {
 		zap.L().Error(err.Error())
 		RespondError(w, apiErr, nil)
@@ -3198,26 +3198,32 @@ func (aH *APIHandler) getProducerThroughputOverview(
 		return
 	}
 
-	latencyColumn := &v3.Result{QueryName: "latency"}
+	byteRateColumn := &v3.Result{QueryName: "byte_rate"}
-	var latencySeries []*v3.Row
+	var byteRateSeries []*v3.Series
 	for _, res := range resultFetchLatency {
-		for _, list := range res.List {
+		for _, series := range res.Series {
-			topic, topicOk := list.Data["topic"].(*string)
+			topic, topicOk := series.Labels["topic"]
-			serviceName, serviceNameOk := list.Data["service_name"].(*string)
+			serviceName, serviceNameOk := series.Labels["service_name"]
-			params := []string{*serviceName, *topic}
+			params := []string{serviceName, topic}
 			hashKey := uniqueIdentifier(params, "#")
 			_, ok := attributeCache.Hash[hashKey]
 			if topicOk && serviceNameOk && ok {
-				latencySeries = append(latencySeries, list)
+				byteRateSeries = append(byteRateSeries, series)
 			}
 		}
 	}
 
-	latencyColumn.List = latencySeries
+	byteRateColumn.Series = byteRateSeries
-	result = append(result, latencyColumn)
+	var latencyColumnResult []*v3.Result
+	latencyColumnResult = append(latencyColumnResult, byteRateColumn)
 
+	resultFetchLatency = postprocess.TransformToTableForBuilderQueries(latencyColumnResult, queryRangeParams)
+
+	result = postprocess.TransformToTableForClickHouseQueries(result)
+
+	result = append(result, resultFetchLatency[0])
 	resp := v3.QueryRangeResponse{
-		Result: resultFetchLatency,
+		Result: result,
 	}
 	aH.Respond(w, resp)
 }

pkg/query-service/app/integrations/messagingQueues/kafka/translator.go

@@ -61,7 +61,7 @@ func buildClickHouseQueryNetwork(messagingQueue *MessagingQueue, queueType strin
 
 func buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd int64, attributeCache *Clients) (map[string]*v3.BuilderQuery, error) {
 	bq := make(map[string]*v3.BuilderQuery)
-	queryName := fmt.Sprintf("latency")
+	queryName := fmt.Sprintf("byte_rate")
 
 	chq := &v3.BuilderQuery{
 		QueryName:    queryName,
@@ -69,6 +69,9 @@ func buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd int64, attrib
 		DataSource:   v3.DataSourceMetrics,
 		AggregateAttribute: v3.AttributeKey{
 			Key:      "kafka_producer_byte_rate",
+			DataType: v3.AttributeKeyDataTypeFloat64,
+			Type:     v3.AttributeKeyType("Gauge"),
+			IsColumn: true,
 		},
 		AggregateOperator: v3.AggregateOperatorAvg,
 		Temporality:       v3.Unspecified,
@@ -276,7 +279,7 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string,
 		cq, err = buildCompositeQuery(&v3.ClickHouseQuery{
 			Query: query,
 		}, queryContext)
-	} else if queryContext == "producer-throughput-overview-latency" {
+	} else if queryContext == "producer-throughput-overview-byte-rate" {
 		bhq, err := buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd, attributeCache)
 		if err != nil {
 			return nil, err
@@ -284,7 +287,8 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string,
 		cq = &v3.CompositeQuery{
 			QueryType:      v3.QueryTypeBuilder,
 			BuilderQueries: bhq,
-			PanelType:      v3.PanelTypeList,
+			PanelType:      v3.PanelTypeTable,
+			FillGaps:       false,
 		}
 	}
 
@@ -315,7 +319,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer
 		if !ok {
 			return nil, fmt.Errorf("invalid type for Topic")
 		}
-		if queryContext != "consumer-throughput-details" {
+		if !(queryContext == "consumer-throughput-details" || queryContext == "producer-throughput-details") {
 			partition, ok = messagingQueue.Variables["partition"]
 			if !ok {
 				return nil, fmt.Errorf("invalid type for Partition")
@@ -364,7 +368,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer
 
 func buildCompositeQuery(chq *v3.ClickHouseQuery, queryContext string) (*v3.CompositeQuery, error) {
 
-	if queryContext == "producer-consumer-eval" || queryContext == "producer-throughput-overview" {
+	if queryContext == "producer-consumer-eval" {
 		return &v3.CompositeQuery{
 			QueryType:         v3.QueryTypeClickHouseSQL,
 			ClickHouseQueries: map[string]*v3.ClickHouseQuery{queryContext: chq},

pkg/query-service/app/traces/v4/query_builder.go

@@ -355,8 +355,9 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.
 					filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery)
 				}
 			} else {
-				column := getColumnName(mq.AggregateAttribute)
+				cType := getClickHouseTracesColumnType(mq.AggregateAttribute.Type)
-				filterSubQuery = fmt.Sprintf("%s AND has(%s, '%s')", filterSubQuery, column, mq.AggregateAttribute.Key)
+				cDataType := getClickHouseTracesColumnDataType(mq.AggregateAttribute.DataType)
+				filterSubQuery = fmt.Sprintf("%s AND mapContains(%s_%s, '%s')", filterSubQuery, cType, cDataType, mq.AggregateAttribute.Key)
 			}
 		}
 		op := "toFloat64(count())"

pkg/query-service/app/traces/v4/query_builder_test.go

@@ -451,6 +451,62 @@ func Test_buildTracesQuery(t *testing.T) {
 				"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') " +
 				"group by `http.method` order by `http.method` ASC",
 		},
+		{
+			name: "Test buildTracesQuery - count with attr",
+			args: args{
+				panelType: v3.PanelTypeTable,
+				start:     1680066360726210000,
+				end:       1680066458000000000,
+				step:      1000,
+				mq: &v3.BuilderQuery{
+					AggregateOperator:  v3.AggregateOperatorCount,
+					AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
+					Filters: &v3.FilterSet{
+						Items: []v3.FilterItem{
+							{
+								Key:      v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
+								Value:    100,
+								Operator: v3.FilterOperatorEqual,
+							},
+						},
+					},
+					GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
+					OrderBy: []v3.OrderBy{
+						{ColumnName: "http.method", Order: "ASC"}},
+				},
+			},
+			want: "SELECT  attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
+				"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND mapContains(attributes_string, 'name') " +
+				"group by `http.method` order by `http.method` ASC",
+		},
+		{
+			name: "Test buildTracesQuery - count with mat attr",
+			args: args{
+				panelType: v3.PanelTypeTable,
+				start:     1680066360726210000,
+				end:       1680066458000000000,
+				step:      1000,
+				mq: &v3.BuilderQuery{
+					AggregateOperator:  v3.AggregateOperatorCount,
+					AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
+					Filters: &v3.FilterSet{
+						Items: []v3.FilterItem{
+							{
+								Key:      v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
+								Value:    100,
+								Operator: v3.FilterOperatorEqual,
+							},
+						},
+					},
+					GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
+					OrderBy: []v3.OrderBy{
+						{ColumnName: "http.method", Order: "ASC"}},
+				},
+			},
+			want: "SELECT  attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
+				"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND name != '' " +
+				"group by `http.method` order by `http.method` ASC",
+		},
 		{
 			name: "Test buildTracesQuery",
 			args: args{

pkg/query-service/migrate/migate.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"github.com/jmoiron/sqlx"
+	"go.uber.org/zap"
 )
 
 type DataMigration struct {
@@ -53,6 +54,17 @@ func Migrate(dsn string) error {
 		return err
 	}
 
+	if m, err := getMigrationVersion(conn, "0.62_ingestion_dashboard"); err == nil && m == nil {
+		if err := migrateIngestionDashboard(conn); err != nil {
+			zap.L().Error("failed to migrate 0.62_ingestion_dashboard", zap.Error(err))
+		} else {
+			_, err := conn.Exec("INSERT INTO data_migrations (version, succeeded) VALUES ('0.62_ingestion_dashboard', true)")
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
 

pkg/query-service/rules/threshold_rule.go

@@ -75,6 +75,7 @@ func NewThresholdRule(
 	t := ThresholdRule{
 		BaseRule:          baseRule,
 		version:           p.Version,
+		useTraceNewSchema: useTraceNewSchema,
 	}
 
 	querierOption := querier.QuerierOptions{