From 02d8fdb2121ec1f239f4b05a6c45103e2ae24253 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Thu, 5 Dec 2024 09:30:39 +0530 Subject: [PATCH 01/10] fix: enable smart trace detail (#6596) --- ee/query-service/app/api/traces.go | 41 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/ee/query-service/app/api/traces.go b/ee/query-service/app/api/traces.go index 5c65089cd0..3864fc672e 100644 --- a/ee/query-service/app/api/traces.go +++ b/ee/query-service/app/api/traces.go @@ -2,31 +2,32 @@ package api import ( "net/http" + + "go.signoz.io/signoz/ee/query-service/app/db" + "go.signoz.io/signoz/ee/query-service/model" + baseapp "go.signoz.io/signoz/pkg/query-service/app" + basemodel "go.signoz.io/signoz/pkg/query-service/model" + "go.uber.org/zap" ) func (ah *APIHandler) searchTraces(w http.ResponseWriter, r *http.Request) { - ah.APIHandler.SearchTraces(w, r) - return + if !ah.CheckFeature(basemodel.SmartTraceDetail) { + zap.L().Info("SmartTraceDetail feature is not enabled in this plan") + ah.APIHandler.SearchTraces(w, r) + return + } + searchTracesParams, err := baseapp.ParseSearchTracesParams(r) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params") + return + } - // This is commented since this will be taken care by new trace API + result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm) + if ah.HandleError(w, err, http.StatusBadRequest) { + return + } - // if !ah.CheckFeature(basemodel.SmartTraceDetail) { - // zap.L().Info("SmartTraceDetail feature is not enabled in this plan") - // ah.APIHandler.SearchTraces(w, r) - // return - // } - // searchTracesParams, err := baseapp.ParseSearchTracesParams(r) - // if err != nil { - // RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params") - // return - // } - - // result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm) - // if ah.HandleError(w, err, http.StatusBadRequest) { - // return - // } - - // ah.WriteJSON(w, r, result) + ah.WriteJSON(w, r, result) } From d4458d65ad3f8f095cf1cfc41a03e7d6eb505bbc Mon Sep 17 00:00:00 2001 From: deepakdinesh1123 <52698821+deepakdinesh1123@users.noreply.github.com> Date: Sat, 7 Dec 2024 20:15:07 +0530 Subject: [PATCH 02/10] fix(docker): otel-collector-migrator command fixed in docker-compose-core.yaml (#6564) Co-authored-by: Srikanth Chekuri --- deploy/docker/clickhouse-setup/docker-compose-core.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml index f4c1d9f1ee..5bade6b2da 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml @@ -72,7 +72,9 @@ services: image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14} container_name: otel-migrator command: + - "sync" - "--dsn=tcp://clickhouse:9000" + - "--up=" depends_on: clickhouse: condition: service_healthy From 8c6225185daadf0f5f24d0dc6319619264760ee7 Mon Sep 17 00:00:00 2001 From: amlannandy Date: Thu, 5 Dec 2024 12:47:18 +0530 Subject: [PATCH 03/10] chore: replace the 'Get started' CTA with 'New source' --- frontend/src/container/SideNav/SideNav.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/container/SideNav/SideNav.tsx b/frontend/src/container/SideNav/SideNav.tsx index f3bdbe0870..6116196bcc 100644 --- a/frontend/src/container/SideNav/SideNav.tsx +++ b/frontend/src/container/SideNav/SideNav.tsx @@ -16,7 +16,7 @@ import history from 'lib/history'; import { AlertTriangle, CheckSquare, - RocketIcon, + PackagePlus, UserCircle, } from 'lucide-react'; import { MouseEvent, useCallback, useEffect, useMemo, useState } from 'react'; @@ -395,9 +395,9 @@ function SideNav({ onClickGetStarted(event); }} > - + -
Get Started
+
New source
)} From 51baf7f8d3766f1723f703d6ddc8ac911eabeda1 Mon Sep 17 00:00:00 2001 From: Shivanshu Raj Shrivastava Date: Mon, 9 Dec 2024 20:48:41 +0530 Subject: [PATCH 04/10] feat: add byte rate to producer API (#6579) * fix: fix partition check for topic throughput --- pkg/query-service/app/http_handler.go | 46 +++++++++++-------- .../messagingQueues/kafka/translator.go | 16 ++++--- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 5031cf7123..3e25ab23c8 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -3141,14 +3141,14 @@ func (aH *APIHandler) getProducerThroughputOverview( Hash: make(map[string]struct{}), } - queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache) + producerQueryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache) if err != nil { zap.L().Error(err.Error()) RespondError(w, apiErr, nil) return } - if err := validateQueryRangeParamsV3(queryRangeParams); err != nil { + if err := validateQueryRangeParamsV3(producerQueryRangeParams); err != nil { zap.L().Error(err.Error()) RespondError(w, apiErr, nil) return @@ -3157,7 +3157,7 @@ func (aH *APIHandler) getProducerThroughputOverview( var result []*v3.Result var errQuriesByName map[string]error - result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), queryRangeParams) + result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), producerQueryRangeParams) if err != nil { apiErrObj := &model.ApiError{Typ: model.ErrorBadData, Err: err} RespondError(w, apiErrObj, errQuriesByName) @@ -3165,21 +3165,21 @@ func (aH *APIHandler) getProducerThroughputOverview( } for _, res := range result { - for _, list := range res.List { - serviceName, serviceNameOk := list.Data["service_name"].(*string) - topicName, topicNameOk := list.Data["topic"].(*string) - params := []string{*serviceName, *topicName} + for _, series := range res.Series { + serviceName, serviceNameOk := series.Labels["service_name"] + topicName, topicNameOk := series.Labels["topic"] + params := []string{serviceName, topicName} hashKey := uniqueIdentifier(params, "#") _, ok := attributeCache.Hash[hashKey] if topicNameOk && serviceNameOk && !ok { attributeCache.Hash[hashKey] = struct{}{} - attributeCache.TopicName = append(attributeCache.TopicName, *topicName) - attributeCache.ServiceName = append(attributeCache.ServiceName, *serviceName) + attributeCache.TopicName = append(attributeCache.TopicName, topicName) + attributeCache.ServiceName = append(attributeCache.ServiceName, serviceName) } } } - queryRangeParams, err = mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-latency", attributeCache) + queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-byte-rate", attributeCache) if err != nil { zap.L().Error(err.Error()) RespondError(w, apiErr, nil) @@ -3198,26 +3198,32 @@ func (aH *APIHandler) getProducerThroughputOverview( return } - latencyColumn := &v3.Result{QueryName: "latency"} - var latencySeries []*v3.Row + byteRateColumn := &v3.Result{QueryName: "byte_rate"} + var byteRateSeries []*v3.Series for _, res := range resultFetchLatency { - for _, list := range res.List { - topic, topicOk := list.Data["topic"].(*string) - serviceName, serviceNameOk := list.Data["service_name"].(*string) - params := []string{*serviceName, *topic} + for _, series := range res.Series { + topic, topicOk := series.Labels["topic"] + serviceName, serviceNameOk := series.Labels["service_name"] + params := []string{serviceName, topic} hashKey := uniqueIdentifier(params, "#") _, ok := attributeCache.Hash[hashKey] if topicOk && serviceNameOk && ok { - latencySeries = append(latencySeries, list) + byteRateSeries = append(byteRateSeries, series) } } } - latencyColumn.List = latencySeries - result = append(result, latencyColumn) + byteRateColumn.Series = byteRateSeries + var latencyColumnResult []*v3.Result + latencyColumnResult = append(latencyColumnResult, byteRateColumn) + resultFetchLatency = postprocess.TransformToTableForBuilderQueries(latencyColumnResult, queryRangeParams) + + result = postprocess.TransformToTableForClickHouseQueries(result) + + result = append(result, resultFetchLatency[0]) resp := v3.QueryRangeResponse{ - Result: resultFetchLatency, + Result: result, } aH.Respond(w, resp) } diff --git a/pkg/query-service/app/integrations/messagingQueues/kafka/translator.go b/pkg/query-service/app/integrations/messagingQueues/kafka/translator.go index d7dc96d470..1a60b90e56 100644 --- a/pkg/query-service/app/integrations/messagingQueues/kafka/translator.go +++ b/pkg/query-service/app/integrations/messagingQueues/kafka/translator.go @@ -61,14 +61,17 @@ func buildClickHouseQueryNetwork(messagingQueue *MessagingQueue, queueType strin func buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd int64, attributeCache *Clients) (map[string]*v3.BuilderQuery, error) { bq := make(map[string]*v3.BuilderQuery) - queryName := fmt.Sprintf("latency") + queryName := fmt.Sprintf("byte_rate") chq := &v3.BuilderQuery{ QueryName: queryName, StepInterval: common.MinAllowedStepInterval(unixMilliStart, unixMilliEnd), DataSource: v3.DataSourceMetrics, AggregateAttribute: v3.AttributeKey{ - Key: "kafka_producer_byte_rate", + Key: "kafka_producer_byte_rate", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, }, AggregateOperator: v3.AggregateOperatorAvg, Temporality: v3.Unspecified, @@ -276,7 +279,7 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string, cq, err = buildCompositeQuery(&v3.ClickHouseQuery{ Query: query, }, queryContext) - } else if queryContext == "producer-throughput-overview-latency" { + } else if queryContext == "producer-throughput-overview-byte-rate" { bhq, err := buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd, attributeCache) if err != nil { return nil, err @@ -284,7 +287,8 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string, cq = &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, BuilderQueries: bhq, - PanelType: v3.PanelTypeList, + PanelType: v3.PanelTypeTable, + FillGaps: false, } } @@ -315,7 +319,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer if !ok { return nil, fmt.Errorf("invalid type for Topic") } - if queryContext != "consumer-throughput-details" { + if !(queryContext == "consumer-throughput-details" || queryContext == "producer-throughput-details") { partition, ok = messagingQueue.Variables["partition"] if !ok { return nil, fmt.Errorf("invalid type for Partition") @@ -364,7 +368,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer func buildCompositeQuery(chq *v3.ClickHouseQuery, queryContext string) (*v3.CompositeQuery, error) { - if queryContext == "producer-consumer-eval" || queryContext == "producer-throughput-overview" { + if queryContext == "producer-consumer-eval" { return &v3.CompositeQuery{ QueryType: v3.QueryTypeClickHouseSQL, ClickHouseQueries: map[string]*v3.ClickHouseQuery{queryContext: chq}, From 8d2666004b658cef095c7e0663091fe54973d797 Mon Sep 17 00:00:00 2001 From: amlannandy Date: Sat, 7 Dec 2024 14:32:29 +0530 Subject: [PATCH 05/10] fix: resolve the unneccessary refetching of graphs on service details page --- .../MetricsApplication/Tabs/Overview.tsx | 72 ++++++++++--------- .../Tabs/Overview/ServiceOverview.tsx | 38 +++++----- 2 files changed, 61 insertions(+), 49 deletions(-) diff --git a/frontend/src/container/MetricsApplication/Tabs/Overview.tsx b/frontend/src/container/MetricsApplication/Tabs/Overview.tsx index 77bf86b80f..6decf72b39 100644 --- a/frontend/src/container/MetricsApplication/Tabs/Overview.tsx +++ b/frontend/src/container/MetricsApplication/Tabs/Overview.tsx @@ -145,41 +145,49 @@ function Application(): JSX.Element { [servicename, topLevelOperations], ); - const operationPerSecWidget = getWidgetQueryBuilder({ - query: { - queryType: EQueryType.QUERY_BUILDER, - promql: [], - builder: operationPerSec({ - servicename, - tagFilterItems, - topLevelOperations: topLevelOperationsRoute, + const operationPerSecWidget = useMemo( + () => + getWidgetQueryBuilder({ + query: { + queryType: EQueryType.QUERY_BUILDER, + promql: [], + builder: operationPerSec({ + servicename, + tagFilterItems, + topLevelOperations: topLevelOperationsRoute, + }), + clickhouse_sql: [], + id: uuid(), + }, + title: GraphTitle.RATE_PER_OPS, + panelTypes: PANEL_TYPES.TIME_SERIES, + yAxisUnit: 'ops', + id: SERVICE_CHART_ID.rps, }), - clickhouse_sql: [], - id: uuid(), - }, - title: GraphTitle.RATE_PER_OPS, - panelTypes: PANEL_TYPES.TIME_SERIES, - yAxisUnit: 'ops', - id: SERVICE_CHART_ID.rps, - }); + [servicename, tagFilterItems, topLevelOperationsRoute], + ); - const errorPercentageWidget = getWidgetQueryBuilder({ - query: { - queryType: EQueryType.QUERY_BUILDER, - promql: [], - builder: errorPercentage({ - servicename, - tagFilterItems, - topLevelOperations: topLevelOperationsRoute, + const errorPercentageWidget = useMemo( + () => + getWidgetQueryBuilder({ + query: { + queryType: EQueryType.QUERY_BUILDER, + promql: [], + builder: errorPercentage({ + servicename, + tagFilterItems, + topLevelOperations: topLevelOperationsRoute, + }), + clickhouse_sql: [], + id: uuid(), + }, + title: GraphTitle.ERROR_PERCENTAGE, + panelTypes: PANEL_TYPES.TIME_SERIES, + yAxisUnit: '%', + id: SERVICE_CHART_ID.errorPercentage, }), - clickhouse_sql: [], - id: uuid(), - }, - title: GraphTitle.ERROR_PERCENTAGE, - panelTypes: PANEL_TYPES.TIME_SERIES, - yAxisUnit: '%', - id: SERVICE_CHART_ID.errorPercentage, - }); + [servicename, tagFilterItems, topLevelOperationsRoute], + ); const stepInterval = useMemo( () => diff --git a/frontend/src/container/MetricsApplication/Tabs/Overview/ServiceOverview.tsx b/frontend/src/container/MetricsApplication/Tabs/Overview/ServiceOverview.tsx index 24e2233244..42f54c1448 100644 --- a/frontend/src/container/MetricsApplication/Tabs/Overview/ServiceOverview.tsx +++ b/frontend/src/container/MetricsApplication/Tabs/Overview/ServiceOverview.tsx @@ -53,24 +53,28 @@ function ServiceOverview({ [isSpanMetricEnable, queries], ); - const latencyWidget = getWidgetQueryBuilder({ - query: { - queryType: EQueryType.QUERY_BUILDER, - promql: [], - builder: latency({ - servicename, - tagFilterItems, - isSpanMetricEnable, - topLevelOperationsRoute, + const latencyWidget = useMemo( + () => + getWidgetQueryBuilder({ + query: { + queryType: EQueryType.QUERY_BUILDER, + promql: [], + builder: latency({ + servicename, + tagFilterItems, + isSpanMetricEnable, + topLevelOperationsRoute, + }), + clickhouse_sql: [], + id: uuid(), + }, + title: GraphTitle.LATENCY, + panelTypes: PANEL_TYPES.TIME_SERIES, + yAxisUnit: 'ns', + id: SERVICE_CHART_ID.latency, }), - clickhouse_sql: [], - id: uuid(), - }, - title: GraphTitle.LATENCY, - panelTypes: PANEL_TYPES.TIME_SERIES, - yAxisUnit: 'ns', - id: SERVICE_CHART_ID.latency, - }); + [isSpanMetricEnable, servicename, tagFilterItems, topLevelOperationsRoute], + ); const isQueryEnabled = !topLevelOperationsIsLoading && topLevelOperationsRoute.length > 0; From f9d4cf19e924ad77e1c21038961685bbc2bf0107 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Tue, 10 Dec 2024 13:43:51 +0530 Subject: [PATCH 06/10] fix: smartTraceAlgo for new schema (#6602) --- .../app/clickhouseReader/reader.go | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 256ab98d68..4f1aa99dfc 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1203,7 +1203,9 @@ func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetU return &usageItems, nil } -func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams) (*[]model.SearchSpansResult, error) { +func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams, + smartTraceAlgorithm func(payload []model.SearchSpanResponseItem, targetSpanId string, + levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) { searchSpansResult := []model.SearchSpansResult{ { Columns: []string{"__time", "SpanId", "TraceId", "ServiceName", "Name", "Kind", "DurationNano", "TagsKeys", "TagsValues", "References", "Events", "HasError", "StatusMessage", "StatusCodeString", "SpanKind"}, @@ -1318,9 +1320,29 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea end = time.Now() zap.L().Debug("getTraceSQLQuery unmarshal took: ", zap.Duration("duration", end.Sub(start))) - for i, item := range searchSpanResponses { - spanEvents := item.GetValues() - searchSpansResult[0].Events[i] = spanEvents + err = r.featureFlags.CheckFeature(model.SmartTraceDetail) + smartAlgoEnabled := err == nil + if len(searchScanResponses) > params.SpansRenderLimit && smartAlgoEnabled { + start = time.Now() + searchSpansResult, err = smartTraceAlgorithm(searchSpanResponses, params.SpanID, params.LevelUp, params.LevelDown, params.SpansRenderLimit) + if err != nil { + return nil, err + } + end = time.Now() + zap.L().Debug("smartTraceAlgo took: ", zap.Duration("duration", end.Sub(start))) + userEmail, err := auth.GetEmailFromJwt(ctx) + if err == nil { + data := map[string]interface{}{ + "traceSize": len(searchScanResponses), + "spansRenderLimit": params.SpansRenderLimit, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LARGE_TRACE_OPENED, data, userEmail, true, false) + } + } else { + for i, item := range searchSpanResponses { + spanEvents := item.GetValues() + searchSpansResult[0].Events[i] = spanEvents + } } searchSpansResult[0].StartTimestampMillis = startTime - (durationNano / 1000000) @@ -1334,7 +1356,7 @@ func (r *ClickHouseReader) SearchTraces(ctx context.Context, params *model.Searc levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) { if r.useTraceNewSchema { - return r.SearchTracesV2(ctx, params) + return r.SearchTracesV2(ctx, params, smartTraceAlgorithm) } var countSpans uint64 From 32fa5a403c9b35a64cec29920a2ece6c754abfcc Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Tue, 10 Dec 2024 18:36:02 +0530 Subject: [PATCH 07/10] fix: update default alert ch queries for traces and logs (#6552) * fix: update default alert ch queries for traces and logs * fix: use service.name instead of peer.service * fix: add link to docs * fix: add alias for service name --- frontend/src/container/CreateAlertRule/defaults.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/container/CreateAlertRule/defaults.ts b/frontend/src/container/CreateAlertRule/defaults.ts index bac7c90865..e8b56e4e85 100644 --- a/frontend/src/container/CreateAlertRule/defaults.ts +++ b/frontend/src/container/CreateAlertRule/defaults.ts @@ -117,7 +117,7 @@ export const logAlertDefaults: AlertDef = { chQueries: { A: { name: 'A', - query: `select \ntoStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 MINUTE) AS interval, \ntoFloat64(count()) as value \nFROM signoz_logs.distributed_logs_v2 \nWHERE timestamp BETWEEN {{.start_timestamp_nano}} AND {{.end_timestamp_nano}} \nGROUP BY interval;\n\n-- available variables:\n-- \t{{.start_timestamp_nano}}\n-- \t{{.end_timestamp_nano}}\n\n-- required columns (or alias):\n-- \tvalue\n-- \tinterval`, + query: `select \ntoStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 MINUTE) AS interval, \ntoFloat64(count()) as value \nFROM signoz_logs.distributed_logs_v2 \nWHERE timestamp BETWEEN {{.start_timestamp_nano}} AND {{.end_timestamp_nano}} \nAND ts_bucket_start BETWEEN {{.start_timestamp}} - 1800 AND {{.end_timestamp}} \nGROUP BY interval;\n\n-- Please check docs here https://signoz.io/docs/userguide/logs_clickhouse_queries/\n\n-- available variables:\n-- \t{{.start_timestamp_nano}}\n-- \t{{.end_timestamp_nano}}\n-- \t{{.start_timestamp}}\n-- \t{{.end_timestamp}}\n\n-- required columns (or alias):\n-- \tvalue\n-- \tinterval`, legend: '', disabled: false, }, @@ -149,7 +149,7 @@ export const traceAlertDefaults: AlertDef = { chQueries: { A: { name: 'A', - query: `SELECT \n\ttoStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, \n\tstringTagMap['peer.service'] AS op_name, \n\ttoFloat64(avg(durationNano)) AS value \nFROM signoz_traces.distributed_signoz_index_v2 \nWHERE stringTagMap['peer.service']!='' \nAND timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} \nGROUP BY (op_name, interval);\n\n-- available variables:\n-- \t{{.start_datetime}}\n-- \t{{.end_datetime}}\n\n-- required column alias:\n-- \tvalue\n-- \tinterval`, + query: `SELECT \n\ttoStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, \n\tresource_string_service$$name AS \`service.name\`, \n\ttoFloat64(avg(duration_nano)) AS value \nFROM signoz_traces.distributed_signoz_index_v3 \nWHERE resource_string_service$$name !='' \nAND timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} \nAND ts_bucket_start BETWEEN {{.start_timestamp}} - 1800 AND {{.end_timestamp}} \nGROUP BY (\`service.name\`, interval);\n\n-- Please check docs here https://signoz.io/docs/userguide/writing-clickhouse-traces-query/\n\n-- available variables:\n-- \t{{.start_datetime}}\n-- \t{{.end_datetime}}\n-- \t{{.start_timestamp}}\n-- \t{{.end_timestamp}}\n\n-- required column alias:\n-- \tvalue\n-- \tinterval`, legend: '', disabled: false, }, From b25df663811f99b40639cf5dd0244498fbf137b7 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Tue, 10 Dec 2024 18:49:21 +0530 Subject: [PATCH 08/10] fix: migration for ingestion dashboard (#6610) * fix: migration for ingestion dashboard * fix: remove redundant uuid * Update pkg/query-service/app/dashboards/model.go Co-authored-by: Srikanth Chekuri --------- Co-authored-by: Srikanth Chekuri --- pkg/query-service/app/dashboards/model.go | 4 + .../migrate/ingestion_dashboard.go | 2737 +++++++++++++++++ pkg/query-service/migrate/migate.go | 12 + 3 files changed, 2753 insertions(+) create mode 100644 pkg/query-service/migrate/ingestion_dashboard.go diff --git a/pkg/query-service/app/dashboards/model.go b/pkg/query-service/app/dashboards/model.go index 99aa53ef9c..fce24aedb8 100644 --- a/pkg/query-service/app/dashboards/model.go +++ b/pkg/query-service/app/dashboards/model.go @@ -289,6 +289,10 @@ func GetDashboard(ctx context.Context, uuid string) (*Dashboard, *model.ApiError return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)} } + if dashboard.Data["title"] == "Ingestion" && dashboard.Data["description"] != nil { + dashboard.Data["description"] = "This dashboard is deprecated. Please use the new Ingestion V2 dashboard. " + dashboard.Data["description"].(string) + } + return &dashboard, nil } diff --git a/pkg/query-service/migrate/ingestion_dashboard.go b/pkg/query-service/migrate/ingestion_dashboard.go new file mode 100644 index 0000000000..e9b33f3e1c --- /dev/null +++ b/pkg/query-service/migrate/ingestion_dashboard.go @@ -0,0 +1,2737 @@ +package migrate + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/app/dashboards" +) + +func migrateIngestionDashboard(conn *sqlx.DB) error { + exists, err := checkIfIngestionDashboardExists(conn) + if err != nil { + return fmt.Errorf("error checking if ingestion dashboard exists: %w", err) + } + if !exists { + return nil + } + return createIngestionDashboardV2(conn) +} + +func checkIfIngestionDashboardExists(conn *sqlx.DB) (bool, error) { + query := `SELECT COUNT(*) FROM dashboards WHERE data LIKE '%"title":"Ingestion"%'` + + var count int + err := conn.Get(&count, query) + if err != nil { + return false, fmt.Errorf("error checking if ingestion dashboard exists: %w", err) + } + + return count > 0, nil +} + +func createIngestionDashboardV2(conn *sqlx.DB) error { + mapData := make(map[string]interface{}) + err := json.Unmarshal([]byte(newIngestionDashboard), &mapData) + if err != nil { + return fmt.Errorf("error unmarshalling ingestion dashboard: %w", err) + } + dash := &dashboards.Dashboard{ + Data: mapData, + } + userEmail := "admin@signoz.cloud" + dash.CreatedAt = time.Now() + dash.CreateBy = &userEmail + dash.UpdatedAt = time.Now() + dash.UpdateBy = &userEmail + dash.UpdateSlug() + dash.Uuid = uuid.New().String() + + dataMarshaled, err := json.Marshal(dash.Data) + if err != nil { + return fmt.Errorf("error marshalling ingestion dashboard: %w", err) + } + + query := "INSERT INTO dashboards (uuid, created_at, created_by, updated_at, updated_by, data) VALUES ($1, $2, $3, $4, $5, $6)" + _, err = conn.Exec(query, dash.Uuid, dash.CreatedAt, dash.CreateBy, dash.UpdatedAt, dash.UpdateBy, dataMarshaled) + if err != nil { + return fmt.Errorf("error inserting ingestion dashboard: %w", err) + } + return nil +} + +var newIngestionDashboard = ` +{ + "description": "This dashboard shows the metrics, traces and logs ingested. Please follow the following guides to control your data sent to SigNoz. Only last 15 minutes of data is shown as this is mainly intended to help you understand the recent volume. \n\nMetrics: https://signoz.io/docs/userguide/drop-metrics/\n\n\nLogs: https://signoz.io/docs/logs-management/guides/drop-logs/\n\n\nTraces: https://signoz.io/docs/traces-management/guides/drop-spans/", + "image": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTgiIHZpZXdCb3g9IjAgMCAxOCAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0Ljk0OTQgMTMuOTQyQzE2LjIzMTggMTIuNDI1OCAxNy4zMjY4IDkuNzAyMiAxNi4xOTU2IDYuNTc0ODdDMTUuNjQ0MyA1LjA1MjQ1IDE1LjAyMTkgNC4yMDI0OSAxNC4yOTY5IDMuNjYyNTJDMTMuODU1NyAzLjMzMzc5IDEyLjA5MzMgMi41MDYzMyA5Ljc1OTY1IDIuODY3NTZDOC4wNTM0OSAzLjEzMjU1IDUuNzc0ODcgNC4yMDg3NCA0LjI5MzY5IDUuOTU5OUMyLjg1NzUyIDcuNjYxMDYgMS43NDg4MyA5LjAwNDc0IDEuNjk3NTggMTAuMzA5N0MxLjYzMTMzIDExLjk4ODMgMi44OTYyNyAxMy40MzA4IDMuMDUwMDEgMTMuNjY0NUMzLjMyMzc0IDE0LjA3OTUgNS4xOTExNSAxNi40NTE4IDguNjk5NzEgMTYuNTczMUMxMS43OTcgMTYuNjc5MyAxMy44MTQ0IDE1LjI4NDQgMTQuOTQ5NCAxMy45NDJaIiBmaWxsPSIjNDAzRDNFIi8+CjxwYXRoIGQ9Ik00LjU1MzYzIDIuNzM3NDdDMi45Mzc0NiAzLjg5MTE2IDEuMTIxMzEgNi4yNTEwMyAxLjQ0NzU0IDkuNTYwODZDMS42MDYyOCAxMS4xNzIgMi4wMDI1MSAxMi4xNDk1IDIuNTcxMjMgMTIuODUwN0MyLjkxNzQ2IDEzLjI3ODIgNC40MTk4OCAxNC41NDkzIDYuNzczNTEgMTQuNzM2OEM5LjE0NTg4IDE0LjkyNTYgMTAuOTQ5NSAxNC4zOTQ0IDEyLjgzMzIgMTMuMDg0NEMxNi42NjE3IDEwLjQyMDggMTYuMDk4IDYuMzkzNTMgMTUuOTM0MyA1LjkyNDhDMTUuNzcwNSA1LjQ1NjA3IDE0LjU0NDQgMi42OTYyMiAxMS4xNzMzIDEuNzE1MDJDOC4xOTg0NCAwLjg1MDA2OCA1Ljk4MzU1IDEuNzE1MDIgNC41NTM2MyAyLjczNzQ3WiIgZmlsbD0iIzVFNjM2NyIvPgo8cGF0aCBkPSJNNy4zOTM1MyAyLjk2MTA5QzUuNjE3MzcgMi44OTczNCAzLjkxOTk2IDQuMjg4NTIgMy43NTYyMiA2LjAwNTkzQzMuNTkyNDggNy43MjIwOSA0LjY1NDkyIDkuMDI5NTIgNi4zMDk4MyA5LjI5NTc2QzcuOTY0NzUgOS41NjA3NCA5Ljg3ODM5IDguNTU1OCAxMC4yNjM0IDYuNDUwOTFDMTAuNjYwOSA0LjI4MjI3IDkuMDg5NjkgMy4wMjIzNCA3LjM5MzUzIDIuOTYxMDlaIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNNy45NDIxNyA1LjkwMTE1QzcuOTQyMTcgNS45MDExNSA4LjM2OTY1IDUuODEyNCA4LjQ1NDY1IDUuMTgyNDRDOC41MzgzOSA0LjU2MjQ3IDguMjMwOTEgNC4wMzM3NSA3LjUxMzQ1IDMuODQzNzZDNi43MzM0OSAzLjYzNzUyIDYuMjA0NzcgNC4wNjYyNSA2LjA2NzI3IDQuNTE3NDdDNS44NzYwMyA1LjE0NDk0IDYuMTU4NTIgNS40NDM2NyA2LjE1ODUyIDUuNDQzNjdDNi4xNTg1MiA1LjQ0MzY3IDUuMzkzNTYgNS42Mjc0MSA1LjMzMjMxIDYuNTI5ODdDNS4yNzQ4MSA3LjM4MTA3IDUuODU2MDMgNy44Mzg1NSA2LjQzOTc1IDcuOTc4NTRDNy4xNjA5NiA4LjE1MjI4IDcuOTc4NDIgNy45NTQ3OSA4LjE3ODQxIDcuMDM0ODRDOC4zNDQ2NSA2LjI3NzM4IDcuOTQyMTcgNS45MDExNSA3Ljk0MjE3IDUuOTAxMTVaIiBmaWxsPSIjMzAzMDMwIi8+CjxwYXRoIGQ9Ik02LjczOTgzIDQuNzUzNjJDNi42NzEwOSA1LjAxMjM1IDYuODA4NTggNS4yNjIzNCA3LjA3ODU3IDUuMzMxMDlDNy4zNjk4IDUuNDA0ODMgNy42MzQ3OSA1LjMwODU5IDcuNzA2MDMgNS4wMTExQzcuNzY4NTMgNC43NDczNyA3LjY0MzU0IDQuNTE0ODggNy4zMzYwNSA0LjQzOTg4QzcuMDgzNTcgNC4zNzczOSA2LjgxNDgzIDQuNDcxMTMgNi43Mzk4MyA0Ljc1MzYyWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTYuOTU5NzggNi4wMzk3NEM2LjYzMjMgNS45Mzg0OSA2LjE5OTgyIDYuMDY0NzMgNi4xMzEwNyA2LjUwNDcxQzYuMDYyMzMgNi45NDQ2OSA2LjMyNjA2IDcuMTY5NjggNi42NzEwNCA3LjIzMjE3QzcuMDE2MDMgNy4yOTQ2NyA3LjM0MjI2IDcuMTEzNDMgNy40MDYwMSA2Ljc2MDk1QzcuNDY4NSA2LjQwOTcyIDcuMjg2MDEgNi4xMzk3MyA2Ljk1OTc4IDYuMDM5NzRaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K", + "layout": [ + { + "h": 1, + "i": "5e1f9c33-8955-4dd6-8dc3-0b6601592c67", + "maxH": 1, + "minH": 1, + "minW": 12, + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 0 + }, + { + "h": 4, + "i": "983bdd34-27b7-4aba-b60d-6baf8bfab535", + "moved": false, + "static": false, + "w": 5, + "x": 0, + "y": 1 + }, + { + "h": 9, + "i": "18095ed2-8bb9-4bc9-bf09-e1f304c08131", + "moved": false, + "static": false, + "w": 7, + "x": 5, + "y": 1 + }, + { + "h": 5, + "i": "9542f6a0-86c7-46f8-8981-63a892b12b4b", + "moved": false, + "static": false, + "w": 5, + "x": 0, + "y": 5 + }, + { + "h": 6, + "i": "b72f44eb-bb37-4777-9b7a-e9bd72ff2f48", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 10 + }, + { + "h": 6, + "i": "a5c66415-030a-4edd-9d08-9df31f8e2ce9", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 10 + }, + { + "h": 1, + "i": "__dropping-elem__", + "isDraggable": true, + "moved": false, + "static": false, + "w": 1, + "x": 8, + "y": 16 + }, + { + "h": 1, + "i": "a3ec6c7d-5136-4aae-85ac-dc0c991100d8", + "maxH": 1, + "minH": 1, + "minW": 12, + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 17 + }, + { + "h": 6, + "i": "56d419bc-9bca-4432-b69e-1f06e8d209f6", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 18 + }, + { + "h": 6, + "i": "e278d906-51d5-428b-bc4a-bc0d9069085a", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 18 + }, + { + "h": 7, + "i": "eec15b67-3b58-4f88-9038-a8138424c3b0", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 24 + }, + { + "h": 7, + "i": "1d06208b-0432-4228-9d54-b52d5d4e3712", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 24 + }, + { + "h": 1, + "i": "2f688301-e8e6-4c48-b0d0-2e2a784e5ed4", + "maxH": 1, + "minH": 1, + "minW": 12, + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 31 + }, + { + "h": 7, + "i": "87c88869-8ae9-4dc4-9542-ff4b46bc9c70", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 32 + }, + { + "h": 7, + "i": "8dff8dad-62e0-4fe1-8c7b-551605f11ef8", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 32 + }, + { + "h": 7, + "i": "6d3824c7-b00f-496b-a308-eb59016eb3ee", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 32 + }, + { + "h": 7, + "i": "bca0969b-3150-4b00-b1e4-ed2a7a9e3594", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 39 + }, + { + "h": 7, + "i": "ce0865eb-973d-40ec-bedb-2f7366e043f8", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 39 + }, + { + "h": 11, + "i": "04b92695-b37f-49c3-806e-d1f3305fb63c", + "moved": false, + "static": false, + "w": 12, + "x": 0, + "y": 46 + }, + { + "h": 9, + "i": "644b2494-167e-43b4-8e0c-4def97cdcce4", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 57 + }, + { + "h": 9, + "i": "3dbd578d-8648-4b76-9e9b-c5e1146753f7", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 57 + } + ], + "panelMap": { + "2f688301-e8e6-4c48-b0d0-2e2a784e5ed4": { + "collapsed": false, + "widgets": [ + { + "h": 7, + "i": "87c88869-8ae9-4dc4-9542-ff4b46bc9c70", + "moved": false, + "static": false, + "w": 3, + "x": 0, + "y": 394 + }, + { + "h": 7, + "i": "8dff8dad-62e0-4fe1-8c7b-551605f11ef8", + "moved": false, + "static": false, + "w": 3, + "x": 3, + "y": 394 + }, + { + "h": 7, + "i": "6d3824c7-b00f-496b-a308-eb59016eb3ee", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 394 + }, + { + "h": 7, + "i": "bca0969b-3150-4b00-b1e4-ed2a7a9e3594", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 401 + }, + { + "h": 7, + "i": "ce0865eb-973d-40ec-bedb-2f7366e043f8", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 401 + }, + { + "h": 9, + "i": "04b92695-b37f-49c3-806e-d1f3305fb63c", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 408 + }, + { + "h": 9, + "i": "3dbd578d-8648-4b76-9e9b-c5e1146753f7", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 408 + }, + { + "h": 9, + "i": "644b2494-167e-43b4-8e0c-4def97cdcce4", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 417 + } + ] + }, + "5e1f9c33-8955-4dd6-8dc3-0b6601592c67": { + "collapsed": false, + "widgets": [ + { + "h": 4, + "i": "983bdd34-27b7-4aba-b60d-6baf8bfab535", + "moved": false, + "static": false, + "w": 5, + "x": 0, + "y": 1 + }, + { + "h": 9, + "i": "18095ed2-8bb9-4bc9-bf09-e1f304c08131", + "moved": false, + "static": false, + "w": 7, + "x": 5, + "y": 1 + }, + { + "h": 5, + "i": "9542f6a0-86c7-46f8-8981-63a892b12b4b", + "moved": false, + "static": false, + "w": 5, + "x": 0, + "y": 5 + }, + { + "h": 6, + "i": "b72f44eb-bb37-4777-9b7a-e9bd72ff2f48", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 10 + }, + { + "h": 6, + "i": "a5c66415-030a-4edd-9d08-9df31f8e2ce9", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 10 + }, + { + "h": 1, + "i": "__dropping-elem__", + "isDraggable": true, + "moved": false, + "static": false, + "w": 1, + "x": 8, + "y": 16 + } + ] + }, + "a3ec6c7d-5136-4aae-85ac-dc0c991100d8": { + "collapsed": false, + "widgets": [ + { + "h": 6, + "i": "56d419bc-9bca-4432-b69e-1f06e8d209f6", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 96 + }, + { + "h": 6, + "i": "e278d906-51d5-428b-bc4a-bc0d9069085a", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 96 + }, + { + "h": 7, + "i": "eec15b67-3b58-4f88-9038-a8138424c3b0", + "moved": false, + "static": false, + "w": 6, + "x": 0, + "y": 102 + }, + { + "h": 7, + "i": "1d06208b-0432-4228-9d54-b52d5d4e3712", + "moved": false, + "static": false, + "w": 6, + "x": 6, + "y": 102 + } + ] + } + }, + "tags": [ + "usage", + "ingestion" + ], + "title": "IngestionV2", + "uploadedGrafana": false, + "version": "v4", + "widgets": [ + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "87c88869-8ae9-4dc4-9542-ff4b46bc9c70", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "SELECT\n count()\nFROM signoz_metrics.distributed_samples_v4\nWHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000" + } + ], + "id": "724ae115-2ce0-4165-8556-9e8a26384e7f", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Total samples", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "This table contain the metric name and it's total samples count. Note: name of the metric is normalized.", + "fillSpans": false, + "id": "6d3824c7-b00f-496b-a308-eb59016eb3ee", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "SELECT\n count() as Count,\n metric_name as \"Normalized Metric Name\"\nFROM signoz_metrics.distributed_samples_v4\nWHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000\nGROUP BY metric_name\nORDER BY Count DESC\nLIMIT 100" + } + ], + "id": "fa0f605c-bf1d-435b-9b0e-4a660113fd97", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Volume breakdown by metric name", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "ce0865eb-973d-40ec-bedb-2f7366e043f8", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "SELECT\n metric_name as \"Normalized Metric Name\",\n deployment_environment as \"Deployment Environment\",\n count() AS \"Count\"\nFROM signoz_metrics.distributed_samples_v4\nINNER JOIN\n(\n SELECT DISTINCT fingerprint, JSONExtractString(labels, 'deployment_environment') AS deployment_environment\n FROM signoz_metrics.time_series_v4_1day\n WHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND (unix_milli >= intDiv({{.start_timestamp_ms}}, 86400000) * 86400000) AND (unix_milli < {{.end_timestamp_ms}})\n) AS filtered_time_series USING (fingerprint)\nWHERE unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000\nGROUP BY\n metric_name,\n deployment_environment\nORDER BY\n count() DESC\nLIMIT 100" + } + ], + "id": "883d6774-752d-4b7d-b3df-5308832d7741", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Volume breakdown by deployment environment and metric name", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "8dff8dad-62e0-4fe1-8c7b-551605f11ef8", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "SELECT\n deployment_environment as \"Deployment Environment\",\n count() AS \"Count\"\nFROM signoz_metrics.distributed_samples_v4\nINNER JOIN\n(\n SELECT DISTINCT fingerprint, JSONExtractString(labels, 'deployment_environment') AS deployment_environment\n FROM signoz_metrics.time_series_v4_1day\n WHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND (unix_milli >= intDiv({{.start_timestamp_ms}}, 86400000) * 86400000) AND (unix_milli < {{.end_timestamp_ms}})\n) AS filtered_time_series USING (fingerprint)\nWHERE unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000\nGROUP BY\n deployment_environment\nORDER BY\n count() DESC\nLIMIT 100" + } + ], + "id": "e5c98da0-b22a-47f0-8397-be914800376b", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Volume breakdown by environment", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "bca0969b-3150-4b00-b1e4-ed2a7a9e3594", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "SELECT\n metric_name as \"Normalized Metric Name\",\n service_name as \"Service Name\",\n count() AS \"Count\"\nFROM signoz_metrics.distributed_samples_v4\nINNER JOIN\n(\n SELECT DISTINCT fingerprint,\n JSONExtractString(labels, 'service_name') AS service_name\n FROM signoz_metrics.time_series_v4_1day\n WHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND (unix_milli >= intDiv({{.start_timestamp_ms}}, 86400000) * 86400000) AND (unix_milli < {{.end_timestamp_ms}})\n) AS filtered_time_series USING (fingerprint)\nWHERE unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000\nGROUP BY\n metric_name,\n service_name\nORDER BY\n count() DESC\nLIMIT 100" + } + ], + "id": "943c1751-4a81-407e-b7f0-70bc426294a8", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Volume by service", + "yAxisUnit": "none" + }, + { + "description": "", + "id": "2f688301-e8e6-4c48-b0d0-2e2a784e5ed4", + "panelTypes": "row", + "title": "Metrics ingestion; Tables show only top 100 by volume" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "3dbd578d-8648-4b76-9e9b-c5e1146753f7", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "SELECT\n metric_name as \"Normalized Metric Name\",\n host_name as \"Host Name\",\n if(count() > 0, 'Yes', 'No') AS \"Has Data\"\nFROM signoz_metrics.distributed_samples_v4\nINNER JOIN\n(\n SELECT DISTINCT fingerprint,\n JSONExtractString(labels, 'host_name') AS host_name\n FROM signoz_metrics.time_series_v4_1day\n WHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND (unix_milli >= intDiv({{.start_timestamp_ms}}, 86400000) * 86400000) AND (unix_milli < {{.end_timestamp_ms}})\n) AS filtered_time_series USING (fingerprint)\nWHERE unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000\nGROUP BY\n metric_name,\n host_name\nORDER BY\n host_name DESC\n" + } + ], + "id": "5d8f9be9-eeff-4f22-8306-366f32a5c061", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Is my host sending metric?", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "644b2494-167e-43b4-8e0c-4def97cdcce4", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "SELECT\n metric_name as \"Normalized Metric Name\",\n k8s_cluster_name as \"K8s Cluster Name\",\n k8s_node_name as \"K8s Node Name\",\n k8s_namespace_name as \"K8s Namespace Name\",\n if(count() > 0, 'Yes', 'No') AS \"Has Data\"\nFROM signoz_metrics.distributed_samples_v4\nINNER JOIN\n(\n SELECT DISTINCT fingerprint,\n JSONExtractString(labels, 'k8s_cluster_name') AS k8s_cluster_name,\n JSONExtractString(labels, 'k8s_node_name') AS k8s_node_name,\n JSONExtractString(labels, 'k8s_namespace_name') AS k8s_namespace_name\n FROM signoz_metrics.time_series_v4_1day\n WHERE (metric_name NOT LIKE 'signoz_%') AND (metric_name NOT LIKE 'chi_%') AND (unix_milli >= intDiv({{.start_timestamp_ms}}, 86400000) * 86400000) AND (unix_milli < {{.end_timestamp_ms}})\n) AS filtered_time_series USING (fingerprint)\nWHERE unix_milli >= toUnixTimestamp(now() - INTERVAL 15 MINUTE) * 1000\nGROUP BY\n metric_name,\n k8s_cluster_name,\n k8s_node_name,\n k8s_namespace_name\nORDER BY\n metric_name DESC,\n k8s_cluster_name DESC,\n k8s_node_name DESC,\n k8s_namespace_name DESC" + } + ], + "id": "7df1077e-439a-4d1f-97a8-9cd514bef8c2", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Is my cluser,node,namespace sending metric?", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "04b92695-b37f-49c3-806e-d1f3305fb63c", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Seconds", + "name": "A", + "query": "SELECT\n env as \"Deployment Environment\",\n metric_name as \"Noramalized Metric Name\",\n service_name \"Service Name\",\n if(ceiling(divide(min(diff), 1000)) > 86400000, -1, ceiling(divide(min(diff), 1000))) AS max_diff_in_secs -- not enough points to calculate the diff\nFROM\n(\n SELECT\n env,\n metric_name,\n service_name,\n unix_milli - lagInFrame(unix_milli, 1, 0) OVER rate_window AS diff\n FROM signoz_metrics.distributed_samples_v4\n INNER JOIN\n (\n SELECT DISTINCT\n env,\n metric_name,\n JSONExtractString(labels, 'service_name') AS service_name,\n anyLast(fingerprint) AS fingerprint\n FROM signoz_metrics.time_series_v4_1day\n WHERE metric_name NOT LIKE 'signoz_%' AND (unix_milli >= intDiv({{.start_timestamp_ms}}, 86400000) * 86400000) AND (unix_milli < {{.end_timestamp_ms}})\n GROUP BY env, metric_name, service_name\n ) AS filtered_time_series USING (fingerprint)\n WHERE unix_milli >= (toUnixTimestamp(now() - toIntervalMinute(30)) * 1000)\n WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, unix_milli)\n)\nWHERE diff > 0\nGROUP BY env, metric_name, service_name\nORDER BY env, metric_name, service_name\n" + } + ], + "id": "dcb8402d-0df4-4bc2-b50d-2777718d5326", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "The observed report interval for (metric, service) combination", + "yAxisUnit": "none" + }, + { + "description": "", + "id": "a3ec6c7d-5136-4aae-85ac-dc0c991100d8", + "panelTypes": "row", + "title": "Traces" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "56d419bc-9bca-4432-b69e-1f06e8d209f6", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "2145bdd2-30e0-4e79-93a8-4ce85ffab2bc", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "LAST_15_MIN", + "title": "Total spans", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "e278d906-51d5-428b-bc4a-bc0d9069085a", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "deployment.environment--string--resource--false", + "isColumn": false, + "isJSON": false, + "key": "deployment.environment", + "type": "resource" + }, + { + "dataType": "string", + "id": "service.name--string--resource--false", + "isColumn": false, + "isJSON": false, + "key": "service.name", + "type": "resource" + } + ], + "having": [], + "legend": "", + "limit": 100, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "" + } + ], + "id": "11061d67-bacf-44a5-b128-f1a7873f8c77", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "LAST_15_MIN", + "title": "Spans count by env, service", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "eec15b67-3b58-4f88-9038-a8138424c3b0", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "traces", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "service.name--string--resource--false", + "isColumn": false, + "isJSON": false, + "key": "service.name", + "type": "resource" + }, + { + "dataType": "string", + "id": "name--string----true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "" + } + ], + "having": [], + "legend": "", + "limit": 500, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "1f54535a-2a8a-4777-b44d-b3eedc5a2551", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "LAST_15_MIN", + "title": "Count by span name", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "1d06208b-0432-4228-9d54-b52d5d4e3712", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "SELECT tagKey as \"Attribute Name\", if(tagType='tag', 'Span Attribute', 'Span Resource Attribute') as \"Type\"\nFROM signoz_traces.distributed_span_attributes_keys WHERE isColumn = false\nGROUP BY tagKey, tagType" + } + ], + "id": "15f1b33e-bc69-4ab6-b5ae-8570423a3038", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "List of attributes", + "yAxisUnit": "none" + }, + { + "description": "", + "id": "5e1f9c33-8955-4dd6-8dc3-0b6601592c67", + "panelTypes": "row", + "title": "Logs" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "983bdd34-27b7-4aba-b60d-6baf8bfab535", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "value", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "logs", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "sum", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "4f9389b8-5231-4893-8ea4-4c3c928628c7", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "LAST_15_MIN", + "title": "Total logs", + "yAxisUnit": "short" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "b72f44eb-bb37-4777-9b7a-e9bd72ff2f48", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "logs", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "", + "id": "deployment.environment------false", + "isColumn": false, + "key": "deployment.environment", + "type": "resource" + }, + { + "dataType": "", + "id": "sevice.name------false", + "isColumn": false, + "key": "sevice.name", + "type": "resource" + } + ], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "" + } + ], + "id": "a21cfccc-d83f-4d3f-811d-5e05340168ec", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "Logs by deployment.environment, service.name", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "9542f6a0-86c7-46f8-8981-63a892b12b4b", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "logs", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "severity_text--string----true", + "isColumn": true, + "isJSON": false, + "key": "severity_text", + "type": "" + } + ], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "Count", + "name": "A", + "query": "" + } + ], + "id": "2eb23fda-d995-47eb-9582-77d66a627cef", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "LAST_15_MIN", + "title": "Logs count by level", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": { + "A": "short" + }, + "description": "", + "fillSpans": false, + "id": "18095ed2-8bb9-4bc9-bf09-e1f304c08131", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "logs", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [ + { + "dataType": "string", + "id": "log.file.path--string--tag--false", + "isColumn": false, + "isJSON": false, + "key": "log.file.path", + "type": "tag" + } + ], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "id": "212f628b-3449-4e01-824d-f412a39d99ae", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "builder" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "LAST_15_MIN", + "title": "Logs count by log file path", + "yAxisUnit": "none" + }, + { + "bucketCount": 30, + "bucketWidth": 0, + "columnUnits": {}, + "description": "", + "fillSpans": false, + "id": "a5c66415-030a-4edd-9d08-9df31f8e2ce9", + "isStacked": false, + "mergeAllActiveQueries": false, + "nullZeroValues": "zero", + "opacity": "1", + "panelTypes": "table", + "query": { + "builder": { + "queryData": [ + { + "aggregateAttribute": { + "dataType": "", + "id": "------false", + "isColumn": false, + "isJSON": false, + "key": "", + "type": "" + }, + "aggregateOperator": "count", + "dataSource": "metrics", + "disabled": false, + "expression": "A", + "filters": { + "items": [], + "op": "AND" + }, + "functions": [], + "groupBy": [], + "having": [], + "legend": "", + "limit": null, + "orderBy": [], + "queryName": "A", + "reduceTo": "avg", + "spaceAggregation": "sum", + "stepInterval": 60, + "timeAggregation": "rate" + } + ], + "queryFormulas": [] + }, + "clickhouse_sql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "SELECT DISTINCT\n name,\n 'Log Record Attribute' AS Type\nFROM signoz_logs.distributed_logs_attribute_keys\nUNION ALL\nSELECT DISTINCT\n name,\n 'Log Resource Attribute' AS Type\nFROM signoz_logs.distributed_logs_resource_keys" + } + ], + "id": "09ad654a-23f9-4bc9-b15c-e0044d2e48c8", + "promql": [ + { + "disabled": false, + "legend": "", + "name": "A", + "query": "" + } + ], + "queryType": "clickhouse_sql" + }, + "selectedLogFields": [ + { + "dataType": "string", + "name": "body", + "type": "" + }, + { + "dataType": "string", + "name": "timestamp", + "type": "" + } + ], + "selectedTracesFields": [ + { + "dataType": "string", + "id": "serviceName--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "serviceName", + "type": "tag" + }, + { + "dataType": "string", + "id": "name--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "name", + "type": "tag" + }, + { + "dataType": "float64", + "id": "durationNano--float64--tag--true", + "isColumn": true, + "isJSON": false, + "key": "durationNano", + "type": "tag" + }, + { + "dataType": "string", + "id": "httpMethod--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "httpMethod", + "type": "tag" + }, + { + "dataType": "string", + "id": "responseStatusCode--string--tag--true", + "isColumn": true, + "isJSON": false, + "key": "responseStatusCode", + "type": "tag" + } + ], + "softMax": 0, + "softMin": 0, + "stackedBarChart": false, + "thresholds": [], + "timePreferance": "GLOBAL_TIME", + "title": "List of attributes", + "yAxisUnit": "none" + } + ] +}` diff --git a/pkg/query-service/migrate/migate.go b/pkg/query-service/migrate/migate.go index 2db7243f58..d73a3e1e0a 100644 --- a/pkg/query-service/migrate/migate.go +++ b/pkg/query-service/migrate/migate.go @@ -7,6 +7,7 @@ import ( "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/jmoiron/sqlx" + "go.uber.org/zap" ) type DataMigration struct { @@ -53,6 +54,17 @@ func Migrate(dsn string) error { return err } + if m, err := getMigrationVersion(conn, "0.62_ingestion_dashboard"); err == nil && m == nil { + if err := migrateIngestionDashboard(conn); err != nil { + zap.L().Error("failed to migrate 0.62_ingestion_dashboard", zap.Error(err)) + } else { + _, err := conn.Exec("INSERT INTO data_migrations (version, succeeded) VALUES ('0.62_ingestion_dashboard', true)") + if err != nil { + return err + } + } + } + return nil } From 0fbfb6b22b0ec0e38d61d52c9468ed97d67417e7 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Wed, 11 Dec 2024 13:37:25 +0530 Subject: [PATCH 09/10] fix: fix count aggregate attribute column name (#6613) * fix: fix count aggregate attribute column name * fix: add test for column as well * fix: use new schema in threshold rule enrichment --- .../app/traces/v4/query_builder.go | 5 +- .../app/traces/v4/query_builder_test.go | 56 +++++++++++++++++++ pkg/query-service/rules/threshold_rule.go | 5 +- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 366f3f6ee4..d650e1cdaa 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -355,8 +355,9 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery) } } else { - column := getColumnName(mq.AggregateAttribute) - filterSubQuery = fmt.Sprintf("%s AND has(%s, '%s')", filterSubQuery, column, mq.AggregateAttribute.Key) + cType := getClickHouseTracesColumnType(mq.AggregateAttribute.Type) + cDataType := getClickHouseTracesColumnDataType(mq.AggregateAttribute.DataType) + filterSubQuery = fmt.Sprintf("%s AND mapContains(%s_%s, '%s')", filterSubQuery, cType, cDataType, mq.AggregateAttribute.Key) } } op := "toFloat64(count())" diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index d29f77a74b..9db2e815f9 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -451,6 +451,62 @@ func Test_buildTracesQuery(t *testing.T) { "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') " + "group by `http.method` order by `http.method` ASC", }, + { + name: "Test buildTracesQuery - count with attr", + args: args{ + panelType: v3.PanelTypeTable, + start: 1680066360726210000, + end: 1680066458000000000, + step: 1000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorCount, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + Value: 100, + Operator: v3.FilterOperatorEqual, + }, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{ + {ColumnName: "http.method", Order: "ASC"}}, + }, + }, + want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND mapContains(attributes_string, 'name') " + + "group by `http.method` order by `http.method` ASC", + }, + { + name: "Test buildTracesQuery - count with mat attr", + args: args{ + panelType: v3.PanelTypeTable, + start: 1680066360726210000, + end: 1680066458000000000, + step: 1000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorCount, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + Value: 100, + Operator: v3.FilterOperatorEqual, + }, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{ + {ColumnName: "http.method", Order: "ASC"}}, + }, + }, + want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND name != '' " + + "group by `http.method` order by `http.method` ASC", + }, { name: "Test buildTracesQuery", args: args{ diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index ad55083f8f..5b5f02a52c 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -73,8 +73,9 @@ func NewThresholdRule( } t := ThresholdRule{ - BaseRule: baseRule, - version: p.Version, + BaseRule: baseRule, + version: p.Version, + useTraceNewSchema: useTraceNewSchema, } querierOption := querier.QuerierOptions{ From a9440c010cbdce530a5b0c98c0912e20b3bf50c4 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Thu, 12 Dec 2024 15:28:09 +0530 Subject: [PATCH 10/10] chore(signoz): pin versions: SigNoz 0.62.0, SigNoz OtelCollector 0.111.15 Signed-off-by: Prashant Shahi --- .../docker-swarm/clickhouse-setup/docker-compose.yaml | 8 ++++---- .../docker/clickhouse-setup/docker-compose-core.yaml | 4 ++-- .../clickhouse-setup/docker-compose-minimal.yaml | 10 +++++----- .../clickhouse-setup/docker-compose.testing.yaml | 8 ++++---- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 88453360f9..a34b1b6e1b 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -146,7 +146,7 @@ services: condition: on-failure query-service: - image: signoz/query-service:0.61.0 + image: signoz/query-service:0.62.0 command: [ "-config=/root/config/prometheus.yml", @@ -186,7 +186,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:0.61.0 + image: signoz/frontend:0.62.0 deploy: restart_policy: condition: on-failure @@ -199,7 +199,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/signoz-otel-collector:0.111.14 + image: signoz/signoz-otel-collector:0.111.15 command: [ "--config=/etc/otel-collector-config.yaml", @@ -237,7 +237,7 @@ services: - query-service otel-collector-migrator: - image: signoz/signoz-schema-migrator:0.111.14 + image: signoz/signoz-schema-migrator:0.111.15 deploy: restart_policy: condition: on-failure diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml index 5bade6b2da..173d39e9fc 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml @@ -69,7 +69,7 @@ services: - --storage.path=/data otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15} container_name: otel-migrator command: - "sync" @@ -86,7 +86,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` otel-collector: container_name: signoz-otel-collector - image: signoz/signoz-otel-collector:0.111.14 + image: signoz/signoz-otel-collector:0.111.15 command: [ "--config=/etc/otel-collector-config.yaml", diff --git a/deploy/docker/clickhouse-setup/docker-compose-minimal.yaml b/deploy/docker/clickhouse-setup/docker-compose-minimal.yaml index f737b7d440..6a75ce1d5b 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-minimal.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-minimal.yaml @@ -162,7 +162,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: - image: signoz/query-service:${DOCKER_TAG:-0.61.0} + image: signoz/query-service:${DOCKER_TAG:-0.62.0} container_name: signoz-query-service command: [ @@ -201,7 +201,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:${DOCKER_TAG:-0.61.0} + image: signoz/frontend:${DOCKER_TAG:-0.62.0} container_name: signoz-frontend restart: on-failure depends_on: @@ -213,7 +213,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector-migrator-sync: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15} container_name: otel-migrator-sync command: - "sync" @@ -228,7 +228,7 @@ services: # condition: service_healthy otel-collector-migrator-async: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15} container_name: otel-migrator-async command: - "async" @@ -245,7 +245,7 @@ services: # condition: service_healthy otel-collector: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.15} container_name: signoz-otel-collector command: [ diff --git a/deploy/docker/clickhouse-setup/docker-compose.testing.yaml b/deploy/docker/clickhouse-setup/docker-compose.testing.yaml index d90773844e..5b5c640308 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.testing.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.testing.yaml @@ -167,7 +167,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: - image: signoz/query-service:${DOCKER_TAG:-0.61.0} + image: signoz/query-service:${DOCKER_TAG:-0.62.0} container_name: signoz-query-service command: [ @@ -208,7 +208,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:${DOCKER_TAG:-0.61.0} + image: signoz/frontend:${DOCKER_TAG:-0.62.0} container_name: signoz-frontend restart: on-failure depends_on: @@ -220,7 +220,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -234,7 +234,7 @@ services: otel-collector: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.15} container_name: signoz-otel-collector command: [ diff --git a/go.mod b/go.mod index 9ab2cd98ec..0ac2593064 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ClickHouse/clickhouse-go/v2 v2.25.0 github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd - github.com/SigNoz/signoz-otel-collector v0.111.14 + github.com/SigNoz/signoz-otel-collector v0.111.15 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974 github.com/antonmedv/expr v1.15.3 diff --git a/go.sum b/go.sum index b46384371a..dd514b47aa 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc= github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE= github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA= -github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0= -github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4= +github.com/SigNoz/signoz-otel-collector v0.111.15 h1:X2aV/FpMZ9kdN10byKrc2ODF39fAcOo6D68rqUFhxSk= +github.com/SigNoz/signoz-otel-collector v0.111.15/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4= github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc= github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo= github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=