diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index a1833c534b..4161a665ce 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -4196,7 +4196,7 @@ func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r * switch req.DataSource { case v3.DataSourceMetrics: - response, err = aH.reader.GetMetricAggregateAttributes(r.Context(), req, true) + response, err = aH.reader.GetMetricAggregateAttributes(r.Context(), req, false) case v3.DataSourceLogs: response, err = aH.reader.GetLogAggregateAttributes(r.Context(), req) case v3.DataSourceTraces: diff --git a/pkg/query-service/app/metrics/v3/query_builder.go b/pkg/query-service/app/metrics/v3/query_builder.go index 25f5990148..9013b0461a 100644 --- a/pkg/query-service/app/metrics/v3/query_builder.go +++ b/pkg/query-service/app/metrics/v3/query_builder.go @@ -217,6 +217,7 @@ func buildMetricQuery(start, end, step int64, mq *v3.BuilderQuery) (string, erro // groupingSets returns a string of comma separated tags for group by clause // `ts` is always added to the group by clause func groupingSets(tags ...string) string { + tags = utils.AddBackTickToFormatTags(tags...) withTs := append(tags, "ts") return strings.Join(withTs, ", ") } @@ -224,12 +225,14 @@ func groupingSets(tags ...string) string { // groupBy returns a string of comma separated tags for group by clause // `ts` is always added to the group by clause func groupBy(tags ...string) string { + tags = utils.AddBackTickToFormatTags(tags...) tags = append(tags, "ts") return strings.Join(tags, ",") } // groupSelect returns a string of comma separated tags for select clause func groupSelect(tags ...string) string { + tags = utils.AddBackTickToFormatTags(tags...) groupTags := strings.Join(tags, ",") if len(tags) != 0 { groupTags += ", " @@ -270,11 +273,13 @@ func orderBy(items []v3.OrderBy, tags []string) string { for _, item := range items { if item.ColumnName == tag { found = true + item.ColumnName = utils.AddBackTickToFormatTag(item.ColumnName) orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) break } } if !found { + tag = utils.AddBackTickToFormatTag(tag) orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag)) } } diff --git a/pkg/query-service/app/metrics/v3/query_builder_test.go b/pkg/query-service/app/metrics/v3/query_builder_test.go index 5b85036007..78d4901234 100644 --- a/pkg/query-service/app/metrics/v3/query_builder_test.go +++ b/pkg/query-service/app/metrics/v3/query_builder_test.go @@ -380,3 +380,163 @@ func TestBuildQueryAdjustedTimes(t *testing.T) { }) } } + +func TestBuildQueryWithDotInMetricAndAttributes(t *testing.T) { + cases := []struct { + name string + params *v3.QueryRangeParamsV3 + expected string + }{ + { + name: "TestBuildQueryWithDotInMetricAndAttributes with dot in metric and attributes", + params: &v3.QueryRangeParamsV3{ + Start: 1735036101000, + End: 1735637901000, + Step: 60, + Variables: map[string]interface{}{ + "SIGNOZ_START_TIME": 1735034992000, + "SIGNOZ_END_TIME": 1735036792000, + }, + FormatForWeb: false, + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeValue, + FillGaps: false, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorAvg, + AggregateAttribute: v3.AttributeKey{ + Key: "system.memory.usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, + }, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationSum, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "linux", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "os.type", + Order: v3.DirectionAsc, + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + }, + }, + }, + expected: "SELECT *, now() AS ts FROM (SELECT avgIf(value, toUnixTimestamp(ts) != 0) as value, anyIf(ts, toUnixTimestamp(ts) != 0) AS timestamp FROM (SELECT `os.type`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.memory.usage' AND temporality = '' AND unix_milli >= 1734998400000 AND unix_milli < 1735637880000 AND JSONExtractString(labels, 'os.type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name = 'system.memory.usage' AND unix_milli >= 1735036080000 AND unix_milli < 1735637880000 GROUP BY `os.type`, ts ORDER BY `os.type` asc, ts) )", + }, + { + name: "TestBuildQueryWithDotInMetricAndAttributes with dot in metric and attributes with rate_avg aggregation", + params: &v3.QueryRangeParamsV3{ + Start: 1735036101000, + End: 1735637901000, + Step: 60, + Variables: map[string]interface{}{ + "SIGNOZ_START_TIME": 1735034992000, + "SIGNOZ_END_TIME": 1735036792000, + }, + FormatForWeb: false, + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeValue, + FillGaps: false, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorRateAvg, + AggregateAttribute: v3.AttributeKey{ + Key: "system.memory.usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, + }, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationSum, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "linux", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "os.type", + Order: v3.DirectionAsc, + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + }, + }, + }, + expected: "SELECT *, now() AS ts FROM (SELECT avgIf(value, toUnixTimestamp(ts) != 0) as value, anyIf(ts, toUnixTimestamp(ts) != 0) AS timestamp FROM (SELECT `os.type`, ts, If((value - lagInFrame(value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (value - lagInFrame(value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as value FROM(SELECT `os.type`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.memory.usage' AND temporality = '' AND unix_milli >= 1734998400000 AND unix_milli < 1735637880000 AND JSONExtractString(labels, 'os.type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name = 'system.memory.usage' AND unix_milli >= 1735036020000 AND unix_milli < 1735637880000 GROUP BY `os.type`, ts ORDER BY `os.type` asc, ts) WINDOW rate_window as (PARTITION BY `os.type` ORDER BY `os.type`, ts) ) )", + }, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + q := testCase.params + query, err := PrepareMetricQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"], Options{PreferRPM: false}) + require.NoError(t, err) + + require.Contains(t, query, testCase.expected) + }) + } +} diff --git a/pkg/query-service/app/metrics/v4/cumulative/table_test.go b/pkg/query-service/app/metrics/v4/cumulative/table_test.go index 70613c9b44..c1a9042b8a 100644 --- a/pkg/query-service/app/metrics/v4/cumulative/table_test.go +++ b/pkg/query-service/app/metrics/v4/cumulative/table_test.go @@ -95,6 +95,61 @@ func TestPrepareTableQuery(t *testing.T) { end: 1701796780000, expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC", }, + { + name: "test time aggregation = avg, space aggregation = avg, temporality = unspecified, testing metrics and attribute name with dot", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorAvg, + AggregateAttribute: v3.AttributeKey{ + Key: "system.memory.usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, + }, + Temporality: v3.Unspecified, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationAvg, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "host.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "signoz-host", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "state", + Order: v3.DirectionDesc, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "state", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + start: 1735295140000, + end: 1735554340000, + expectedQueryContains: "SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.memory.usage' AND temporality = 'Unspecified' AND unix_milli >= 1735257600000 AND unix_milli < 1735554340000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name = 'system.memory.usage' AND unix_milli >= 1735295140000 AND unix_milli < 1735554340000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC", + }, } for _, testCase := range testCases { diff --git a/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go b/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go index ce47e10b10..68930ea841 100644 --- a/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go +++ b/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go @@ -212,6 +212,61 @@ func TestPrepareTimeseriesQuery(t *testing.T) { end: 1701796780000, expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC", }, + { + name: "test time aggregation = avg, space aggregation = avg, temporality = unspecified, testing metrics and attribute name with dot", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorAvg, + AggregateAttribute: v3.AttributeKey{ + Key: "system.memory.usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, + }, + Temporality: v3.Unspecified, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationAvg, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "host.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "signoz-host", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "state", + Order: v3.DirectionDesc, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "state", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + start: 1735295140000, + end: 1735554340000, + expectedQueryContains: "SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.memory.usage' AND temporality = 'Unspecified' AND unix_milli >= 1735257600000 AND unix_milli < 1735554340000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name = 'system.memory.usage' AND unix_milli >= 1735295140000 AND unix_milli < 1735554340000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC", + }, } for _, testCase := range testCases { diff --git a/pkg/query-service/app/metrics/v4/delta/table_test.go b/pkg/query-service/app/metrics/v4/delta/table_test.go index 40b9297dc2..59687c3429 100644 --- a/pkg/query-service/app/metrics/v4/delta/table_test.go +++ b/pkg/query-service/app/metrics/v4/delta/table_test.go @@ -97,6 +97,54 @@ func TestPrepareTableQuery(t *testing.T) { end: 1701796780000, expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'http_requests' AND temporality = 'Delta' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC", }, + { + name: "test time aggregation = rate, space aggregation = avg, temporality = delta, testing metrics and attribute name with dot", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorRate, + AggregateAttribute: v3.AttributeKey{ + Key: "signoz.latency.sum", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Sum"), + IsColumn: true, + }, + Temporality: v3.Delta, + TimeAggregation: v3.TimeAggregationRate, + SpaceAggregation: v3.SpaceAggregationAvg, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "host.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "4f6ec470feea", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "status.code", + Order: v3.DirectionAsc, + }, + }, + GroupBy: []v3.AttributeKey{}, + Legend: "", + ReduceTo: v3.ReduceToOperatorLast, + Having: []v3.Having{}, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT ts, avg(per_series_value) as value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'signoz.latency.sum' AND temporality = 'Delta' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'host.name') = '4f6ec470feea') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz.latency.sum' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC", + }, } for _, testCase := range testCases { diff --git a/pkg/query-service/app/metrics/v4/delta/time_series_test.go b/pkg/query-service/app/metrics/v4/delta/time_series_test.go index 5b6d71b3de..f55dfc8b08 100644 --- a/pkg/query-service/app/metrics/v4/delta/time_series_test.go +++ b/pkg/query-service/app/metrics/v4/delta/time_series_test.go @@ -246,6 +246,61 @@ func TestPrepareTimeseriesQuery(t *testing.T) { end: 1701796780000, expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, quantilesDDMerge(0.01, 0.990000)(sketch)[1] as value FROM signoz_metrics.distributed_exp_hist INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'signoz_latency' AND temporality = 'Delta' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000) as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC", }, + { + name: "test time aggregation = rate, space aggregation = max, temporality = delta, testing metrics and attribute name with dot", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorRate, + AggregateAttribute: v3.AttributeKey{ + Key: "signoz.latency.sum", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Sum"), + IsColumn: true, + }, + Temporality: v3.Delta, + TimeAggregation: v3.TimeAggregationRate, + SpaceAggregation: v3.SpaceAggregationMax, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "host_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "4f6ec470feea", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "status.code", + Order: v3.DirectionAsc, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "host.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + start: 1735036101000, + end: 1735637901000, + expectedQueryContains: "SELECT `host.name`, ts, max(per_series_value) as value FROM (SELECT fingerprint, any(`host.name`) as `host.name`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host.name') as `host.name`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'signoz.latency.sum' AND temporality = 'Delta' AND unix_milli >= 1734998400000 AND unix_milli < 1735637901000 AND JSONExtractString(labels, 'host_name') = '4f6ec470feea') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz.latency.sum' AND unix_milli >= 1735036101000 AND unix_milli < 1735637901000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY `host.name`, ts ORDER BY `host.name` ASC, ts ASC", + }, } for _, testCase := range testCases { diff --git a/pkg/query-service/app/metrics/v4/helpers/clauses.go b/pkg/query-service/app/metrics/v4/helpers/clauses.go index e99951df69..3787b40a33 100644 --- a/pkg/query-service/app/metrics/v4/helpers/clauses.go +++ b/pkg/query-service/app/metrics/v4/helpers/clauses.go @@ -2,6 +2,7 @@ package helpers import ( "fmt" + "go.signoz.io/signoz/pkg/query-service/utils" "strings" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" @@ -18,6 +19,7 @@ func groupingSets(tags ...string) string { func GroupingSetsByAttributeKeyTags(tags ...v3.AttributeKey) string { groupTags := []string{} for _, tag := range tags { + tag.Key = utils.AddBackTickToFormatTag(tag.Key) groupTags = append(groupTags, tag.Key) } return groupingSets(groupTags...) @@ -27,6 +29,7 @@ func GroupingSetsByAttributeKeyTags(tags ...v3.AttributeKey) string { func GroupByAttributeKeyTags(tags ...v3.AttributeKey) string { groupTags := []string{} for _, tag := range tags { + tag.Key = utils.AddBackTickToFormatTag(tag.Key) groupTags = append(groupTags, tag.Key) } groupTags = append(groupTags, "ts") @@ -42,11 +45,13 @@ func OrderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string for _, item := range items { if item.ColumnName == tag.Key { found = true + item.ColumnName = utils.AddBackTickToFormatTag(item.ColumnName) orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) break } } if !found { + tag.Key = utils.AddBackTickToFormatTag(tag.Key) orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag.Key)) } } @@ -59,6 +64,7 @@ func OrderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string func SelectLabelsAny(tags []v3.AttributeKey) string { var selectLabelsAny []string for _, tag := range tags { + tag.Key = utils.AddBackTickToFormatTag(tag.Key) selectLabelsAny = append(selectLabelsAny, fmt.Sprintf("any(%s) as %s,", tag.Key, tag.Key)) } return strings.Join(selectLabelsAny, " ") @@ -67,6 +73,7 @@ func SelectLabelsAny(tags []v3.AttributeKey) string { func SelectLabels(tags []v3.AttributeKey) string { var selectLabels []string for _, tag := range tags { + tag.Key = utils.AddBackTickToFormatTag(tag.Key) selectLabels = append(selectLabels, fmt.Sprintf("%s,", tag.Key)) } return strings.Join(selectLabels, " ") diff --git a/pkg/query-service/app/metrics/v4/helpers/sub_query.go b/pkg/query-service/app/metrics/v4/helpers/sub_query.go index 3ce933661a..793cf2c375 100644 --- a/pkg/query-service/app/metrics/v4/helpers/sub_query.go +++ b/pkg/query-service/app/metrics/v4/helpers/sub_query.go @@ -322,7 +322,7 @@ func PrepareTimeseriesFilterQuery(start, end int64, mq *v3.BuilderQuery) (string var selectLabels string for _, tag := range groupTags { - selectLabels += fmt.Sprintf("JSONExtractString(labels, '%s') as %s, ", tag.Key, tag.Key) + selectLabels += fmt.Sprintf("JSONExtractString(labels, '%s') as %s, ", tag.Key, utils.AddBackTickToFormatTag(tag.Key)) } // The table JOIN key always exists @@ -406,7 +406,7 @@ func PrepareTimeseriesFilterQueryV3(start, end int64, mq *v3.BuilderQuery) (stri selectLabels += "labels, " } else { for _, tag := range groupTags { - selectLabels += fmt.Sprintf("JSONExtractString(labels, '%s') as %s, ", tag.Key, tag.Key) + selectLabels += fmt.Sprintf("JSONExtractString(labels, '%s') as %s, ", tag.Key, utils.AddBackTickToFormatTag(tag.Key)) } } diff --git a/pkg/query-service/app/metrics/v4/query_builder_test.go b/pkg/query-service/app/metrics/v4/query_builder_test.go index 2fc83a9e1f..be1f5f65ba 100644 --- a/pkg/query-service/app/metrics/v4/query_builder_test.go +++ b/pkg/query-service/app/metrics/v4/query_builder_test.go @@ -533,6 +533,75 @@ func TestPrepareMetricQueryGauge(t *testing.T) { }, expectedQueryContains: "SELECT host_name, ts, sum(per_series_value) as value FROM (SELECT fingerprint, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system_cpu_usage' AND temporality = 'Unspecified' AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name = 'system_cpu_usage' AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY host_name, ts ORDER BY host_name ASC, ts ASC", }, + { + name: "test gauge query with multiple group by with metric and attribute name containing dot", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorMax, + AggregateAttribute: v3.AttributeKey{ + Key: "system.memory.usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, + }, + Temporality: v3.Unspecified, + TimeAggregation: v3.TimeAggregationMax, + SpaceAggregation: v3.SpaceAggregationMax, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "host.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "signoz-host", + }, + }, + }, + Expression: "A", + Disabled: false, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "os.type", + Order: v3.DirectionDesc, + }, + { + ColumnName: "state", + Order: v3.DirectionAsc, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + { + Key: "state", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + { + Key: "host.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + expectedQueryContains: "SELECT `os.type`, state, `host.name`, ts, max(per_series_value) as value FROM (SELECT fingerprint, any(`os.type`) as `os.type`, any(state) as state, any(`host.name`) as `host.name`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, JSONExtractString(labels, 'state') as state, JSONExtractString(labels, 'host.name') as `host.name`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.memory.usage' AND temporality = 'Unspecified' AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name = 'system.memory.usage' AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY `os.type`, state, `host.name`, ts ORDER BY `os.type` desc, state asc, `host.name` ASC, ts ASC", + }, } for _, testCase := range testCases { diff --git a/pkg/query-service/app/queryBuilder/query_builder_test.go b/pkg/query-service/app/queryBuilder/query_builder_test.go index 36365060c3..1ee73cd095 100644 --- a/pkg/query-service/app/queryBuilder/query_builder_test.go +++ b/pkg/query-service/app/queryBuilder/query_builder_test.go @@ -216,6 +216,136 @@ func TestBuildQueryWithThreeOrMoreQueriesRefAndFormula(t *testing.T) { // So(queries["F5"], ShouldContainSubstring, "SELECT A.ts as ts, ((A.value - B.value) / B.value) * 100") // So(strings.Count(queries["F5"], " ON "), ShouldEqual, 1) }) + t.Run("TestBuildQueryWithDotMetricNameAndAttribute", func(t *testing.T) { + q := &v3.QueryRangeParamsV3{ + Start: 1735036101000, + End: 1735637901000, + Step: 60, + Variables: map[string]interface{}{ + "SIGNOZ_START_TIME": 1735034992000, + "SIGNOZ_END_TIME": 1735036792000, + }, + FormatForWeb: false, + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + PanelType: v3.PanelTypeGraph, + FillGaps: false, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorAvg, + AggregateAttribute: v3.AttributeKey{ + Key: "system.memory.usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Gauge"), + IsColumn: true, + }, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationSum, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + Operator: v3.FilterOperatorEqual, + Value: "linux", + }, + }, + }, + Expression: "A", + Disabled: true, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + ColumnName: "#SIGNOZ_VALUE", + Order: v3.DirectionAsc, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{}, + }, + "B": { + QueryName: "B", + DataSource: v3.DataSourceMetrics, + AggregateOperator: v3.AggregateOperatorSum, + AggregateAttribute: v3.AttributeKey{ + Key: "system.network.io", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType("Sum"), + IsColumn: true, + }, + TimeAggregation: v3.TimeAggregationIncrease, + SpaceAggregation: v3.SpaceAggregationSum, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{}, + }, + Expression: "B", + Disabled: true, + StepInterval: 60, + OrderBy: []v3.OrderBy{ + { + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "os.type", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: false, + }, + }, + Legend: "", + ReduceTo: v3.ReduceToOperatorAvg, + Having: []v3.Having{ + { + ColumnName: "SUM(system.network.io)", + Operator: v3.HavingOperatorGreaterThan, + Value: 4, + }, + }, + }, + "F1": { + QueryName: "F1", + Expression: "A + B", + Disabled: false, + Legend: "", + OrderBy: []v3.OrderBy{}, + Limit: 2, + }, + }, + }, + } + qbOptions := QueryBuilderOptions{ + BuildMetricQuery: metricsv3.PrepareMetricQuery, + } + fm := featureManager.StartManager() + qb := NewQueryBuilder(qbOptions, fm) + + queries, err := qb.PrepareQueries(q) + require.Contains(t, queries["F1"], "SELECT A.`os.type` as `os.type`, A.`ts` as `ts`, A.value + B.value as value FROM (SELECT `os.type`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.memory.usage' AND temporality = '' AND unix_milli >= 1734998400000 AND unix_milli < 1735637880000 AND JSONExtractString(labels, 'os.type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name = 'system.memory.usage' AND unix_milli >= 1735036080000 AND unix_milli < 1735637880000 GROUP BY `os.type`, ts ORDER BY `os.type` ASC, ts) as A INNER JOIN (SELECT * FROM (SELECT `os.type`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value) as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name = 'system.network.io' AND temporality = '' AND unix_milli >= 1734998400000 AND unix_milli < 1735637880000) as filtered_time_series USING fingerprint WHERE metric_name = 'system.network.io' AND unix_milli >= 1735036020000 AND unix_milli < 1735637880000 GROUP BY `os.type`, ts ORDER BY `os.type` ASC, ts) HAVING value > 4) as B ON A.`os.type` = B.`os.type` AND A.`ts` = B.`ts`") + require.NoError(t, err) + + }) } func TestDeltaQueryBuilder(t *testing.T) { diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go index 3533f803a5..3a34d3d80f 100644 --- a/pkg/query-service/utils/format.go +++ b/pkg/query-service/utils/format.go @@ -230,6 +230,22 @@ func ClickHouseFormattedValue(v interface{}) string { } } +func AddBackTickToFormatTag(str string) string { + if strings.Contains(str, ".") { + return "`" + str + "`" + } else { + return str + } +} + +func AddBackTickToFormatTags(inputs ...string) []string { + result := make([]string, len(inputs)) + for i, str := range inputs { + result[i] = AddBackTickToFormatTag(str) + } + return result +} + func getPointerValue(v interface{}) interface{} { switch x := v.(type) { case *uint8: