diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml index 8f13b6506f..f8a710d535 100644 --- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml @@ -98,6 +98,7 @@ processors: latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] dimensions_cache_size: 100000 aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true dimensions: - name: service.namespace default: default diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml index 8211364efb..d382a252e5 100644 --- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml @@ -101,6 +101,7 @@ processors: latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] dimensions_cache_size: 100000 aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA + enable_exp_histogram: true dimensions: - name: service.namespace default: default diff --git a/pkg/query-service/app/metrics/v4/delta/time_series_test.go b/pkg/query-service/app/metrics/v4/delta/time_series_test.go index 0af2c91154..29c2340ec4 100644 --- a/pkg/query-service/app/metrics/v4/delta/time_series_test.go +++ b/pkg/query-service/app/metrics/v4/delta/time_series_test.go @@ -212,6 +212,40 @@ func TestPrepareTimeseriesQuery(t *testing.T) { end: 1701796780000, expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'http_requests' AND temporality = 'Delta' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY GROUPING SETS ( (service_name, ts), (service_name) ) ORDER BY service_name ASC, ts ASC", }, + { + name: "test time aggregation = rate, space aggregation percentile99, type = ExponentialHistogram", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "signoz_latency", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyType(v3.MetricTypeExponentialHistogram), + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Delta, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{}, + }, + GroupBy: []v3.AttributeKey{ + { + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationRate, + SpaceAggregation: v3.SpaceAggregationPercentile99, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, quantilesDDMerge(0.01, 0.990000)(sketch)[1] as value FROM signoz_metrics.distributed_exp_hist INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name = 'signoz_latency' AND temporality = 'Delta' AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000) as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency' AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY GROUPING SETS ( (service_name, ts), (service_name) ) ORDER BY service_name ASC, ts ASC", + }, } for _, testCase := range testCases { diff --git a/pkg/query-service/app/metrics/v4/delta/timeseries.go b/pkg/query-service/app/metrics/v4/delta/timeseries.go index 03781dfcd1..365b09c56d 100644 --- a/pkg/query-service/app/metrics/v4/delta/timeseries.go +++ b/pkg/query-service/app/metrics/v4/delta/timeseries.go @@ -9,6 +9,11 @@ import ( "go.signoz.io/signoz/pkg/query-service/utils" ) +// TODO(srikanthccv): support multiple quantiles; see https://github.com/SigNoz/signoz/issues/4016#issuecomment-1838583305 +var ( + sketchFmt = "quantilesDDMerge(0.01, %f)(sketch)[1]" +) + // prepareTimeAggregationSubQuery builds the sub-query to be used for temporal aggregation func prepareTimeAggregationSubQuery(start, end, step int64, mq *v3.BuilderQuery) (string, error) { @@ -84,12 +89,16 @@ func prepareQueryOptimized(start, end, step int64, mq *v3.BuilderQuery) (string, samplesTableFilter := fmt.Sprintf("metric_name = %s AND unix_milli >= %d AND unix_milli < %d", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key), start, end) + var tableName string = constants.SIGNOZ_SAMPLES_V4_TABLENAME + if mq.AggregateAttribute.Type == v3.AttributeKeyType(v3.MetricTypeExponentialHistogram) { + tableName = "distributed_exp_hist" + } // Select the aggregate value for interval queryTmpl := "SELECT %s" + " toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL %d SECOND) as ts," + " %s as value" + - " FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + constants.SIGNOZ_SAMPLES_V4_TABLENAME + + " FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + tableName + " INNER JOIN" + " (%s) as filtered_time_series" + " USING fingerprint" + @@ -110,6 +119,13 @@ func prepareQueryOptimized(start, end, step int64, mq *v3.BuilderQuery) (string, case v3.SpaceAggregationMax: op := "max(value)" query = fmt.Sprintf(queryTmpl, selectLabels, step, op, timeSeriesSubQuery, groupBy, orderBy) + case v3.SpaceAggregationPercentile50, + v3.SpaceAggregationPercentile75, + v3.SpaceAggregationPercentile90, + v3.SpaceAggregationPercentile95, + v3.SpaceAggregationPercentile99: + op := fmt.Sprintf(sketchFmt, v3.GetPercentileFromOperator(mq.SpaceAggregation)) + query = fmt.Sprintf(queryTmpl, selectLabels, step, op, timeSeriesSubQuery, groupBy, orderBy) } return query, nil } @@ -178,6 +194,9 @@ func PrepareMetricQueryDeltaTimeSeries(start, end, step int64, mq *v3.BuilderQue // 4. time aggregation = max and space aggregation = max // - max of maxs is same as max of all values // +// 5. special case exphist, there is no need for per series/fingerprint aggregation +// we can directly use the quantilesDDMerge function +// // all of this is true only for delta metrics func canShortCircuit(mq *v3.BuilderQuery) bool { if (mq.TimeAggregation == v3.TimeAggregationRate || mq.TimeAggregation == v3.TimeAggregationIncrease) && mq.SpaceAggregation == v3.SpaceAggregationSum { @@ -192,5 +211,8 @@ func canShortCircuit(mq *v3.BuilderQuery) bool { if mq.TimeAggregation == v3.TimeAggregationMax && mq.SpaceAggregation == v3.SpaceAggregationMax { return true } + if mq.AggregateAttribute.Type == v3.AttributeKeyType(v3.MetricTypeExponentialHistogram) && v3.IsPercentileOperator(mq.SpaceAggregation) { + return true + } return false } diff --git a/pkg/query-service/app/metrics/v4/helpers/clauses.go b/pkg/query-service/app/metrics/v4/helpers/clauses.go index 8714df51da..06f4b13cea 100644 --- a/pkg/query-service/app/metrics/v4/helpers/clauses.go +++ b/pkg/query-service/app/metrics/v4/helpers/clauses.go @@ -37,17 +37,6 @@ func GroupByAttributeKeyTags(tags ...v3.AttributeKey) string { return strings.Join(groupTags, ", ") } -func GroupByAttributeKeyTagsWithoutLe(tags ...v3.AttributeKey) string { - groupTags := []string{} - for _, tag := range tags { - if tag.Key != "le" { - groupTags = append(groupTags, tag.Key) - } - } - groupTags = append(groupTags, "ts") - return strings.Join(groupTags, ", ") -} - // OrderByAttributeKeyTags returns a string of comma separated tags for order by clause // if the order is not specified, it defaults to ASC func OrderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string { @@ -71,29 +60,6 @@ func OrderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string return strings.Join(orderBy, ", ") } -func OrderByAttributeKeyTagsWithoutLe(items []v3.OrderBy, tags []v3.AttributeKey) string { - var orderBy []string - for _, tag := range tags { - if tag.Key != "le" { - found := false - for _, item := range items { - if item.ColumnName == tag.Key { - found = true - orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) - break - } - } - if !found { - orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag.Key)) - } - } - } - - orderBy = append(orderBy, "ts ASC") - - return strings.Join(orderBy, ", ") -} - func SelectLabelsAny(tags []v3.AttributeKey) string { var selectLabelsAny []string for _, tag := range tags { diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 010b0b41c1..c01660d6e7 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -489,6 +489,17 @@ func (t TimeAggregation) IsRateOperator() bool { } } +type MetricType string + +const ( + MetricTypeUnspecified MetricType = "" + MetricTypeSum MetricType = "Sum" + MetricTypeGauge MetricType = "Gauge" + MetricTypeHistogram MetricType = "Histogram" + MetricTypeSummary MetricType = "Summary" + MetricTypeExponentialHistogram MetricType = "ExponentialHistogram" +) + type SpaceAggregation string const (