From e04d5fa7e865ce6fea28e0d0a90a0641a261ce7f Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 22 Aug 2023 17:09:58 +0530 Subject: [PATCH] Align the query start and end timestamps to the nearest multiple of step (#2844) --- .../app/logs/v3/query_builder.go | 8 + .../app/logs/v3/query_builder_test.go | 150 +++++++------- .../app/metrics/v3/query_builder.go | 9 + .../app/metrics/v3/query_builder_test.go | 184 +++++++++++++++++- pkg/query-service/app/querier/querier_test.go | 8 +- .../app/queryBuilder/query_builder_test.go | 18 +- .../app/traces/v3/query_builder.go | 6 + .../app/traces/v3/query_builder_test.go | 110 +++++------ pkg/query-service/model/v3/v3.go | 2 +- 9 files changed, 341 insertions(+), 154 deletions(-) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index 48265745b4..4a95fe9d1d 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -420,7 +420,15 @@ func isOrderByTs(orderBy []v3.OrderBy) bool { return false } +// PrepareLogsQuery prepares the query for logs +// start and end are in epoch millisecond +// step is in seconds func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { + + // adjust the start and end time to the step interval + start = start - (start % (mq.StepInterval * 1000)) + end = end - (end % (mq.StepInterval * 1000)) + if options.IsLivetailQuery { query, err := buildLogsLiveTailQuery(mq) if err != nil { diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index ebc62ec5e5..d8c0eedd32 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -259,9 +259,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, @@ -273,9 +273,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", @@ -288,9 +288,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCount, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -306,9 +306,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -322,9 +322,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -337,9 +337,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -366,9 +366,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -397,9 +397,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorAvg, Expression: "A", @@ -426,9 +426,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, AggregateOperator: v3.AggregateOperatorSum, Expression: "A", @@ -455,9 +455,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, AggregateOperator: v3.AggregateOperatorMin, Expression: "A", @@ -484,9 +484,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, AggregateOperator: v3.AggregateOperatorMax, Expression: "A", @@ -513,9 +513,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, AggregateOperator: v3.AggregateOperatorP05, Expression: "A", @@ -538,9 +538,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, AggregateOperator: v3.AggregateOperatorRateSum, Expression: "A", @@ -561,9 +561,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64}, AggregateOperator: v3.AggregateOperatorRate, Expression: "A", @@ -585,9 +585,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64}, AggregateOperator: v3.AggregateOperatorRateSum, Expression: "A", @@ -610,10 +610,10 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeList, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ SelectColumns: []v3.AttributeKey{}, QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, @@ -628,7 +628,6 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeList, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ SelectColumns: []v3.AttributeKey{}, QueryName: "A", @@ -647,7 +646,6 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeList, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ SelectColumns: []v3.AttributeKey{}, QueryName: "A", @@ -667,9 +665,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -689,9 +687,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -715,9 +713,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -741,9 +739,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeGraph, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -769,9 +767,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeTable, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", }, @@ -783,9 +781,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeTable, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", GroupBy: []v3.AttributeKey{ @@ -800,9 +798,9 @@ var testBuildLogsQueryData = []struct { PanelType: v3.PanelTypeTable, Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", GroupBy: []v3.AttributeKey{ @@ -820,7 +818,7 @@ var testBuildLogsQueryData = []struct { func TestBuildLogsQuery(t *testing.T) { for _, tt := range testBuildLogsQueryData { Convey("TestBuildLogsQuery", t, func() { - query, err := buildLogsQuery(tt.PanelType, tt.Start, tt.End, tt.Step, tt.BuilderQuery, "", tt.PreferRPM) + query, err := buildLogsQuery(tt.PanelType, tt.Start, tt.End, tt.BuilderQuery.StepInterval, tt.BuilderQuery, "", tt.PreferRPM) So(err, ShouldBeNil) So(query, ShouldEqual, tt.ExpectedQuery) @@ -980,11 +978,11 @@ var testPrepLogsQueryData = []struct { { Name: "Test TS with limit- first", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -996,17 +994,17 @@ var testPrepLogsQueryData = []struct { GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, }, TableName: "logs", - ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by value DESC) LIMIT 10", + ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by value DESC) LIMIT 10", Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by value", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -1019,17 +1017,17 @@ var testPrepLogsQueryData = []struct { OrderBy: []v3.OrderBy{{ColumnName: constants.SigNozOrderByValue, Order: "ASC"}}, }, TableName: "logs", - ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by value ASC) LIMIT 10", + ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by value ASC) LIMIT 10", Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by attribute", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -1042,17 +1040,17 @@ var testPrepLogsQueryData = []struct { OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, }, TableName: "logs", - ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by method ASC) LIMIT 10", + ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by method ASC) LIMIT 10", Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- second", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -1064,17 +1062,17 @@ var testPrepLogsQueryData = []struct { Limit: 2, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 0 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND (method) GLOBAL IN (%s) group by method,ts order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND (method) GLOBAL IN (%s) group by method,ts order by value DESC", Options: Options{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, { Name: "Test TS with limit- second - with order by", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -1087,18 +1085,18 @@ var testPrepLogsQueryData = []struct { Limit: 2, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 0 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND (method) GLOBAL IN (%s) group by method,ts order by method ASC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND (method) GLOBAL IN (%s) group by method,ts order by method ASC", Options: Options{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, // Live tail { Name: "Live Tail Query", PanelType: v3.PanelTypeList, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -1113,11 +1111,11 @@ var testPrepLogsQueryData = []struct { { Name: "Live Tail Query W/O filter", PanelType: v3.PanelTypeList, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, @@ -1129,34 +1127,34 @@ var testPrepLogsQueryData = []struct { { Name: "Table query w/o limit", PanelType: v3.PanelTypeTable, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, }, TableName: "logs", - ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) order by value DESC", + ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) order by value DESC", Options: Options{}, }, { Name: "Table query with limit", PanelType: v3.PanelTypeTable, - Start: 1680066360726210000, - End: 1680066458000000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, Limit: 10, }, TableName: "logs", - ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) order by value DESC LIMIT 10", + ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) order by value DESC LIMIT 10", Options: Options{}, }, } @@ -1188,11 +1186,11 @@ var testPrepLogsQueryLimitOffsetData = []struct { { Name: "Test limit less than pageSize - order by ts", PanelType: v3.PanelTypeList, - Start: 1680518666000000000, - End: 1691618704365000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, @@ -1202,16 +1200,16 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 5, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) order by timestamp desc LIMIT 1", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) order by timestamp desc LIMIT 1", }, { Name: "Test limit greater than pageSize - order by ts", PanelType: v3.PanelTypeList, - Start: 1680518666000000000, - End: 1691618704365000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -1223,16 +1221,16 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 10, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by timestamp desc LIMIT 10", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by timestamp desc LIMIT 10", }, { Name: "Test limit less than pageSize - order by custom", PanelType: v3.PanelTypeList, - Start: 1680518666000000000, - End: 1691618704365000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, @@ -1242,16 +1240,16 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 5, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 1 OFFSET 0", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 1 OFFSET 0", }, { Name: "Test limit greater than pageSize - order by custom", PanelType: v3.PanelTypeList, - Start: 1680518666000000000, - End: 1691618704365000000, - Step: 60, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorNoOp, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -1263,7 +1261,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 50, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 50 OFFSET 50", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 50 OFFSET 50", }, } diff --git a/pkg/query-service/app/metrics/v3/query_builder.go b/pkg/query-service/app/metrics/v3/query_builder.go index 03ac87d90f..1b058d5d93 100644 --- a/pkg/query-service/app/metrics/v3/query_builder.go +++ b/pkg/query-service/app/metrics/v3/query_builder.go @@ -422,7 +422,16 @@ func reduceQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v return query, nil } +// PrepareMetricQuery prepares the query to be used for fetching metrics +// from the database +// start and end are in milliseconds +// step is in seconds func PrepareMetricQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { + + // adjust the start and end time to be aligned with the step interval + start = start - (start % (mq.StepInterval * 1000)) + end = end - (end % (mq.StepInterval * 1000)) + var query string var err error if mq.Temporality == v3.Delta { diff --git a/pkg/query-service/app/metrics/v3/query_builder_test.go b/pkg/query-service/app/metrics/v3/query_builder_test.go index 7978a5d2c0..d315c52081 100644 --- a/pkg/query-service/app/metrics/v3/query_builder_test.go +++ b/pkg/query-service/app/metrics/v3/query_builder_test.go @@ -13,11 +13,11 @@ func TestBuildQuery(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name"}, AggregateOperator: v3.AggregateOperatorRateMax, Expression: "A", @@ -38,11 +38,11 @@ func TestBuildQueryWithFilters(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name"}, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "a"}, Value: "b", Operator: v3.FilterOperatorNotEqual}, @@ -68,11 +68,11 @@ func TestBuildQueryWithMultipleQueries(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name"}, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, @@ -230,6 +230,7 @@ func TestBuildQueryOperators(t *testing.T) { t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { mq := v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"}, AggregateOperator: v3.AggregateOperatorSum, } @@ -243,7 +244,7 @@ func TestBuildQueryOperators(t *testing.T) { func TestBuildQueryXRate(t *testing.T) { t.Run("TestBuildQueryXRate", func(t *testing.T) { - tmpl := `SELECT ts, %s(value) as value FROM (SELECT ts, if(runningDifference(ts) <= 0, nan, if(runningDifference(value) < 0, (value) / runningDifference(ts), runningDifference(value) / runningDifference(ts))) as value FROM(SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 0 SECOND) as ts, max(value) as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'name' AND temporality IN ['Cumulative', 'Unspecified']) as filtered_time_series USING fingerprint WHERE metric_name = 'name' AND timestamp_ms >= 1650991982000 AND timestamp_ms <= 1651078382000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(value) = 0) GROUP BY GROUPING SETS ( (ts), () ) ORDER BY ts` + tmpl := `SELECT ts, %s(value) as value FROM (SELECT ts, if(runningDifference(ts) <= 0, nan, if(runningDifference(value) < 0, (value) / runningDifference(ts), runningDifference(value) / runningDifference(ts))) as value FROM(SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, max(value) as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'name' AND temporality IN ['Cumulative', 'Unspecified']) as filtered_time_series USING fingerprint WHERE metric_name = 'name' AND timestamp_ms >= 1650991980000 AND timestamp_ms <= 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(value) = 0) GROUP BY GROUPING SETS ( (ts), () ) ORDER BY ts` cases := []struct { aggregateOperator v3.AggregateOperator @@ -272,11 +273,11 @@ func TestBuildQueryXRate(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name"}, AggregateOperator: c.aggregateOperator, Expression: "A", @@ -296,7 +297,7 @@ func TestBuildQueryXRate(t *testing.T) { func TestBuildQueryRPM(t *testing.T) { t.Run("TestBuildQueryXRate", func(t *testing.T) { - tmpl := `SELECT ts, ceil(value * 60) as value FROM (SELECT ts, %s(value) as value FROM (SELECT ts, if(runningDifference(ts) <= 0, nan, if(runningDifference(value) < 0, (value) / runningDifference(ts), runningDifference(value) / runningDifference(ts))) as value FROM(SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 0 SECOND) as ts, max(value) as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'name' AND temporality IN ['Cumulative', 'Unspecified']) as filtered_time_series USING fingerprint WHERE metric_name = 'name' AND timestamp_ms >= 1650991982000 AND timestamp_ms <= 1651078382000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(value) = 0) GROUP BY GROUPING SETS ( (ts), () ) ORDER BY ts)` + tmpl := `SELECT ts, ceil(value * 60) as value FROM (SELECT ts, %s(value) as value FROM (SELECT ts, if(runningDifference(ts) <= 0, nan, if(runningDifference(value) < 0, (value) / runningDifference(ts), runningDifference(value) / runningDifference(ts))) as value FROM(SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, max(value) as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'name' AND temporality IN ['Cumulative', 'Unspecified']) as filtered_time_series USING fingerprint WHERE metric_name = 'name' AND timestamp_ms >= 1650991980000 AND timestamp_ms <= 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(value) = 0) GROUP BY GROUPING SETS ( (ts), () ) ORDER BY ts)` cases := []struct { aggregateOperator v3.AggregateOperator @@ -325,11 +326,11 @@ func TestBuildQueryRPM(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name"}, AggregateOperator: c.aggregateOperator, Expression: "A", @@ -345,3 +346,172 @@ func TestBuildQueryRPM(t *testing.T) { } }) } + +func TestBuildQueryAdjustedTimes(t *testing.T) { + cases := []struct { + name string + params *v3.QueryRangeParamsV3 + expected string + }{ + { + name: "TestBuildQueryAdjustedTimes start close to 30 seconds", + params: &v3.QueryRangeParamsV3{ + // 20:11:29 + Start: 1686082289000, + // 20:41:00 + End: 1686084060000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + }, + }, + }, + // 20:11:00 - 20:41:00 + expected: "timestamp_ms >= 1686082260000 AND timestamp_ms <= 1686084060000", + }, + { + name: "TestBuildQueryAdjustedTimes start close to 50 seconds", + params: &v3.QueryRangeParamsV3{ + // 20:11:52 + Start: 1686082312000, + // 20:41:00 + End: 1686084060000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + }, + }, + }, + // 20:11:00 - 20:41:00 + expected: "timestamp_ms >= 1686082260000 AND timestamp_ms <= 1686084060000", + }, + { + name: "TestBuildQueryAdjustedTimes start close to 42 seconds with step 30 seconds", + params: &v3.QueryRangeParamsV3{ + // 20:11:42 + Start: 1686082302000, + // 20:41:00 + End: 1686084060000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 30, + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + }, + }, + }, + // 20:11:30 - 20:41:00 + expected: "timestamp_ms >= 1686082290000 AND timestamp_ms <= 1686084060000", + }, + { + name: "TestBuildQueryAdjustedTimes start close to 42 seconds with step 30 seconds and end close to 30 seconds", + params: &v3.QueryRangeParamsV3{ + // 20:11:42 + Start: 1686082302000, + // 20:41:29 + End: 1686084089000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 30, + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + }, + }, + }, + // 20:11:30 - 20:41:00 + expected: "timestamp_ms >= 1686082290000 AND timestamp_ms <= 1686084060000", + }, + { + name: "TestBuildQueryAdjustedTimes start close to 42 seconds with step 300 seconds and end close to 30 seconds", + params: &v3.QueryRangeParamsV3{ + // 20:11:42 + Start: 1686082302000, + // 20:41:29 + End: 1686084089000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 300, + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + }, + }, + }, + // 20:10:00 - 20:40:00 + expected: "timestamp_ms >= 1686082200000 AND timestamp_ms <= 1686084000000", + }, + { + name: "TestBuildQueryAdjustedTimes start close to 42 seconds with step 180 seconds and end close to 30 seconds", + params: &v3.QueryRangeParamsV3{ + // 20:11:42 + Start: 1686082302000, + // 20:41:29 + End: 1686084089000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 180, + AggregateAttribute: v3.AttributeKey{Key: "name"}, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "in"}, Value: []interface{}{"a", "b", "c"}, Operator: v3.FilterOperatorIn}, + }}, + AggregateOperator: v3.AggregateOperatorRateAvg, + Expression: "A", + }, + }, + }, + }, + // 20:09:00 - 20:39:00 + expected: "timestamp_ms >= 1686082140000 AND timestamp_ms <= 1686083940000", + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + q := testCase.params + query, err := PrepareMetricQuery(q.Start, q.End, q.CompositeQuery.QueryType, q.CompositeQuery.PanelType, q.CompositeQuery.BuilderQueries["A"], Options{PreferRPM: false}) + require.NoError(t, err) + + require.Contains(t, query, testCase.expected) + }) + } +} diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 51293ad493..46855add7d 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -404,12 +404,12 @@ func TestQueryRange(t *testing.T) { { Start: 1675115596722, End: 1675115596722 + 120*60*1000, - Step: 5 * time.Minute.Microseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "http_server_requests_seconds_count", Type: v3.AttributeKeyTypeUnspecified, DataType: "float64", IsColumn: true}, Filters: &v3.FilterSet{ Operator: "AND", @@ -434,12 +434,12 @@ func TestQueryRange(t *testing.T) { { Start: 1675115596722 + 60*60*1000, End: 1675115596722 + 180*60*1000, - Step: 5 * time.Minute.Microseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "http_server_requests_seconds_count", Type: v3.AttributeKeyTypeUnspecified, DataType: "float64", IsColumn: true}, Filters: &v3.FilterSet{ Operator: "AND", @@ -487,8 +487,8 @@ func TestQueryRange(t *testing.T) { } q := NewQuerier(opts) expectedTimeRangeInQueryString := []string{ - fmt.Sprintf("timestamp_ms >= %d AND timestamp_ms <= %d", 1675115596722, 1675115596722+120*60*1000), - fmt.Sprintf("timestamp_ms >= %d AND timestamp_ms <= %d", 1675115596722+120*60*1000+1, 1675115596722+180*60*1000), + fmt.Sprintf("timestamp_ms >= %d AND timestamp_ms <= %d", 1675115580000, 1675115580000+120*60*1000), + fmt.Sprintf("timestamp_ms >= %d AND timestamp_ms <= %d", 1675115580000+120*60*1000, 1675115580000+180*60*1000), } for i, param := range params { diff --git a/pkg/query-service/app/queryBuilder/query_builder_test.go b/pkg/query-service/app/queryBuilder/query_builder_test.go index 3dc80ae798..50693f642b 100644 --- a/pkg/query-service/app/queryBuilder/query_builder_test.go +++ b/pkg/query-service/app/queryBuilder/query_builder_test.go @@ -15,11 +15,11 @@ func TestBuildQueryWithMultipleQueriesAndFormula(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, DataSource: v3.DataSourceMetrics, AggregateAttribute: v3.AttributeKey{Key: "name"}, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -30,6 +30,7 @@ func TestBuildQueryWithMultipleQueriesAndFormula(t *testing.T) { }, "B": { QueryName: "B", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name2"}, DataSource: v3.DataSourceMetrics, AggregateOperator: v3.AggregateOperatorRateAvg, @@ -63,11 +64,11 @@ func TestBuildQueryWithIncorrectQueryRef(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, DataSource: v3.DataSourceMetrics, AggregateAttribute: v3.AttributeKey{Key: "name"}, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -101,11 +102,11 @@ func TestBuildQueryWithThreeOrMoreQueriesRefAndFormula(t *testing.T) { q := &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ "A": { QueryName: "A", + StepInterval: 60, DataSource: v3.DataSourceMetrics, AggregateAttribute: v3.AttributeKey{Key: "name"}, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -117,6 +118,7 @@ func TestBuildQueryWithThreeOrMoreQueriesRefAndFormula(t *testing.T) { }, "B": { QueryName: "B", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name2"}, DataSource: v3.DataSourceMetrics, @@ -126,6 +128,7 @@ func TestBuildQueryWithThreeOrMoreQueriesRefAndFormula(t *testing.T) { }, "C": { QueryName: "C", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name3"}, DataSource: v3.DataSourceMetrics, @@ -222,7 +225,6 @@ func TestDeltaQueryBuilder(t *testing.T) { query: &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, PanelType: v3.PanelTypeGraph, @@ -254,14 +256,13 @@ func TestDeltaQueryBuilder(t *testing.T) { }, }, queryToTest: "A", - expected: "SELECT toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_count' AND temporality = 'Delta' AND JSONExtractString(labels, 'service_name') IN ['frontend'] AND JSONExtractString(labels, 'operation') IN ['HTTP GET /dispatch'] AND JSONExtractString(labels, '__temporality__') = 'Delta') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_count' AND timestamp_ms >= 1650991982000 AND timestamp_ms <= 1651078382000 GROUP BY ts ORDER BY ts", + expected: "SELECT toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_count' AND temporality = 'Delta' AND JSONExtractString(labels, 'service_name') IN ['frontend'] AND JSONExtractString(labels, 'operation') IN ['HTTP GET /dispatch'] AND JSONExtractString(labels, '__temporality__') = 'Delta') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_count' AND timestamp_ms >= 1650991980000 AND timestamp_ms <= 1651078380000 GROUP BY ts ORDER BY ts", }, { name: "TestQueryWithExpression - Error rate", query: &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, PanelType: v3.PanelTypeGraph, @@ -325,14 +326,13 @@ func TestDeltaQueryBuilder(t *testing.T) { }, }, queryToTest: "C", - expected: "SELECT A.ts as ts, A.value * 100 / B.value as value FROM (SELECT toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_count' AND temporality = 'Delta' AND JSONExtractString(labels, 'service_name') IN ['frontend'] AND JSONExtractString(labels, 'operation') IN ['HTTP GET /dispatch'] AND JSONExtractString(labels, 'status_code') IN ['STATUS_CODE_ERROR'] AND JSONExtractString(labels, '__temporality__') = 'Delta') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_count' AND timestamp_ms >= 1650991982000 AND timestamp_ms <= 1651078382000 GROUP BY ts ORDER BY ts) as A INNER JOIN (SELECT toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_count' AND temporality = 'Delta' AND JSONExtractString(labels, 'service_name') IN ['frontend'] AND JSONExtractString(labels, 'operation') IN ['HTTP GET /dispatch'] AND JSONExtractString(labels, '__temporality__') = 'Delta') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_count' AND timestamp_ms >= 1650991982000 AND timestamp_ms <= 1651078382000 GROUP BY ts ORDER BY ts) as B ON A.ts = B.ts", + expected: "SELECT A.ts as ts, A.value * 100 / B.value as value FROM (SELECT toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_count' AND temporality = 'Delta' AND JSONExtractString(labels, 'service_name') IN ['frontend'] AND JSONExtractString(labels, 'operation') IN ['HTTP GET /dispatch'] AND JSONExtractString(labels, 'status_code') IN ['STATUS_CODE_ERROR'] AND JSONExtractString(labels, '__temporality__') = 'Delta') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_count' AND timestamp_ms >= 1650991980000 AND timestamp_ms <= 1651078380000 GROUP BY ts ORDER BY ts) as A INNER JOIN (SELECT toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_count' AND temporality = 'Delta' AND JSONExtractString(labels, 'service_name') IN ['frontend'] AND JSONExtractString(labels, 'operation') IN ['HTTP GET /dispatch'] AND JSONExtractString(labels, '__temporality__') = 'Delta') as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_count' AND timestamp_ms >= 1650991980000 AND timestamp_ms <= 1651078380000 GROUP BY ts ORDER BY ts) as B ON A.ts = B.ts", }, { name: "TestQuery - Quantile", query: &v3.QueryRangeParamsV3{ Start: 1650991982000, End: 1651078382000, - Step: 60, CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, PanelType: v3.PanelTypeGraph, @@ -353,7 +353,7 @@ func TestDeltaQueryBuilder(t *testing.T) { }, }, queryToTest: "A", - expected: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) as value FROM (SELECT service_name,le, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_bucket' AND temporality = 'Delta' ) as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_bucket' AND timestamp_ms >= 1650991982000 AND timestamp_ms <= 1651078382000 GROUP BY service_name,le,ts ORDER BY service_name ASC,le ASC, ts) GROUP BY service_name,ts ORDER BY service_name ASC, ts", + expected: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) as value FROM (SELECT service_name,le, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'signoz_latency_bucket' AND temporality = 'Delta' ) as filtered_time_series USING fingerprint WHERE metric_name = 'signoz_latency_bucket' AND timestamp_ms >= 1650991980000 AND timestamp_ms <= 1651078380000 GROUP BY service_name,le,ts ORDER BY service_name ASC,le ASC, ts) GROUP BY service_name,ts ORDER BY service_name ASC, ts", }, } diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index 38ab6fc70b..1ffd8d5e3a 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -502,7 +502,13 @@ func addOffsetToQuery(query string, offset uint64) string { return fmt.Sprintf("%s OFFSET %d", query, offset) } +// PrepareTracesQuery returns the query string for traces +// start and end are in epoch millisecond +// step is in seconds func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey, options Options) (string, error) { + // adjust the start and end time to the step interval + start = start - (start % (mq.StepInterval * 1000)) + end = end - (end % (mq.StepInterval * 1000)) if options.GraphLimitQtype == constants.FirstQueryGraphLimit { // give me just the group by names query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, options) diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 91eebf01ca..fd798c1530 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -486,9 +486,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count on fixed column of float64 type", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", AggregateAttribute: v3.AttributeKey{Key: "durationNano", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -503,9 +503,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate rate without aggregate attribute", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorRate, Expression: "A", }, @@ -520,9 +520,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count on fixed column of float64 type with filter", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", AggregateAttribute: v3.AttributeKey{Key: "durationNano", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -539,9 +539,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count on fixed column of bool type", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", AggregateAttribute: v3.AttributeKey{Key: "hasError", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -556,9 +556,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count on a attribute", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", @@ -573,9 +573,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count on a fixed column of string type", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", @@ -590,9 +590,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count with filter", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCount, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -610,9 +610,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count distinct and order by value", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -628,9 +628,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count distinct on string key", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -645,9 +645,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count distinct with filter and groupBy", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -673,9 +673,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate count with multiple filter,groupBy and orderBy", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -705,9 +705,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate avg", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorAvg, Expression: "A", @@ -733,9 +733,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate sum", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorSum, Expression: "A", @@ -761,9 +761,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate min", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorMin, Expression: "A", @@ -789,9 +789,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate max", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorMax, Expression: "A", @@ -817,9 +817,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate PXX", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorP05, Expression: "A", @@ -841,9 +841,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate RateSum", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorRateSum, Expression: "A", @@ -865,9 +865,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate rate", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64}, AggregateOperator: v3.AggregateOperatorRate, Expression: "A", @@ -888,9 +888,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate RateSum without fixed column", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64}, AggregateOperator: v3.AggregateOperatorRateSum, Expression: "A", @@ -912,9 +912,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate with having clause", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -936,9 +936,9 @@ var testBuildTracesQueryData = []struct { Name: "Test count aggregate with having clause and filters", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", @@ -964,9 +964,9 @@ var testBuildTracesQueryData = []struct { Name: "Test count distinct aggregate with having clause and filters", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", @@ -992,9 +992,9 @@ var testBuildTracesQueryData = []struct { Name: "Test count with having clause and filters", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorCount, Expression: "A", @@ -1020,9 +1020,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate PXX", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorP05, Expression: "A", @@ -1043,9 +1043,9 @@ var testBuildTracesQueryData = []struct { Name: "Test aggregate PXX", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", + StepInterval: 60, AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, AggregateOperator: v3.AggregateOperatorP05, Expression: "A", @@ -1063,7 +1063,6 @@ var testBuildTracesQueryData = []struct { Name: "Test Noop list view", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ SelectColumns: []v3.AttributeKey{ {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -1082,7 +1081,6 @@ var testBuildTracesQueryData = []struct { Name: "Test Noop list view with order by", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ SelectColumns: []v3.AttributeKey{ {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -1102,7 +1100,6 @@ var testBuildTracesQueryData = []struct { Name: "Test Noop list view with order by and filter", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ SelectColumns: []v3.AttributeKey{ {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -1125,7 +1122,6 @@ var testBuildTracesQueryData = []struct { Name: "Test Noop trace view", Start: 1680066360726210000, End: 1680066458000000000, - Step: 60, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateOperator: v3.AggregateOperatorNoOp, @@ -1150,7 +1146,7 @@ var testBuildTracesQueryData = []struct { func TestBuildTracesQuery(t *testing.T) { for _, tt := range testBuildTracesQueryData { Convey("TestBuildTracesQuery", t, func() { - query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{ + query, err := buildTracesQuery(tt.Start, tt.End, tt.BuilderQuery.StepInterval, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{ "name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, }, tt.PanelType, tt.Options) So(err, ShouldBeNil) @@ -1172,8 +1168,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit- first", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, @@ -1189,7 +1185,7 @@ var testPrepTracesQueryData = []struct { }, ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," + " toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2" + - " where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND" + + " where (timestamp >= '1680066360000000000' AND timestamp <= '1680066420000000000') AND" + " stringTagMap['method'] = 'GET' AND has(stringTagMap, 'method') group by `method` order by value DESC) LIMIT 10", Keys: map[string]v3.AttributeKey{"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, Options: Options{ @@ -1199,8 +1195,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit- first - with order by value", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, @@ -1217,8 +1213,8 @@ var testPrepTracesQueryData = []struct { }, ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," + " toFloat64(count(distinct(stringTagMap['name']))) as value from " + - "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + - " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + + " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') group by `method` order by value ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, Options: Options{ @@ -1228,8 +1224,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit- first - with order by attribute", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -1243,8 +1239,8 @@ var testPrepTracesQueryData = []struct { }, ExpectedQuery: "SELECT `serviceName` from (SELECT serviceName as `serviceName`," + " toFloat64(count(distinct(serviceName))) as value from " + - "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + - " AND timestamp <= '1680066458000000000') " + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + + " AND timestamp <= '1680066420000000000') " + "group by `serviceName` order by `serviceName` ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, Options: Options{ @@ -1254,8 +1250,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit- first - with 2 group by and 2 order by", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, @@ -1273,8 +1269,8 @@ var testPrepTracesQueryData = []struct { ExpectedQuery: "SELECT `serviceName`,`http.method` from (SELECT serviceName as `serviceName`," + " stringTagMap['http.method'] as `http.method`," + " toFloat64(count(distinct(serviceName))) as value from " + - "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + - " AND timestamp <= '1680066458000000000') AND has(stringTagMap, 'http.method') " + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + + " AND timestamp <= '1680066420000000000') AND has(stringTagMap, 'http.method') " + "group by `serviceName`,`http.method` order by `serviceName` ASC,value ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, Options: Options{ @@ -1284,8 +1280,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit- second", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, @@ -1301,8 +1297,8 @@ var testPrepTracesQueryData = []struct { }, ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + "stringTagMap['method'] as `method`, toFloat64(count(distinct(stringTagMap['name'])))" + - " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + - " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + + " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by value DESC", Keys: map[string]v3.AttributeKey{}, Options: Options{ @@ -1312,8 +1308,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit- second - with order by", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, @@ -1330,8 +1326,8 @@ var testPrepTracesQueryData = []struct { }, ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + "stringTagMap['method'] as `method`, toFloat64(count(distinct(stringTagMap['name'])))" + - " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + - " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + + " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by `method` ASC", Keys: map[string]v3.AttributeKey{}, Options: Options{ GraphLimitQtype: constants.SecondQueryGraphLimit, @@ -1340,8 +1336,8 @@ var testPrepTracesQueryData = []struct { { Name: "Test TS with limit - second - with two group by and two order by", PanelType: v3.PanelTypeGraph, - Start: 1680066360726210000, - End: 1680066458000000000, + Start: 1680066360726, + End: 1680066458000, BuilderQuery: &v3.BuilderQuery{ QueryName: "A", AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, @@ -1362,8 +1358,8 @@ var testPrepTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + "stringTagMap['method'] as `method`, stringTagMap['name'] as `name`," + " toFloat64(count(distinct(stringTagMap['name'])))" + - " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + - " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + + " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND has(stringTagMap, 'name') " + "AND (`method`,`name`) GLOBAL IN (%s) group by `method`,`name`,ts " + "order by `method` ASC,`name` ASC", diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 0cf4e2ed68..55242199b3 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -324,7 +324,7 @@ type FilterAttributeValueResponse struct { type QueryRangeParamsV3 struct { Start int64 `json:"start"` End int64 `json:"end"` - Step int64 `json:"step"` + Step int64 `json:"step"` // step is in seconds; used for prometheus queries CompositeQuery *CompositeQuery `json:"compositeQuery"` Variables map[string]interface{} `json:"variables,omitempty"` NoCache bool `json:"noCache"`