diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index cf7ed5503c..7185581ab7 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -4223,8 +4223,11 @@ func (r *ClickHouseReader) GetListResultV3(ctx context.Context, query string) ([ for idx, v := range vars { if columnNames[idx] == "timestamp" { t = time.Unix(0, int64(*v.(*uint64))) + } else if columnNames[idx] == "timestamp_datetime" { + t = *v.(*time.Time) + } else { + row[columnNames[idx]] = v } - row[columnNames[idx]] = v } rowList = append(rowList, &v3.Row{Timestamp: t, Data: row}) } diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 3025000db3..604b1f046a 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -2684,6 +2684,11 @@ func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.Qu if err != nil { return nil, err } + // Add timestamp as a span key to allow ordering by timestamp + spanKeys["timestamp"] = v3.AttributeKey{ + Key: "timestamp", + IsColumn: true, + } return spanKeys, nil } } @@ -2725,7 +2730,7 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que return } - if queryRangeParams.CompositeQuery.PanelType == v3.PanelTypeList { + if queryRangeParams.CompositeQuery.PanelType == v3.PanelTypeList || queryRangeParams.CompositeQuery.PanelType == v3.PanelTypeTrace { result, err, errQuriesByName = aH.execClickHouseListQueries(r.Context(), queries) } else { result, err, errQuriesByName = aH.execClickHouseGraphQueries(r.Context(), queries) diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index d71f802c8c..abf38b6b40 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -95,7 +95,7 @@ func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) } // getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator -func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) { +func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey, keys map[string]v3.AttributeKey) string { var selectLabels string if aggregatorOperator == v3.AggregateOperatorNoOp { selectLabels = "" @@ -105,7 +105,16 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri selectLabels += fmt.Sprintf(", %s as `%s`", filterName, tag.Key) } } - return selectLabels, nil + return selectLabels +} + +func getSelectColumns(sc []v3.AttributeKey, keys map[string]v3.AttributeKey) string { + var columns []string + for _, tag := range sc { + columnName := getColumnName(tag, keys) + columns = append(columns, fmt.Sprintf("%s as `%s` ", columnName, tag.Key)) + } + return strings.Join(columns, ",") } // getZerosForEpochNano returns the number of zeros to be appended to the epoch time for converting it to nanoseconds @@ -208,7 +217,7 @@ func handleEmptyValuesInGroupBy(keys map[string]v3.AttributeKey, groupBy []v3.At return "", nil } -func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey) (string, error) { +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey, panelType v3.PanelType) (string, error) { filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys) if err != nil { @@ -217,10 +226,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str // timerange will be sent in epoch millisecond spanIndexTableTimeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d')", start*getZerosForEpochNano(start), end*getZerosForEpochNano(end)) - selectLabels, err := getSelectLabels(mq.AggregateOperator, mq.GroupBy, keys) - if err != nil { - return "", err - } + selectLabels := getSelectLabels(mq.AggregateOperator, mq.GroupBy, keys) having := having(mq.Having) if having != "" { @@ -234,7 +240,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where " + spanIndexTableTimeFilter + "%s " + "group by %s%s " + - "order by %sts" + "order by %s" emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy) if err != nil { @@ -243,7 +249,8 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str filterSubQuery += emptyValuesInGroupByFilter groupBy := groupByAttributeKeyTags(keys, mq.GroupBy...) - orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) + enrichedOrderBy := enrichOrderBy(mq.OrderBy, keys) + orderBy := orderByAttributeKeyTags(panelType, enrichedOrderBy, mq.GroupBy, keys) aggregationKey := "" if mq.AggregateAttribute.Key != "" { @@ -297,15 +304,48 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorNoOp: - // queryTmpl := constants.TracesSQLSelect + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s" - // query := fmt.Sprintf(queryTmpl, spanIndexTableTimeFilter, filterSubQuery) - // return query, nil - return "", fmt.Errorf("not implemented, part of traces page") + var query string + if panelType == v3.PanelTypeTrace { + withSubQuery := fmt.Sprintf(constants.TracesExplorerViewSQLSelectWithSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME, spanIndexTableTimeFilter, filterSubQuery) + withSubQuery = addLimitToQuery(withSubQuery, mq.Limit, panelType) + if mq.Offset != 0 { + withSubQuery = addOffsetToQuery(withSubQuery, mq.Offset) + } + query = withSubQuery + ") " + fmt.Sprintf(constants.TracesExplorerViewSQLSelectQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME) + } else if panelType == v3.PanelTypeList { + if len(mq.SelectColumns) == 0 { + return "", fmt.Errorf("select columns cannot be empty for panelType %s", panelType) + } + selectColumns := getSelectColumns(mq.SelectColumns, keys) + queryNoOpTmpl := fmt.Sprintf("SELECT timestamp as timestamp_datetime, spanID, traceID, "+"%s ", selectColumns) + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s" + " order by %s" + query = fmt.Sprintf(queryNoOpTmpl, spanIndexTableTimeFilter, filterSubQuery, orderBy) + } else { + return "", fmt.Errorf("unsupported aggregate operator %s for panelType %s", mq.AggregateOperator, panelType) + } + return query, nil default: - return "", fmt.Errorf("unsupported aggregate operator") + return "", fmt.Errorf("unsupported aggregate operator %s", mq.AggregateOperator) } } +func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.OrderBy { + enrichedItems := []v3.OrderBy{} + for i := 0; i < len(items); i++ { + attributeKey := enrichKeyWithMetadata(v3.AttributeKey{ + Key: items[i].ColumnName, + }, keys) + enrichedItems = append(enrichedItems, v3.OrderBy{ + ColumnName: items[i].ColumnName, + Order: items[i].Order, + Key: attributeKey.Key, + DataType: attributeKey.DataType, + Type: attributeKey.Type, + IsColumn: attributeKey.IsColumn, + }) + } + return enrichedItems +} + // groupBy returns a string of comma separated tags for group by clause // `ts` is always added to the group by clause func groupBy(tags ...string) string { @@ -322,41 +362,66 @@ func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.Attribu } // orderBy returns a string of comma separated tags for order by clause +// if there are remaining items which are not present in tags they are also added // if the order is not specified, it defaults to ASC -func orderBy(items []v3.OrderBy, tags []string) string { +func orderBy(panelType v3.PanelType, items []v3.OrderBy, tags []string, keys map[string]v3.AttributeKey) []string { var orderBy []string + + // create a lookup + addedToOrderBy := map[string]bool{} + itemsLookup := map[string]v3.OrderBy{} + + for i := 0; i < len(items); i++ { + addedToOrderBy[items[i].ColumnName] = false + itemsLookup[items[i].ColumnName] = items[i] + } + for _, tag := range tags { - found := false - for _, item := range items { - if item.ColumnName == tag { - found = true - orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) - break - } - } - if !found { + if item, ok := itemsLookup[tag]; ok { + orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) + addedToOrderBy[item.ColumnName] = true + } else { orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag)) } } - // users might want to order by value of aggreagation + // users might want to order by value of aggregation for _, item := range items { if item.ColumnName == constants.SigNozOrderByValue { orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order)) + addedToOrderBy[item.ColumnName] = true } } - return strings.Join(orderBy, ",") + + // add the remaining items + if panelType == v3.PanelTypeList { + for _, item := range items { + // since these are not present in tags we will have to select them correctly + // for list view there is no need to check if it was added since they wont be added yet but this is just for safety + if !addedToOrderBy[item.ColumnName] { + attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} + name := getColumnName(attr, keys) + orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) + } + } + } + return orderBy } -func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string { +func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags []v3.AttributeKey, keys map[string]v3.AttributeKey) string { var groupTags []string for _, tag := range tags { groupTags = append(groupTags, tag.Key) } - str := orderBy(items, groupTags) - if len(str) > 0 { - str = str + "," + orderByArray := orderBy(panelType, items, groupTags, keys) + + if panelType == v3.PanelTypeList && len(orderByArray) == 0 { + orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC") + } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeTable { + orderByArray = append(orderByArray, "ts") } + + str := strings.Join(orderByArray, ",") return str } @@ -393,10 +458,7 @@ func addLimitToQuery(query string, limit uint64, panelType v3.PanelType) string if limit == 0 { limit = 100 } - if panelType == v3.PanelTypeList { - return fmt.Sprintf("%s LIMIT %d", query, limit) - } - return query + return fmt.Sprintf("%s LIMIT %d", query, limit) } func addOffsetToQuery(query string, offset uint64) string { @@ -404,17 +466,19 @@ func addOffsetToQuery(query string, offset uint64) string { } func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) { - query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys) + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType) if err != nil { return "", err } if panelType == v3.PanelTypeValue { query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) } - query = addLimitToQuery(query, mq.Limit, panelType) + if panelType == v3.PanelTypeList { + query = addLimitToQuery(query, mq.Limit, panelType) - if mq.Offset != 0 { - query = addOffsetToQuery(query, mq.Offset) + if mq.Offset != 0 { + query = addOffsetToQuery(query, mq.Offset) + } } return query, err } diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 9cda548b9e..97c739d8c1 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -244,13 +244,51 @@ var testGetSelectLabelsData = []struct { func TestGetSelectLabels(t *testing.T) { for _, tt := range testGetSelectLabelsData { Convey("testGetSelectLabelsData", t, func() { - selectLabels, err := getSelectLabels(tt.AggregateOperator, tt.GroupByTags, map[string]v3.AttributeKey{}) - So(err, ShouldBeNil) + selectLabels := getSelectLabels(tt.AggregateOperator, tt.GroupByTags, map[string]v3.AttributeKey{}) So(selectLabels, ShouldEqual, tt.SelectLabels) }) } } +var testGetSelectColumnsData = []struct { + Name string + sc []v3.AttributeKey + SelectColumns string +}{ + { + Name: "select columns attribute", + sc: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + SelectColumns: "stringTagMap['user.name'] as `user.name` ", + }, + { + Name: "select columns resource", + sc: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}}, + SelectColumns: "resourceTagsMap['user.name'] as `user.name` ", + }, + { + Name: "select columns attribute and resource", + sc: []v3.AttributeKey{ + {Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + {Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + SelectColumns: "resourceTagsMap['user.name'] as `user.name` ,stringTagMap['host'] as `host` ", + }, + { + Name: "select columns fixed column", + sc: []v3.AttributeKey{{Key: "host", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + SelectColumns: "host as `host` ", + }, +} + +func TestGetSelectColumns(t *testing.T) { + for _, tt := range testGetSelectColumnsData { + Convey("testGetSelectColumnsData", t, func() { + selectColumns := getSelectColumns(tt.sc, map[string]v3.AttributeKey{}) + So(selectColumns, ShouldEqual, tt.SelectColumns) + }) + } +} + var testGetZerosForEpochNanoData = []struct { Name string Epoch int64 @@ -282,13 +320,15 @@ func TestGetZerosForEpochNano(t *testing.T) { } var testOrderBy = []struct { - Name string - Items []v3.OrderBy - Tags []string - Result string + Name string + PanelType v3.PanelType + Items []v3.OrderBy + Tags []string + Result []string }{ { - Name: "Test 1", + Name: "Test 1", + PanelType: v3.PanelTypeGraph, Items: []v3.OrderBy{ { ColumnName: "name", @@ -300,10 +340,11 @@ var testOrderBy = []struct { }, }, Tags: []string{"name"}, - Result: "name asc,value desc", + Result: []string{"name asc", "value desc"}, }, { - Name: "Test 2", + Name: "Test 2", + PanelType: v3.PanelTypeList, Items: []v3.OrderBy{ { ColumnName: "name", @@ -315,10 +356,11 @@ var testOrderBy = []struct { }, }, Tags: []string{"name", "bytes"}, - Result: "name asc,bytes asc", + Result: []string{"name asc", "bytes asc"}, }, { - Name: "Test 3", + Name: "Test 3", + PanelType: v3.PanelTypeList, Items: []v3.OrderBy{ { ColumnName: "name", @@ -334,15 +376,62 @@ var testOrderBy = []struct { }, }, Tags: []string{"name", "bytes"}, - Result: "name asc,bytes asc,value asc", + Result: []string{"name asc", "bytes asc", "value asc"}, + }, + { + Name: "Test 4", + PanelType: v3.PanelTypeList, + Items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: "bytes", + Order: "asc", + }, + { + ColumnName: "response_time", + Order: "desc", + Key: "response_time", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + }, + Tags: []string{"name", "bytes"}, + Result: []string{"name asc", "bytes asc", "stringTagMap['response_time'] desc"}, + }, + { + Name: "Test 4", + PanelType: v3.PanelTypeList, + Items: []v3.OrderBy{ + { + ColumnName: "name", + Order: "asc", + }, + { + ColumnName: "bytes", + Order: "asc", + }, + { + ColumnName: "response_time", + Order: "desc", + }, + }, + Tags: []string{}, + Result: []string{"name asc", "bytes asc", "stringTagMap['response_time'] desc"}, }, } func TestOrderBy(t *testing.T) { for _, tt := range testOrderBy { Convey("testOrderBy", t, func() { - res := orderBy(tt.Items, tt.Tags) - So(res, ShouldEqual, tt.Result) + res := orderBy(tt.PanelType, tt.Items, tt.Tags, map[string]v3.AttributeKey{ + "name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + "bytes": {Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + "response_time": {Key: "response_time", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + }) + So(res, ShouldResemble, tt.Result) }) } } @@ -357,6 +446,7 @@ var testBuildTracesQueryData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string + PanelType v3.PanelType }{ { Name: "Test aggregate count on fixed column of float64 type", @@ -373,6 +463,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate rate without aggregate attribute", @@ -388,6 +479,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, count()/60 as value from" + " signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <=" + " '1680066458000000000') group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count on fixed column of float64 type with filter", @@ -406,6 +498,7 @@ var testBuildTracesQueryData = []struct { " toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2" + " where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND stringTagMap['customer_id'] = '10001' group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count on fixed column of bool type", @@ -422,6 +515,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count on a attribute", @@ -438,6 +532,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND has(stringTagMap, 'user_name') group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count on a fixed column of string type", @@ -454,6 +549,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND name != '' group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count with filter", @@ -473,6 +569,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count distinct and order by value", @@ -490,6 +587,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " group by ts order by value ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count distinct on string key", @@ -506,6 +604,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name'])))" + " as value from signoz_traces.distributed_signoz_index_v2 where" + " (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count distinct with filter and groupBy", @@ -533,6 +632,7 @@ var testBuildTracesQueryData = []struct { "AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + "AND has(stringTagMap, 'http.method') group by http.method,ts " + "order by http.method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate count with multiple filter,groupBy and orderBy", @@ -564,6 +664,7 @@ var testBuildTracesQueryData = []struct { "AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + "AND has(stringTagMap, 'method') AND has(resourceTagsMap, 'x') group by method,x,ts " + "order by method ASC,x ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate avg", @@ -591,6 +692,7 @@ var testBuildTracesQueryData = []struct { "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate sum", @@ -618,6 +720,7 @@ var testBuildTracesQueryData = []struct { "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate min", @@ -645,6 +748,7 @@ var testBuildTracesQueryData = []struct { "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate max", @@ -672,6 +776,7 @@ var testBuildTracesQueryData = []struct { "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate PXX", @@ -695,6 +800,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate RateSum", @@ -715,6 +821,7 @@ var testBuildTracesQueryData = []struct { ", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND has(stringTagMap, 'method') group by method,ts order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate rate", @@ -736,6 +843,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate RateSum without fixed column", @@ -758,6 +866,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND has(stringTagMap, 'method') group by method,ts " + "order by method ASC,ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test aggregate with having clause", @@ -781,6 +890,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " group by ts having value > 10 order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test count aggregate with having clause and filters", @@ -808,6 +918,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from " + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts", + PanelType: v3.PanelTypeGraph, }, { Name: "Test count distinct aggregate with having clause and filters", @@ -835,32 +946,104 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts", + PanelType: v3.PanelTypeGraph, + }, + { + Name: "Test Noop list view", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + SelectColumns: []v3.AttributeKey{ + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + }, + ExpectedQuery: "SELECT timestamp as timestamp_datetime, spanID, traceID," + + " name as `name` from signoz_traces.distributed_signoz_index_v2 where " + + "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') order by timestamp DESC", + PanelType: v3.PanelTypeList, + }, + { + Name: "Test Noop list view with order by", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + SelectColumns: []v3.AttributeKey{ + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, + OrderBy: []v3.OrderBy{{ColumnName: "name", Order: "ASC"}}, + }, + ExpectedQuery: "SELECT timestamp as timestamp_datetime, spanID, traceID," + + " name as `name` from signoz_traces.distributed_signoz_index_v2 where " + + "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') order by name ASC", + PanelType: v3.PanelTypeList, + }, + { + Name: "Test Noop list view with order by and filter", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + SelectColumns: []v3.AttributeKey{ + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }}, + OrderBy: []v3.OrderBy{{ColumnName: "name", Order: "ASC"}}, + }, + ExpectedQuery: "SELECT timestamp as timestamp_datetime, spanID, traceID," + + " name as `name` from signoz_traces.distributed_signoz_index_v2 where " + + "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + + " AND stringTagMap['method'] = 'GET' order by name ASC", + PanelType: v3.PanelTypeList, + }, + { + Name: "Test Noop trace view", + Start: 1680066360726210000, + End: 1680066458000000000, + Step: 60, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateOperator: v3.AggregateOperatorNoOp, + Expression: "A", + Filters: &v3.FilterSet{ + Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + }, + ExpectedQuery: "WITH subQuery AS (SELECT distinct on (traceID) traceID, durationNano, serviceName," + + " name FROM signoz_traces.distributed_signoz_index_v2 WHERE parentSpanID = '' AND (timestamp >= '1680066360726210000' AND " + + "timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' ORDER BY durationNano DESC LIMIT 100)" + + " SELECT subQuery.serviceName, subQuery.name, count() AS span_count, subQuery.durationNano, traceID" + + " FROM signoz_traces.distributed_signoz_index_v2 INNER JOIN subQuery ON distributed_signoz_index_v2.traceID" + + " = subQuery.traceID GROUP BY traceID, subQuery.durationNano, subQuery.name, subQuery.serviceName " + + "ORDER BY subQuery.durationNano desc;", + PanelType: v3.PanelTypeTrace, }, - // { - // Name: "Test Noop", - // Start: 1680066360726210000, - // End: 1680066458000000000, - // Step: 60, - // BuilderQuery: &v3.BuilderQuery{ - // SelectColumns: []v3.AttributeKey{}, - // QueryName: "A", - // AggregateOperator: v3.AggregateOperatorNoOp, - // Expression: "A", - // Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, - // // GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - // // OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, - // }, - // ExpectedQuery: "", - // }, } func TestBuildTracesQuery(t *testing.T) { for _, tt := range testBuildTracesQueryData { Convey("TestBuildTracesQuery", t, func() { - query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{}) + query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{ + "name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, tt.PanelType) So(err, ShouldBeNil) So(query, ShouldEqual, tt.ExpectedQuery) - }) } } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index d86d6df205..31d02b19f6 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -236,6 +236,11 @@ const ( "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64," + "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64," + "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string " + TracesExplorerViewSQLSelectWithSubQuery = "WITH subQuery AS (SELECT distinct on (traceID) traceID, durationNano, " + + "serviceName, name FROM %s.%s WHERE parentSpanID = '' AND %s %s ORDER BY durationNano DESC " + TracesExplorerViewSQLSelectQuery = "SELECT subQuery.serviceName, subQuery.name, count() AS " + + "span_count, subQuery.durationNano, traceID FROM %s.%s INNER JOIN subQuery ON %s.traceID = subQuery.traceID GROUP " + + "BY traceID, subQuery.durationNano, subQuery.name, subQuery.serviceName ORDER BY subQuery.durationNano desc;" ) // ReservedColumnTargetAliases identifies result value from a user diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 7d3028b3d8..57f290f133 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -178,11 +178,12 @@ const ( PanelTypeGraph PanelType = "graph" PanelTypeTable PanelType = "table" PanelTypeList PanelType = "list" + PanelTypeTrace PanelType = "trace" ) func (p PanelType) Validate() error { switch p { - case PanelTypeValue, PanelTypeGraph, PanelTypeTable, PanelTypeList: + case PanelTypeValue, PanelTypeGraph, PanelTypeTable, PanelTypeList, PanelTypeTrace: return nil default: return fmt.Errorf("invalid panel type: %s", p)