GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-18 02:15:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: enable limit on ts (traces) (#3157)

Browse Source 
* feat: enable limit on ts
This commit is contained in:
Vishal Sharma 2023-07-19 09:54:27 +05:30 committed by GitHub
parent 7822b4efee
commit 5a2a987a9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 365 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/query-service/app/querier/querier.go
View File

@ -250,7 +250,7 @@ func (q *querier) runBuilderQueries(ctx context.Context, params *v3.QueryRangePa
}
if builderQuery.DataSource == v3.DataSourceTraces {
query, err := tracesV3.PrepareTracesQuery(params.Start, params.End, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, keys)
query, err := tracesV3.PrepareTracesQuery(params.Start, params.End, params.CompositeQuery.PanelType, builderQuery, keys, "")
if err != nil {
errQueriesByName[queryName] = err.Error()
continue

24
pkg/query-service/app/queryBuilder/query_builder.go
View File

@ -39,7 +39,7 @@ var SupportedFunctions = []string{
var EvalFuncs = map[string]govaluate.ExpressionFunction{}
type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error)
type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey, graphLimitQtype string) (string, error)
type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, graphLimitQtype string) (string, error)
type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error)
@ -147,11 +147,25 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3, args ...in
if len(args) > 0 {
keys = args[0].(map[string]v3.AttributeKey)
}
queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, keys)
if err != nil {
return nil, err
// for ts query with group by and limit form two queries
if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 {
limitQuery, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.PanelType, query, keys, constants.FirstQueryGraphLimit)
if err != nil {
return nil, err
}
placeholderQuery, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.PanelType, query, keys, constants.SecondQueryGraphLimit)
if err != nil {
return nil, err
}
query := fmt.Sprintf(placeholderQuery, limitQuery)
queries[queryName] = query
} else {
queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.PanelType, query, keys, "")
if err != nil {
return nil, err
}
queries[queryName] = queryString
}
queries[queryName] = queryString
case v3.DataSourceLogs:
// for ts query with limit replace it as it is already formed
if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 {

157
pkg/query-service/app/traces/v3/query_builder.go
View File

@ -108,6 +108,18 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri
return selectLabels
}
func getSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string {
var selectLabels []string
if aggregatorOperator == v3.AggregateOperatorNoOp {
return ""
} else {
for _, tag := range groupBy {
selectLabels = append(selectLabels, fmt.Sprintf("`%s`", tag.Key))
}
}
return strings.Join(selectLabels, ",")
}
func getSelectColumns(sc []v3.AttributeKey, keys map[string]v3.AttributeKey) string {
var columns []string
for _, tag := range sc {
@ -219,7 +231,7 @@ func handleEmptyValuesInGroupBy(keys map[string]v3.AttributeKey, groupBy []v3.At
return "", nil
}
func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey, panelType v3.PanelType) (string, error) {
func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey, panelType v3.PanelType, graphLimitQtype string) (string, error) {
filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys)
if err != nil {
@ -236,24 +248,27 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
}
var queryTmpl string
if panelType == v3.PanelTypeTable {
if graphLimitQtype == constants.FirstQueryGraphLimit {
queryTmpl = "SELECT"
} else if panelType == v3.PanelTypeTable {
queryTmpl =
"SELECT now() as ts," + selectLabels +
" %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s" +
"%s%s" +
"%s"
"SELECT now() as ts,"
} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
// Select the aggregate value for interval
queryTmpl =
fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts,", step) + selectLabels +
" %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s" +
"%s%s" +
"%s"
fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts,", step)
}
queryTmpl = queryTmpl + selectLabels +
" %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s" +
"%s%s" +
"%s"
// we don't need value for first query
if graphLimitQtype == constants.FirstQueryGraphLimit {
queryTmpl = "SELECT " + getSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")"
}
emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy)
@ -262,7 +277,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
}
filterSubQuery += emptyValuesInGroupByFilter
groupBy := groupByAttributeKeyTags(panelType, mq.GroupBy...)
groupBy := groupByAttributeKeyTags(panelType, graphLimitQtype, mq.GroupBy...)
if groupBy != "" {
groupBy = " group by " + groupBy
}
@ -271,6 +286,11 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
if orderBy != "" {
orderBy = " order by " + orderBy
}
if graphLimitQtype == constants.SecondQueryGraphLimit {
filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", getSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)"
}
aggregationKey := ""
if mq.AggregateAttribute.Key != "" {
aggregationKey = getColumnName(mq.AggregateAttribute, keys)
@ -326,7 +346,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
var query string
if panelType == v3.PanelTypeTrace {
withSubQuery := fmt.Sprintf(constants.TracesExplorerViewSQLSelectWithSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME, spanIndexTableTimeFilter, filterSubQuery)
withSubQuery = addLimitToQuery(withSubQuery, mq.Limit, panelType)
withSubQuery = addLimitToQuery(withSubQuery, mq.Limit)
if mq.Offset != 0 {
withSubQuery = addOffsetToQuery(withSubQuery, mq.Offset)
}
@ -367,84 +387,60 @@ func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.Ord
// groupBy returns a string of comma separated tags for group by clause
// `ts` is always added to the group by clause
func groupBy(panelType v3.PanelType, tags ...string) string {
if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
func groupBy(panelType v3.PanelType, graphLimitQtype string, tags ...string) string {
if (graphLimitQtype != constants.FirstQueryGraphLimit) && (panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue) {
tags = append(tags, "ts")
}
return strings.Join(tags, ",")
}
func groupByAttributeKeyTags(panelType v3.PanelType, tags ...v3.AttributeKey) string {
func groupByAttributeKeyTags(panelType v3.PanelType, graphLimitQtype string, tags ...v3.AttributeKey) string {
groupTags := []string{}
for _, tag := range tags {
groupTags = append(groupTags, fmt.Sprintf("`%s`", tag.Key))
}
return groupBy(panelType, groupTags...)
return groupBy(panelType, graphLimitQtype, groupTags...)
}
// orderBy returns a string of comma separated tags for order by clause
// if there are remaining items which are not present in tags they are also added
// if the order is not specified, it defaults to ASC
func orderBy(panelType v3.PanelType, items []v3.OrderBy, tags []string, keys map[string]v3.AttributeKey) []string {
func orderBy(panelType v3.PanelType, items []v3.OrderBy, tagLookup map[string]struct{}, keys map[string]v3.AttributeKey) []string {
var orderBy []string
// create a lookup
addedToOrderBy := map[string]bool{}
itemsLookup := map[string]v3.OrderBy{}
for i := 0; i < len(items); i++ {
addedToOrderBy[items[i].ColumnName] = false
itemsLookup[items[i].ColumnName] = items[i]
}
for _, tag := range tags {
if item, ok := itemsLookup[tag]; ok {
orderBy = append(orderBy, fmt.Sprintf("`%s` %s", item.ColumnName, item.Order))
addedToOrderBy[item.ColumnName] = true
} else {
orderBy = append(orderBy, fmt.Sprintf("`%s` ASC", tag))
}
}
// users might want to order by value of aggregation
for _, item := range items {
if item.ColumnName == constants.SigNozOrderByValue {
orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order))
addedToOrderBy[item.ColumnName] = true
}
}
// add the remaining items
if panelType == v3.PanelTypeList {
for _, item := range items {
// since these are not present in tags we will have to select them correctly
// for list view there is no need to check if it was added since they wont be added yet but this is just for safety
if !addedToOrderBy[item.ColumnName] {
attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn}
name := getColumnName(attr, keys)
if item.IsColumn {
orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order))
} else {
orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order))
}
} else if _, ok := tagLookup[item.ColumnName]; ok {
orderBy = append(orderBy, fmt.Sprintf("`%s` %s", item.ColumnName, item.Order))
} else if panelType == v3.PanelTypeList {
attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn}
name := getColumnName(attr, keys)
if item.IsColumn {
orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order))
} else {
orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order))
}
}
}
return orderBy
}
func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags []v3.AttributeKey, keys map[string]v3.AttributeKey) string {
var groupTags []string
for _, tag := range tags {
groupTags = append(groupTags, tag.Key)
tagLookup := map[string]struct{}{}
for _, v := range tags {
tagLookup[v.Key] = struct{}{}
}
orderByArray := orderBy(panelType, items, groupTags, keys)
if panelType == v3.PanelTypeList && len(orderByArray) == 0 {
orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC")
} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
orderByArray = append(orderByArray, "ts")
orderByArray := orderBy(panelType, items, tagLookup, keys)
if len(orderByArray) == 0 {
if panelType == v3.PanelTypeList {
orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC")
} else if panelType == v3.PanelTypeGraph {
orderByArray = append(orderByArray, "value DESC")
}
}
str := strings.Join(orderByArray, ",")
@ -480,7 +476,7 @@ func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator
return query, nil
}
func addLimitToQuery(query string, limit uint64, panelType v3.PanelType) string {
func addLimitToQuery(query string, limit uint64) string {
if limit == 0 {
limit = 100
}
@ -491,16 +487,33 @@ func addOffsetToQuery(query string, offset uint64) string {
return fmt.Sprintf("%s OFFSET %d", query, offset)
}
func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) {
query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType)
func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey, graphLimitQtype string) (string, error) {
if graphLimitQtype == constants.FirstQueryGraphLimit {
// give me just the group by names
query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, graphLimitQtype)
if err != nil {
return "", err
}
query = addLimitToQuery(query, mq.Limit)
return query, nil
} else if graphLimitQtype == constants.SecondQueryGraphLimit {
query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, graphLimitQtype)
if err != nil {
return "", err
}
return query, nil
}
query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, graphLimitQtype)
if err != nil {
return "", err
}
if panelType == v3.PanelTypeValue {
query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator)
}
if panelType == v3.PanelTypeList {
query = addLimitToQuery(query, mq.Limit, panelType)
if panelType == v3.PanelTypeList || panelType == v3.PanelTypeTable {
query = addLimitToQuery(query, mq.Limit)
if mq.Offset != 0 {
query = addOffsetToQuery(query, mq.Offset)

298
pkg/query-service/app/traces/v3/query_builder_test.go
View File

@ -323,8 +323,8 @@ var testOrderBy = []struct {
Name string
PanelType v3.PanelType
Items []v3.OrderBy
Tags []string
Result []string
Tags []v3.AttributeKey
Result string
}{
{
Name: "Test 1",
@ -339,8 +339,10 @@ var testOrderBy = []struct {
Order: "desc",
},
},
Tags: []string{"name"},
Result: []string{"`name` asc", "value desc"},
Tags: []v3.AttributeKey{
{Key: "name"},
},
Result: "`name` asc,value desc",
},
{
Name: "Test 2",
@ -355,8 +357,11 @@ var testOrderBy = []struct {
Order: "asc",
},
},
Tags: []string{"name", "bytes"},
Result: []string{"`name` asc", "`bytes` asc"},
Tags: []v3.AttributeKey{
{Key: "name"},
{Key: "bytes"},
},
Result: "`name` asc,`bytes` asc",
},
{
Name: "Test 3",
@ -375,8 +380,11 @@ var testOrderBy = []struct {
Order: "asc",
},
},
Tags: []string{"name", "bytes"},
Result: []string{"`name` asc", "`bytes` asc", "value asc"},
Tags: []v3.AttributeKey{
{Key: "name"},
{Key: "bytes"},
},
Result: "`name` asc,value asc,`bytes` asc",
},
{
Name: "Test 4",
@ -398,8 +406,11 @@ var testOrderBy = []struct {
DataType: v3.AttributeKeyDataTypeString,
},
},
Tags: []string{"name", "bytes"},
Result: []string{"`name` asc", "`bytes` asc", "stringTagMap['response_time'] desc"},
Tags: []v3.AttributeKey{
{Key: "name"},
{Key: "bytes"},
},
Result: "`name` asc,`bytes` asc,stringTagMap['response_time'] desc",
},
{
Name: "Test 5",
@ -426,15 +437,15 @@ var testOrderBy = []struct {
Order: "desc",
},
},
Tags: []string{},
Result: []string{"`name` asc", "`bytes` asc", "stringTagMap['response_time'] desc"},
Tags: []v3.AttributeKey{},
Result: "`name` asc,`bytes` asc,stringTagMap['response_time'] desc",
},
}
func TestOrderBy(t *testing.T) {
for _, tt := range testOrderBy {
Convey("testOrderBy", t, func() {
res := orderBy(tt.PanelType, tt.Items, tt.Tags, map[string]v3.AttributeKey{
res := orderByAttributeKeyTags(tt.PanelType, tt.Items, tt.Tags, map[string]v3.AttributeKey{
"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
"bytes": {Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
"response_time": {Key: "response_time", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false},
@ -470,7 +481,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts order by ts",
" group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -486,7 +497,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, count()/60 as value from" +
" signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <=" +
" '1680066458000000000') group by ts order by ts",
" '1680066458000000000') group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -505,7 +516,7 @@ var testBuildTracesQueryData = []struct {
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2" +
" where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND stringTagMap['customer_id'] = '10001' group by ts order by ts",
" AND stringTagMap['customer_id'] = '10001' group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -522,7 +533,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts order by ts",
" group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -539,7 +550,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND has(stringTagMap, 'user_name') group by ts order by ts",
" AND has(stringTagMap, 'user_name') group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -556,7 +567,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND name != '' group by ts order by ts",
" AND name != '' group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -576,7 +587,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by ts",
" AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -594,7 +605,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts order by value ASC,ts",
" group by ts order by value ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -611,7 +622,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name'])))" +
" as value from signoz_traces.distributed_signoz_index_v2 where" +
" (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts",
" (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -630,7 +641,7 @@ var testBuildTracesQueryData = []struct {
},
},
GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "http.method", Order: "ASC"}, {ColumnName: "ts", Order: "ASC"}},
OrderBy: []v3.OrderBy{{ColumnName: "http.method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
@ -639,7 +650,7 @@ var testBuildTracesQueryData = []struct {
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " +
"AND has(stringTagMap, 'http.method') group by `http.method`,ts " +
"order by `http.method` ASC,ts",
"order by `http.method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -671,7 +682,7 @@ var testBuildTracesQueryData = []struct {
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " +
"AND has(stringTagMap, 'method') AND has(resourceTagsMap, 'x') group by `method`,`x`,ts " +
"order by `method` ASC,`x` ASC,ts",
"order by `method` ASC,`x` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -699,7 +710,7 @@ var testBuildTracesQueryData = []struct {
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -727,7 +738,7 @@ var testBuildTracesQueryData = []struct {
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -755,7 +766,7 @@ var testBuildTracesQueryData = []struct {
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -783,7 +794,7 @@ var testBuildTracesQueryData = []struct {
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -807,7 +818,7 @@ var testBuildTracesQueryData = []struct {
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -828,7 +839,7 @@ var testBuildTracesQueryData = []struct {
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" +
", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND has(stringTagMap, 'method') group by `method`,ts order by `method` ASC,ts",
" AND has(stringTagMap, 'method') group by `method`,ts order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -850,7 +861,7 @@ var testBuildTracesQueryData = []struct {
", count(numberTagMap['bytes'])/60 as value " +
"from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -873,7 +884,7 @@ var testBuildTracesQueryData = []struct {
"sum(numberTagMap['bytes'])/60 as value " +
"from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by `method`,ts " +
"order by `method` ASC,ts",
"order by `method` ASC",
PanelType: v3.PanelTypeGraph,
},
{
@ -897,7 +908,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts having value > 10 order by ts",
" group by ts having value > 10 order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -925,7 +936,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from " +
"signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts",
"AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -953,7 +964,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts",
"AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by value DESC",
PanelType: v3.PanelTypeGraph,
},
{
@ -981,7 +992,7 @@ var testBuildTracesQueryData = []struct {
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts",
"AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10",
PanelType: v3.PanelTypeValue,
},
{
@ -1120,7 +1131,218 @@ func TestBuildTracesQuery(t *testing.T) {
Convey("TestBuildTracesQuery", t, func() {
query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{
"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
}, tt.PanelType)
}, tt.PanelType, "")
So(err, ShouldBeNil)
So(query, ShouldEqual, tt.ExpectedQuery)
})
}
}
var testPrepTracesQueryData = []struct {
Name string
PanelType v3.PanelType
Start int64
End int64
BuilderQuery *v3.BuilderQuery
ExpectedQuery string
Keys map[string]v3.AttributeKey
Type string
}{
{
Name: "Test TS with limit- first",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
Limit: 10,
StepInterval: 60,
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
},
ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," +
" toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2" +
" where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND" +
" stringTagMap['method'] = 'GET' AND has(stringTagMap, 'method') group by `method` order by value DESC) LIMIT 10",
Keys: map[string]v3.AttributeKey{"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}},
Type: constants.FirstQueryGraphLimit,
},
{
Name: "Test TS with limit- first - with order by value",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
Limit: 10,
StepInterval: 60,
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: constants.SigNozOrderByValue, Order: "ASC"}},
},
ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," +
" toFloat64(count(distinct(stringTagMap['name']))) as value from " +
"signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" +
" AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" +
" has(stringTagMap, 'method') group by `method` order by value ASC) LIMIT 10",
Keys: map[string]v3.AttributeKey{},
Type: constants.FirstQueryGraphLimit,
},
{
Name: "Test TS with limit- first - with order by attribute",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{},
Limit: 10,
StepInterval: 60,
GroupBy: []v3.AttributeKey{{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}},
OrderBy: []v3.OrderBy{{ColumnName: "serviceName", Order: "ASC"}},
},
ExpectedQuery: "SELECT `serviceName` from (SELECT serviceName as `serviceName`," +
" toFloat64(count(distinct(serviceName))) as value from " +
"signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" +
" AND timestamp <= '1680066458000000000') " +
"group by `serviceName` order by `serviceName` ASC) LIMIT 10",
Keys: map[string]v3.AttributeKey{},
Type: constants.FirstQueryGraphLimit,
},
{
Name: "Test TS with limit- first - with 2 group by and 2 order by",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{},
Limit: 10,
StepInterval: 60,
GroupBy: []v3.AttributeKey{
{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
},
OrderBy: []v3.OrderBy{{ColumnName: "serviceName", Order: "ASC"}, {ColumnName: constants.SigNozOrderByValue, Order: "ASC"}},
},
ExpectedQuery: "SELECT `serviceName`,`http.method` from (SELECT serviceName as `serviceName`," +
" stringTagMap['http.method'] as `http.method`," +
" toFloat64(count(distinct(serviceName))) as value from " +
"signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" +
" AND timestamp <= '1680066458000000000') AND has(stringTagMap, 'http.method') " +
"group by `serviceName`,`http.method` order by `serviceName` ASC,value ASC) LIMIT 10",
Keys: map[string]v3.AttributeKey{},
Type: constants.FirstQueryGraphLimit,
},
{
Name: "Test TS with limit- second",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
Limit: 2,
StepInterval: 60,
},
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " +
"stringTagMap['method'] as `method`, toFloat64(count(distinct(stringTagMap['name'])))" +
" as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" +
" AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" +
" has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by value DESC",
Keys: map[string]v3.AttributeKey{},
Type: constants.SecondQueryGraphLimit,
},
{
Name: "Test TS with limit- second - with order by",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
Limit: 2,
StepInterval: 60,
},
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " +
"stringTagMap['method'] as `method`, toFloat64(count(distinct(stringTagMap['name'])))" +
" as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" +
" AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" +
" has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by `method` ASC", Keys: map[string]v3.AttributeKey{},
Type: constants.SecondQueryGraphLimit,
},
{
Name: "Test TS with limit - second - with two group by and two order by",
PanelType: v3.PanelTypeGraph,
Start: 1680066360726210000,
End: 1680066458000000000,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{
{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "name", Order: "ASC"}},
Limit: 2,
StepInterval: 60,
},
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " +
"stringTagMap['method'] as `method`, stringTagMap['name'] as `name`," +
" toFloat64(count(distinct(stringTagMap['name'])))" +
" as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" +
" AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" +
" has(stringTagMap, 'method') AND has(stringTagMap, 'name') " +
"AND (`method`,`name`) GLOBAL IN (%s) group by `method`,`name`,ts " +
"order by `method` ASC,`name` ASC",
Keys: map[string]v3.AttributeKey{},
Type: constants.SecondQueryGraphLimit,
},
}
func TestPrepareTracesQuery(t *testing.T) {
for _, tt := range testPrepTracesQueryData {
Convey("TestPrepareTracesQuery", t, func() {
query, err := PrepareTracesQuery(tt.Start, tt.End, tt.PanelType, tt.BuilderQuery, tt.Keys, tt.Type)
So(err, ShouldBeNil)
So(query, ShouldEqual, tt.ExpectedQuery)
})