GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-11 13:59:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: traces QB (#2571)

Browse Source
This commit is contained in:
Vishal Sharma 2023-04-25 21:53:46 +05:30 committed by GitHub
parent dcad77746a
commit bd248c46b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1327 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/query-service/app/clickhouseReader/options.go
View File

@ -29,6 +29,7 @@ const (
defaultDependencyGraphTable string = "distributed_dependency_graph_minutes_v2"
defaultTopLevelOperationsTable string = "distributed_top_level_operations"
defaultSpanAttributeTable string = "distributed_span_attributes"
defaultSpanAttributeKeysTable string = "distributed_span_attributes_keys"
defaultLogsDB string = "signoz_logs"
defaultLogsTable string = "distributed_logs"
defaultLogsLocalTable string = "logs"
@ -65,6 +66,7 @@ type namespaceConfig struct {
SpansTable string
ErrorTable string
SpanAttributeTable string
SpanAttributeKeysTable string
DependencyGraphTable string
TopLevelOperationsTable string
LogsDB string
@ -135,6 +137,7 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s
UsageExplorerTable: defaultUsageExplorerTable,
SpansTable: defaultSpansTable,
SpanAttributeTable: defaultSpanAttributeTable,
SpanAttributeKeysTable: defaultSpanAttributeKeysTable,
DependencyGraphTable: defaultDependencyGraphTable,
TopLevelOperationsTable: defaultTopLevelOperationsTable,
LogsDB: defaultLogsDB,

41
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -96,6 +96,7 @@ type ClickHouseReader struct {
usageExplorerTable string
SpansTable string
spanAttributeTable string
spanAttributesKeysTable string
dependencyGraphTable string
topLevelOperationsTable string
logsDB string
@ -147,6 +148,7 @@ func NewReader(localDB *sqlx.DB, configFile string, featureFlag interfaces.Featu
durationTable: options.primary.DurationTable,
SpansTable: options.primary.SpansTable,
spanAttributeTable: options.primary.SpanAttributeTable,
spanAttributesKeysTable: options.primary.SpanAttributeKeysTable,
dependencyGraphTable: options.primary.DependencyGraphTable,
topLevelOperationsTable: options.primary.TopLevelOperationsTable,
logsDB: options.primary.LogsDB,
@ -3809,7 +3811,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v
switch req.Operator {
case
v3.AggregateOperatorCountDistinct,
v3.AggregateOpeatorCount:
v3.AggregateOperatorCount:
where = "tagKey ILIKE $1"
stringAllowed = true
case
@ -4236,7 +4238,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req
switch req.Operator {
case
v3.AggregateOperatorCountDistinct,
v3.AggregateOpeatorCount:
v3.AggregateOperatorCount:
where = "tagKey ILIKE $1"
case
v3.AggregateOperatorRateSum,
@ -4387,3 +4389,38 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.
return &attributeValues, nil
}
func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error) {
var query string
var err error
var rows driver.Rows
response := map[string]v3.AttributeKey{}
query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType, isColumn FROM %s.%s", r.TraceDB, r.spanAttributesKeysTable)
rows, err = r.db.Query(ctx, query)
if err != nil {
zap.S().Error(err)
return nil, fmt.Errorf("error while executing query: %s", err.Error())
}
defer rows.Close()
var tagKey string
var dataType string
var tagType string
var isColumn bool
for rows.Next() {
if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil {
return nil, fmt.Errorf("error while scanning rows: %s", err.Error())
}
key := v3.AttributeKey{
Key: tagKey,
DataType: v3.AttributeKeyDataType(dataType),
Type: v3.AttributeKeyType(tagType),
IsColumn: isColumn,
}
response[tagKey] = key
}
return response, nil
}

31
pkg/query-service/app/http_handler.go
View File

@ -25,6 +25,7 @@ import (
"go.signoz.io/signoz/pkg/query-service/app/metrics"
metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3"
"go.signoz.io/signoz/pkg/query-service/app/parser"
tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3"
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
@ -107,10 +108,8 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) {
builderOpts := queryBuilderOptions{
BuildMetricQuery: metricsv3.PrepareMetricQuery,
BuildTraceQuery: func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error) {
return "", errors.New("not implemented")
},
BuildLogQuery: logsv3.PrepareLogsQuery,
BuildTraceQuery: tracesV3.PrepareTracesQuery,
BuildLogQuery: logsv3.PrepareLogsQuery,
}
aH.queryBuilder = NewQueryBuilder(builderOpts)
@ -2669,6 +2668,20 @@ func (aH *APIHandler) getLogFieldsV3(ctx context.Context, queryRangeParams *v3.Q
return data, nil
}
func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3) (map[string]v3.AttributeKey, error) {
data := map[string]v3.AttributeKey{}
for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {
if query.DataSource == v3.DataSourceTraces {
spanKeys, err := aH.reader.GetSpanAttributeKeys(ctx)
if err != nil {
return nil, err
}
return spanKeys, nil
}
}
return data, nil
}
func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3, w http.ResponseWriter, r *http.Request) {
var result []*v3.Result
@ -2686,7 +2699,15 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que
return
}
queries, err = aH.queryBuilder.prepareQueries(queryRangeParams, fields)
var spanKeys map[string]v3.AttributeKey
spanKeys, err = aH.getSpanKeysV3(ctx, queryRangeParams)
if err != nil {
apiErrObj := &model.ApiError{Typ: model.ErrorInternal, Err: err}
RespondError(w, apiErrObj, errQuriesByName)
return
}
queries, err = aH.queryBuilder.prepareQueries(queryRangeParams, fields, spanKeys)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return

2
pkg/query-service/app/logs/v3/query_builder.go
View File

@ -267,7 +267,7 @@ func buildLogsQuery(start, end, step int64, mq *v3.BuilderQuery, fields map[stri
op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOpeatorCount:
case v3.AggregateOperatorCount:
if mq.AggregateAttribute.Key != "" {
field, err := encrichFieldWithMetadata(mq.AggregateAttribute, fields)
if err != nil {

14
pkg/query-service/app/logs/v3/query_builder_test.go
View File

@ -58,19 +58,19 @@ var testGetSelectLabelsData = []struct {
}{
{
Name: "select fields for groupBy attribute",
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
SelectLabels: ", attributes_string_value[indexOf(attributes_string_key, 'user_name')] as user_name",
},
{
Name: "select fields for groupBy resource",
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}},
SelectLabels: ", resources_string_value[indexOf(resources_string_key, 'user_name')] as user_name",
},
{
Name: "select fields for groupBy attribute and resource",
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{
{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
@ -79,7 +79,7 @@ var testGetSelectLabelsData = []struct {
},
{
Name: "select fields for groupBy materialized columns",
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "host", IsColumn: true}},
SelectLabels: ", host as host",
},
@ -219,7 +219,7 @@ var testBuildLogsQueryData = []struct {
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
},
TableName: "logs",
@ -233,7 +233,7 @@ var testBuildLogsQueryData = []struct {
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
},
TableName: "logs",
@ -247,7 +247,7 @@ var testBuildLogsQueryData = []struct {
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOpeatorCount,
AggregateOperator: v3.AggregateOperatorCount,
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: ">"},
}},

2
pkg/query-service/app/metrics/v3/query_builder.go
View File

@ -260,7 +260,7 @@ func buildMetricQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
op := fmt.Sprintf("%s(value)", aggregateOperatorToSQLFunc[mq.AggregateOperator])
query := fmt.Sprintf(queryTmpl, groupTags, step, op, filterSubQuery, groupBy, orderBy)
return query, nil
case v3.AggregateOpeatorCount:
case v3.AggregateOperatorCount:
op := "toFloat64(count(*))"
query := fmt.Sprintf(queryTmpl, groupTags, step, op, filterSubQuery, groupBy, orderBy)
return query, nil

8
pkg/query-service/app/query_builder.go
View File

@ -35,7 +35,7 @@ var SupportedFunctions = []string{
var evalFuncs = map[string]govaluate.ExpressionFunction{}
type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error)
type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error)
type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, fields map[string]v3.AttributeKey) (string, error)
type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error)
@ -139,7 +139,11 @@ func (qb *queryBuilder) prepareQueries(params *v3.QueryRangeParamsV3, args ...in
if query.Expression == queryName {
switch query.DataSource {
case v3.DataSourceTraces:
queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query)
keys := map[string]v3.AttributeKey{}
if len(args) == 2 {
keys = args[1].(map[string]v3.AttributeKey)
}
queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, keys)
if err != nil {
return nil, err
}

421
pkg/query-service/app/traces/v3/query_builder.go Normal file
View File

@ -0,0 +1,421 @@
package v3
import (
"fmt"
"math"
"strings"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
"go.signoz.io/signoz/pkg/query-service/utils"
)
var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{
v3.AggregateOperatorP05: 0.05,
v3.AggregateOperatorP10: 0.10,
v3.AggregateOperatorP20: 0.20,
v3.AggregateOperatorP25: 0.25,
v3.AggregateOperatorP50: 0.50,
v3.AggregateOperatorP75: 0.75,
v3.AggregateOperatorP90: 0.90,
v3.AggregateOperatorP95: 0.95,
v3.AggregateOperatorP99: 0.99,
}
var aggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{
v3.AggregateOperatorAvg: "avg",
v3.AggregateOperatorMax: "max",
v3.AggregateOperatorMin: "min",
v3.AggregateOperatorSum: "sum",
v3.AggregateOperatorRate: "count",
v3.AggregateOperatorRateSum: "sum",
v3.AggregateOperatorRateAvg: "avg",
v3.AggregateOperatorRateMax: "max",
v3.AggregateOperatorRateMin: "min",
}
var tracesOperatorMappingV3 = map[v3.FilterOperator]string{
v3.FilterOperatorIn: "IN",
v3.FilterOperatorNotIn: "NOT IN",
v3.FilterOperatorEqual: "=",
v3.FilterOperatorNotEqual: "!=",
v3.FilterOperatorLessThan: "<",
v3.FilterOperatorLessThanOrEq: "<=",
v3.FilterOperatorGreaterThan: ">",
v3.FilterOperatorGreaterThanOrEq: ">=",
v3.FilterOperatorLike: "ILIKE",
v3.FilterOperatorNotLike: "NOT ILIKE",
v3.FilterOperatorContains: "ILIKE",
v3.FilterOperatorNotContains: "NOT ILIKE",
v3.FilterOperatorExists: "has(%s%s, '%s')",
v3.FilterOperatorNotExists: "NOT has(%s%s, '%s')",
}
func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) string {
key = enrichKeyWithMetadata(key, keys)
if key.IsColumn {
return key.Key
}
filterType, filterDataType := getClickhouseTracesColumnDataTypeAndType(key)
return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key)
}
func getClickhouseTracesColumnDataTypeAndType(key v3.AttributeKey) (v3.AttributeKeyType, string) {
filterType := key.Type
filterDataType := "string"
if key.DataType == v3.AttributeKeyDataTypeFloat64 || key.DataType == v3.AttributeKeyDataTypeInt64 {
filterDataType = "number"
} else if key.DataType == v3.AttributeKeyDataTypeBool {
filterDataType = "bool"
}
if filterType == v3.AttributeKeyTypeTag {
filterType = "TagMap"
} else {
filterType = "resourceTagsMap"
filterDataType = ""
}
return filterType, filterDataType
}
func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey {
if key.Type == "" || key.DataType == "" {
// check if the key is present in the keys map
if existingKey, ok := keys[key.Key]; ok {
key.IsColumn = existingKey.IsColumn
key.Type = existingKey.Type
key.DataType = existingKey.DataType
} else { // if not present then set the default values
key.Type = v3.AttributeKeyTypeTag
key.DataType = v3.AttributeKeyDataTypeString
key.IsColumn = false
return key
}
}
return key
}
// getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator
func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) {
var selectLabels string
if aggregatorOperator == v3.AggregateOperatorNoOp {
selectLabels = ""
} else {
for _, tag := range groupBy {
filterName := getColumnName(tag, keys)
selectLabels += fmt.Sprintf(", %s as `%s`", filterName, tag.Key)
}
}
return selectLabels, nil
}
// getZerosForEpochNano returns the number of zeros to be appended to the epoch time for converting it to nanoseconds
func getZerosForEpochNano(epoch int64) int64 {
count := 0
if epoch == 0 {
count = 1
} else {
for epoch != 0 {
epoch /= 10
count++
}
}
return int64(math.Pow(10, float64(19-count)))
}
func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) (string, error) {
var conditions []string
if fs != nil && len(fs.Items) != 0 {
for _, item := range fs.Items {
val := item.Value
// generate the key
columnName := getColumnName(item.Key, keys)
var fmtVal string
key := enrichKeyWithMetadata(item.Key, keys)
if item.Operator != v3.FilterOperatorExists && item.Operator != v3.FilterOperatorNotExists {
var err error
val, err = utils.ValidateAndCastValue(val, key.DataType)
if err != nil {
return "", fmt.Errorf("invalid value for key %s: %v", item.Key.Key, err)
}
}
fmtVal = utils.ClickHouseFormattedValue(val)
if operator, ok := tracesOperatorMappingV3[item.Operator]; ok {
switch item.Operator {
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, item.Value))
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
if key.IsColumn {
subQuery, err := existsSubQueryForFixedColumn(key, item.Operator)
if err != nil {
return "", err
}
conditions = append(conditions, subQuery)
} else {
columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key)
conditions = append(conditions, fmt.Sprintf(operator, columnDataType, columnType, key.Key))
}
default:
conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal))
}
} else {
return "", fmt.Errorf("unsupported operator %s", item.Operator)
}
}
}
queryString := strings.Join(conditions, " AND ")
if len(queryString) > 0 {
queryString = " AND " + queryString
}
return queryString, nil
}
func existsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) {
if key.DataType == v3.AttributeKeyDataTypeString {
if op == v3.FilterOperatorExists {
return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorNotEqual]), nil
} else {
return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorEqual]), nil
}
} else {
return "", fmt.Errorf("unsupported operation, exists and not exists can only be applied on custom attributes or string type columns")
}
}
func handleEmptyValuesInGroupBy(keys map[string]v3.AttributeKey, groupBy []v3.AttributeKey) (string, error) {
filterItems := []v3.FilterItem{}
if len(groupBy) != 0 {
for _, item := range groupBy {
key := enrichKeyWithMetadata(item, keys)
if !key.IsColumn {
filterItems = append(filterItems, v3.FilterItem{
Key: item,
Operator: v3.FilterOperatorExists,
})
}
}
}
if len(filterItems) != 0 {
filterSet := v3.FilterSet{
Operator: "AND",
Items: filterItems,
}
return buildTracesFilterQuery(&filterSet, keys)
}
return "", nil
}
func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey) (string, error) {
filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys)
if err != nil {
return "", err
}
// timerange will be sent in epoch millisecond
spanIndexTableTimeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d')", start*getZerosForEpochNano(start), end*getZerosForEpochNano(end))
selectLabels, err := getSelectLabels(mq.AggregateOperator, mq.GroupBy, keys)
if err != nil {
return "", err
}
having := having(mq.Having)
if having != "" {
having = " having " + having
}
// Select the aggregate value for interval
queryTmpl :=
"SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts" + selectLabels +
", %s as value " +
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
" where " + spanIndexTableTimeFilter + "%s " +
"group by %s%s " +
"order by %sts"
emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy)
if err != nil {
return "", err
}
filterSubQuery += emptyValuesInGroupByFilter
groupBy := groupByAttributeKeyTags(keys, mq.GroupBy...)
orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
aggregationKey := ""
if mq.AggregateAttribute.Key != "" {
aggregationKey = getColumnName(mq.AggregateAttribute, keys)
}
switch mq.AggregateOperator {
case v3.AggregateOperatorRateSum,
v3.AggregateOperatorRateMax,
v3.AggregateOperatorRateAvg,
v3.AggregateOperatorRateMin,
v3.AggregateOperatorRate:
op := fmt.Sprintf("%s(%s)/%d", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, step)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case
v3.AggregateOperatorP05,
v3.AggregateOperatorP10,
v3.AggregateOperatorP20,
v3.AggregateOperatorP25,
v3.AggregateOperatorP50,
v3.AggregateOperatorP75,
v3.AggregateOperatorP90,
v3.AggregateOperatorP95,
v3.AggregateOperatorP99:
op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax:
op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorCount:
if mq.AggregateAttribute.Key != "" {
key := enrichKeyWithMetadata(mq.AggregateAttribute, keys)
if key.IsColumn {
subQuery, err := existsSubQueryForFixedColumn(key, v3.FilterOperatorExists)
if err != nil {
filterSubQuery = ""
}
filterSubQuery = fmt.Sprintf(" AND %s", subQuery)
} else {
columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key)
filterSubQuery = fmt.Sprintf("%s AND has(%s%s, '%s')", filterSubQuery, columnDataType, columnType, mq.AggregateAttribute.Key)
}
}
op := "toFloat64(count())"
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorCountDistinct:
op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey)
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
return query, nil
case v3.AggregateOperatorNoOp:
// queryTmpl := constants.TracesSQLSelect + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s"
// query := fmt.Sprintf(queryTmpl, spanIndexTableTimeFilter, filterSubQuery)
// return query, nil
return "", fmt.Errorf("not implemented, part of traces page")
default:
return "", fmt.Errorf("unsupported aggregate operator")
}
}
// groupBy returns a string of comma separated tags for group by clause
// `ts` is always added to the group by clause
func groupBy(tags ...string) string {
tags = append(tags, "ts")
return strings.Join(tags, ",")
}
func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.AttributeKey) string {
groupTags := []string{}
for _, tag := range tags {
groupTags = append(groupTags, tag.Key)
}
return groupBy(groupTags...)
}
// orderBy returns a string of comma separated tags for order by clause
// if the order is not specified, it defaults to ASC
func orderBy(items []v3.OrderBy, tags []string) string {
var orderBy []string
for _, tag := range tags {
found := false
for _, item := range items {
if item.ColumnName == tag {
found = true
orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order))
break
}
}
if !found {
orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag))
}
}
// users might want to order by value of aggreagation
for _, item := range items {
if item.ColumnName == constants.SigNozOrderByValue {
orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order))
}
}
return strings.Join(orderBy, ",")
}
func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string {
var groupTags []string
for _, tag := range tags {
groupTags = append(groupTags, tag.Key)
}
str := orderBy(items, groupTags)
if len(str) > 0 {
str = str + ","
}
return str
}
func having(items []v3.Having) string {
// aggregate something and filter on that aggregate
var having []string
for _, item := range items {
having = append(having, fmt.Sprintf("value %s %s", item.Operator, utils.ClickHouseFormattedValue(item.Value)))
}
return strings.Join(having, " AND ")
}
func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v3.AggregateOperator) (string, error) {
var groupBy string
switch reduceTo {
case v3.ReduceToOperatorLast:
query = fmt.Sprintf("SELECT anyLast(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
case v3.ReduceToOperatorSum:
query = fmt.Sprintf("SELECT sum(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
case v3.ReduceToOperatorAvg:
query = fmt.Sprintf("SELECT avg(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
case v3.ReduceToOperatorMax:
query = fmt.Sprintf("SELECT max(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
case v3.ReduceToOperatorMin:
query = fmt.Sprintf("SELECT min(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
default:
return "", fmt.Errorf("unsupported reduce operator")
}
return query, nil
}
func addLimitToQuery(query string, limit uint64, panelType v3.PanelType) string {
if limit == 0 {
limit = 100
}
if panelType == v3.PanelTypeList {
return fmt.Sprintf("%s LIMIT %d", query, limit)
}
return query
}
func addOffsetToQuery(query string, offset uint64) string {
return fmt.Sprintf("%s OFFSET %d", query, offset)
}
func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) {
query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys)
if err != nil {
return "", err
}
if panelType == v3.PanelTypeValue {
query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator)
}
query = addLimitToQuery(query, mq.Limit, panelType)
if mq.Offset != 0 {
query = addOffsetToQuery(query, mq.Offset)
}
return query, err
}

816
pkg/query-service/app/traces/v3/query_builder_test.go Normal file
View File

@ -0,0 +1,816 @@
package v3
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
var buildFilterQueryData = []struct {
Name string
FilterSet *v3.FilterSet
ExpectedFilter string
}{
{
Name: "Test attribute and resource attribute",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "john", Operator: "="},
{Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="},
}},
ExpectedFilter: " AND stringTagMap['user.name'] = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'",
},
{
Name: "Test fixed column",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "john", Operator: "="},
{Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="},
}},
ExpectedFilter: " AND user.name = 'john' AND resourceTagsMap['k8s_namespace'] != 'my_service'",
},
{
Name: "Test fixed column with empty value",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "="},
{Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="},
}},
ExpectedFilter: " AND user.name = '' AND resourceTagsMap['k8s_namespace'] != 'my_service'",
},
{
Name: "Test like",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.%", Operator: "like"},
}},
ExpectedFilter: " AND stringTagMap['host'] ILIKE '102.%'",
},
{
Name: "Test IN",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{1, 2, 3, 4}, Operator: "in"},
}},
ExpectedFilter: " AND numberTagMap['bytes'] IN [1,2,3,4]",
},
{
Name: "Test DataType int64",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: ">"},
}},
ExpectedFilter: " AND numberTagMap['bytes'] > 10",
},
{
Name: "Test NOT IN",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{"john", "bunny"}, Operator: "nin"},
}},
ExpectedFilter: " AND stringTagMap['name'] NOT IN ['john','bunny']",
},
{
Name: "Test exists",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "", Operator: "exists"},
}},
ExpectedFilter: " AND has(stringTagMap, 'bytes')",
},
{
Name: "Test exists with fixed column",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "exists"},
}},
ExpectedFilter: " AND name != ''",
},
{
Name: "Test not exists",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "", Operator: "nexists"},
}},
ExpectedFilter: " AND NOT has(stringTagMap, 'bytes')",
},
{
Name: "Test not exists with fixed column",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "", Operator: "nexists"},
}},
ExpectedFilter: " AND name = ''",
},
{
Name: "Test contains",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: "contains"},
}},
ExpectedFilter: " AND stringTagMap['host'] ILIKE '%102.%'",
},
{
Name: "Test not contains",
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: "ncontains"},
}},
ExpectedFilter: " AND stringTagMap['host'] NOT ILIKE '%102.%'",
},
}
func TestBuildTracesFilterQuery(t *testing.T) {
for _, tt := range buildFilterQueryData {
Convey("TestBuildTracesFilterQuery", t, func() {
query, err := buildTracesFilterQuery(tt.FilterSet, map[string]v3.AttributeKey{})
So(err, ShouldBeNil)
So(query, ShouldEqual, tt.ExpectedFilter)
})
}
}
var handleEmptyValuesInGroupByData = []struct {
Name string
GroupBy []v3.AttributeKey
ExpectedFilter string
}{
{
Name: "String type key",
GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
ExpectedFilter: " AND has(stringTagMap, 'bytes')",
},
{
Name: "fixed column type key",
GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}},
ExpectedFilter: "",
},
{
Name: "String, float64 and fixed column type key",
GroupBy: []v3.AttributeKey{
{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
},
ExpectedFilter: " AND has(stringTagMap, 'bytes') AND has(numberTagMap, 'count')",
},
}
func TestBuildTracesHandleEmptyValuesInGroupBy(t *testing.T) {
for _, tt := range handleEmptyValuesInGroupByData {
Convey("TestBuildTracesHandleEmptyValuesInGroupBy", t, func() {
query, err := handleEmptyValuesInGroupBy(map[string]v3.AttributeKey{}, tt.GroupBy)
So(err, ShouldBeNil)
So(query, ShouldEqual, tt.ExpectedFilter)
})
}
}
var testColumnName = []struct {
Name string
AttributeKey v3.AttributeKey
ExpectedColumn string
}{
{
Name: "resource",
AttributeKey: v3.AttributeKey{Key: "collector_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: false},
ExpectedColumn: "resourceTagsMap['collector_id']",
},
{
Name: "stringAttribute",
AttributeKey: v3.AttributeKey{Key: "customer_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false},
ExpectedColumn: "stringTagMap['customer_id']",
},
{
Name: "boolAttribute",
AttributeKey: v3.AttributeKey{Key: "has_error", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: false},
ExpectedColumn: "boolTagMap['has_error']",
},
{
Name: "float64Attribute",
AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: false},
ExpectedColumn: "numberTagMap['count']",
},
{
Name: "int64Attribute",
AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag, IsColumn: false},
ExpectedColumn: "numberTagMap['count']",
},
{
Name: "column",
AttributeKey: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
ExpectedColumn: "name",
},
{
Name: "missing key",
AttributeKey: v3.AttributeKey{Key: "xyz"},
ExpectedColumn: "stringTagMap['xyz']",
},
}
func TestColumnName(t *testing.T) {
for _, tt := range testColumnName {
Convey("testColumnName", t, func() {
Column := getColumnName(tt.AttributeKey, map[string]v3.AttributeKey{})
So(Column, ShouldEqual, tt.ExpectedColumn)
})
}
}
var testGetSelectLabelsData = []struct {
Name string
AggregateOperator v3.AggregateOperator
GroupByTags []v3.AttributeKey
SelectLabels string
}{
{
Name: "select keys for groupBy attribute",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
SelectLabels: ", stringTagMap['user.name'] as `user.name`",
},
{
Name: "select keys for groupBy resource",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}},
SelectLabels: ", resourceTagsMap['user.name'] as `user.name`",
},
{
Name: "select keys for groupBy attribute and resource",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{
{Key: "user.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
},
SelectLabels: ", resourceTagsMap['user.name'] as `user.name`, stringTagMap['host'] as `host`",
},
{
Name: "select keys for groupBy fixed columns",
AggregateOperator: v3.AggregateOperatorCount,
GroupByTags: []v3.AttributeKey{{Key: "host", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
SelectLabels: ", host as `host`",
},
}
func TestGetSelectLabels(t *testing.T) {
for _, tt := range testGetSelectLabelsData {
Convey("testGetSelectLabelsData", t, func() {
selectLabels, err := getSelectLabels(tt.AggregateOperator, tt.GroupByTags, map[string]v3.AttributeKey{})
So(err, ShouldBeNil)
So(selectLabels, ShouldEqual, tt.SelectLabels)
})
}
}
var testGetZerosForEpochNanoData = []struct {
Name string
Epoch int64
Multiplier int64
Result int64
}{
{
Name: "Test 1",
Epoch: 1680712080000,
Multiplier: 1000000,
Result: 1680712080000000000,
},
{
Name: "Test 2",
Epoch: 1680712080000000000,
Multiplier: 1,
Result: 1680712080000000000,
},
}
func TestGetZerosForEpochNano(t *testing.T) {
for _, tt := range testGetZerosForEpochNanoData {
Convey("testGetZerosForEpochNanoData", t, func() {
multiplier := getZerosForEpochNano(tt.Epoch)
So(multiplier, ShouldEqual, tt.Multiplier)
So(tt.Epoch*multiplier, ShouldEqual, tt.Result)
})
}
}
var testOrderBy = []struct {
Name string
Items []v3.OrderBy
Tags []string
Result string
}{
{
Name: "Test 1",
Items: []v3.OrderBy{
{
ColumnName: "name",
Order: "asc",
},
{
ColumnName: constants.SigNozOrderByValue,
Order: "desc",
},
},
Tags: []string{"name"},
Result: "name asc,value desc",
},
{
Name: "Test 2",
Items: []v3.OrderBy{
{
ColumnName: "name",
Order: "asc",
},
{
ColumnName: "bytes",
Order: "asc",
},
},
Tags: []string{"name", "bytes"},
Result: "name asc,bytes asc",
},
{
Name: "Test 3",
Items: []v3.OrderBy{
{
ColumnName: "name",
Order: "asc",
},
{
ColumnName: constants.SigNozOrderByValue,
Order: "asc",
},
{
ColumnName: "bytes",
Order: "asc",
},
},
Tags: []string{"name", "bytes"},
Result: "name asc,bytes asc,value asc",
},
}
func TestOrderBy(t *testing.T) {
for _, tt := range testOrderBy {
Convey("testOrderBy", t, func() {
res := orderBy(tt.Items, tt.Tags)
So(res, ShouldEqual, tt.Result)
})
}
}
var testBuildTracesQueryData = []struct {
Name string
Start int64
End int64
Step int64
BuilderQuery *v3.BuilderQuery
GroupByTags []v3.AttributeKey
TableName string
AggregateOperator v3.AggregateOperator
ExpectedQuery string
}{
{
Name: "Test aggregate count on column",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts order by ts",
},
{
Name: "Test aggregate count on a attribute",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND has(stringTagMap, 'user_name') group by ts order by ts",
},
{
Name: "Test aggregate count on a fixed column",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND name != '' group by ts order by ts",
},
{
Name: "Test aggregate count with filter",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCount,
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: ">"},
}},
Expression: "A",
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by ts",
},
{
Name: "Test aggregate count distinct and order by value",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts order by value ASC,ts",
},
{
Name: "Test aggregate count distinct on string key",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name'])))" +
" as value from signoz_traces.distributed_signoz_index_v2 where" +
" (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts",
},
{
Name: "Test aggregate count distinct with filter and groupBy",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
{Key: v3.AttributeKey{Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "abc", Operator: "!="},
},
},
GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "http.method", Order: "ASC"}, {ColumnName: "ts", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['http.method'] as `http.method`, " +
"toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " +
"AND has(stringTagMap, 'http.method') group by http.method,ts " +
"order by http.method ASC,ts",
},
{
Name: "Test aggregate count with multiple filter,groupBy and orderBy",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
{Key: v3.AttributeKey{Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "abc", Operator: "!="},
},
},
GroupBy: []v3.AttributeKey{
{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
{Key: "x", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "x", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['method'] as `method`, " +
"resourceTagsMap['x'] as `x`, " +
"toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " +
"AND has(stringTagMap, 'method') AND has(resourceTagsMap, 'x') group by method,x,ts " +
"order by method ASC,x ASC,ts",
},
{
Name: "Test aggregate avg",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorAvg,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "x", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['method'] as `method`, " +
"avg(numberTagMap['bytes']) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate sum",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorSum,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['method'] as `method`, " +
"sum(bytes) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate min",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorMin,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['method'] as `method`, " +
"min(bytes) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate max",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorMax,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['method'] as `method`, " +
"max(bytes) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate PXX",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorP05,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," +
" stringTagMap['method'] as `method`, " +
"quantile(0.05)(bytes) as value " +
"from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate RateSum",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorRateSum,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" +
", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " +
"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" AND has(stringTagMap, 'method') group by method,ts order by method ASC,ts",
},
{
Name: "Test aggregate rate",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64},
AggregateOperator: v3.AggregateOperatorRate,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" +
", count(numberTagMap['bytes'])/60 as value " +
"from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate RateSum without fixed column",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeFloat64},
AggregateOperator: v3.AggregateOperatorRateSum,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " +
"stringTagMap['method'] as `method`, " +
"sum(numberTagMap['bytes'])/60 as value " +
"from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND has(stringTagMap, 'method') group by method,ts " +
"order by method ASC,ts",
},
{
Name: "Test aggregate with having clause",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Having: []v3.Having{
{
ColumnName: "name",
Operator: ">",
Value: 10,
},
},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" +
" group by ts having value > 10 order by ts",
},
{
Name: "Test count aggregate with having clause and filters",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCount,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
Having: []v3.Having{
{
ColumnName: "name",
Operator: ">",
Value: 10,
},
},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from " +
"signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts",
},
{
Name: "Test count distinct aggregate with having clause and filters",
Start: 1680066360726210000,
End: 1680066458000000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
AggregateOperator: v3.AggregateOperatorCountDistinct,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="},
},
},
Having: []v3.Having{
{
ColumnName: "name",
Operator: ">",
Value: 10,
},
},
},
TableName: "signoz_traces.distributed_signoz_index_v2",
ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" +
" from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts",
},
// {
// Name: "Test Noop",
// Start: 1680066360726210000,
// End: 1680066458000000000,
// Step: 60,
// BuilderQuery: &v3.BuilderQuery{
// SelectColumns: []v3.AttributeKey{},
// QueryName: "A",
// AggregateOperator: v3.AggregateOperatorNoOp,
// Expression: "A",
// Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
// // GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
// // OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}},
// },
// ExpectedQuery: "",
// },
}
func TestBuildTracesQuery(t *testing.T) {
for _, tt := range testBuildTracesQueryData {
Convey("TestBuildTracesQuery", t, func() {
query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{})
So(err, ShouldBeNil)
So(query, ShouldEqual, tt.ExpectedQuery)
})
}
}

2
pkg/query-service/constants/constants.go
View File

@ -144,6 +144,8 @@ const (
SIGNOZ_METRIC_DBNAME = "signoz_metrics"
SIGNOZ_SAMPLES_TABLENAME = "distributed_samples_v2"
SIGNOZ_TIMESERIES_TABLENAME = "distributed_time_series_v2"
SIGNOZ_TRACE_DBNAME = "signoz_traces"
SIGNOZ_SPAN_INDEX_TABLENAME = "distributed_signoz_index_v2"
)
var TimeoutExcludedRoutes = map[string]bool{

1
pkg/query-service/interfaces/interface.go
View File

@ -38,6 +38,7 @@ type Reader interface {
GetTraceAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error)
GetTraceAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error)
GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error)
GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error)
GetTagFilters(ctx context.Context, query *model.TagFilterParams) (*model.TagFilters, *model.ApiError)
GetTagValues(ctx context.Context, query *model.TagFilterParams) (*model.TagValues, *model.ApiError)
GetFilteredSpans(ctx context.Context, query *model.GetFilteredSpansParams) (*model.GetFilterSpansResponse, *model.ApiError)

6
pkg/query-service/model/v3/v3.go
View File

@ -30,7 +30,7 @@ type AggregateOperator string
const (
AggregateOperatorNoOp AggregateOperator = "noop"
AggregateOpeatorCount AggregateOperator = "count"
AggregateOperatorCount AggregateOperator = "count"
AggregateOperatorCountDistinct AggregateOperator = "count_distinct"
AggregateOperatorSum AggregateOperator = "sum"
AggregateOperatorAvg AggregateOperator = "avg"
@ -64,7 +64,7 @@ const (
func (a AggregateOperator) Validate() error {
switch a {
case AggregateOperatorNoOp,
AggregateOpeatorCount,
AggregateOperatorCount,
AggregateOperatorCountDistinct,
AggregateOperatorSum,
AggregateOperatorAvg,
@ -104,7 +104,7 @@ func (a AggregateOperator) Validate() error {
func (a AggregateOperator) RequireAttribute() bool {
switch a {
case AggregateOperatorNoOp,
AggregateOpeatorCount,
AggregateOperatorCount,
AggregateOperatorCountDistinct:
return false
default:

1
pkg/query-service/utils/format.go
View File

@ -10,6 +10,7 @@ import (
"go.uber.org/zap"
)
// ValidateAndCastValue validates and casts the value of a key to the corresponding data type of the key
func ValidateAndCastValue(v interface{}, dataType v3.AttributeKeyDataType) (interface{}, error) {
switch dataType {
case v3.AttributeKeyDataTypeString: