mirror of
https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz
synced 2025-06-04 11:25:52 +08:00
422 lines
14 KiB
Go
422 lines
14 KiB
Go
package v3
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"strings"
|
|
|
|
"go.signoz.io/signoz/pkg/query-service/constants"
|
|
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
|
|
"go.signoz.io/signoz/pkg/query-service/utils"
|
|
)
|
|
|
|
var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{
|
|
v3.AggregateOperatorP05: 0.05,
|
|
v3.AggregateOperatorP10: 0.10,
|
|
v3.AggregateOperatorP20: 0.20,
|
|
v3.AggregateOperatorP25: 0.25,
|
|
v3.AggregateOperatorP50: 0.50,
|
|
v3.AggregateOperatorP75: 0.75,
|
|
v3.AggregateOperatorP90: 0.90,
|
|
v3.AggregateOperatorP95: 0.95,
|
|
v3.AggregateOperatorP99: 0.99,
|
|
}
|
|
|
|
var aggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{
|
|
v3.AggregateOperatorAvg: "avg",
|
|
v3.AggregateOperatorMax: "max",
|
|
v3.AggregateOperatorMin: "min",
|
|
v3.AggregateOperatorSum: "sum",
|
|
v3.AggregateOperatorRate: "count",
|
|
v3.AggregateOperatorRateSum: "sum",
|
|
v3.AggregateOperatorRateAvg: "avg",
|
|
v3.AggregateOperatorRateMax: "max",
|
|
v3.AggregateOperatorRateMin: "min",
|
|
}
|
|
|
|
var tracesOperatorMappingV3 = map[v3.FilterOperator]string{
|
|
v3.FilterOperatorIn: "IN",
|
|
v3.FilterOperatorNotIn: "NOT IN",
|
|
v3.FilterOperatorEqual: "=",
|
|
v3.FilterOperatorNotEqual: "!=",
|
|
v3.FilterOperatorLessThan: "<",
|
|
v3.FilterOperatorLessThanOrEq: "<=",
|
|
v3.FilterOperatorGreaterThan: ">",
|
|
v3.FilterOperatorGreaterThanOrEq: ">=",
|
|
v3.FilterOperatorLike: "ILIKE",
|
|
v3.FilterOperatorNotLike: "NOT ILIKE",
|
|
v3.FilterOperatorContains: "ILIKE",
|
|
v3.FilterOperatorNotContains: "NOT ILIKE",
|
|
v3.FilterOperatorExists: "has(%s%s, '%s')",
|
|
v3.FilterOperatorNotExists: "NOT has(%s%s, '%s')",
|
|
}
|
|
|
|
func getColumnName(key v3.AttributeKey, keys map[string]v3.AttributeKey) string {
|
|
key = enrichKeyWithMetadata(key, keys)
|
|
if key.IsColumn {
|
|
return key.Key
|
|
}
|
|
filterType, filterDataType := getClickhouseTracesColumnDataTypeAndType(key)
|
|
return fmt.Sprintf("%s%s['%s']", filterDataType, filterType, key.Key)
|
|
}
|
|
|
|
func getClickhouseTracesColumnDataTypeAndType(key v3.AttributeKey) (v3.AttributeKeyType, string) {
|
|
filterType := key.Type
|
|
filterDataType := "string"
|
|
if key.DataType == v3.AttributeKeyDataTypeFloat64 || key.DataType == v3.AttributeKeyDataTypeInt64 {
|
|
filterDataType = "number"
|
|
} else if key.DataType == v3.AttributeKeyDataTypeBool {
|
|
filterDataType = "bool"
|
|
}
|
|
if filterType == v3.AttributeKeyTypeTag {
|
|
filterType = "TagMap"
|
|
} else {
|
|
filterType = "resourceTagsMap"
|
|
filterDataType = ""
|
|
}
|
|
return filterType, filterDataType
|
|
}
|
|
|
|
func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey {
|
|
if key.Type == "" || key.DataType == "" {
|
|
// check if the key is present in the keys map
|
|
if existingKey, ok := keys[key.Key]; ok {
|
|
key.IsColumn = existingKey.IsColumn
|
|
key.Type = existingKey.Type
|
|
key.DataType = existingKey.DataType
|
|
} else { // if not present then set the default values
|
|
key.Type = v3.AttributeKeyTypeTag
|
|
key.DataType = v3.AttributeKeyDataTypeString
|
|
key.IsColumn = false
|
|
return key
|
|
}
|
|
}
|
|
return key
|
|
}
|
|
|
|
// getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator
|
|
func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey, keys map[string]v3.AttributeKey) (string, error) {
|
|
var selectLabels string
|
|
if aggregatorOperator == v3.AggregateOperatorNoOp {
|
|
selectLabels = ""
|
|
} else {
|
|
for _, tag := range groupBy {
|
|
filterName := getColumnName(tag, keys)
|
|
selectLabels += fmt.Sprintf(", %s as `%s`", filterName, tag.Key)
|
|
}
|
|
}
|
|
return selectLabels, nil
|
|
}
|
|
|
|
// getZerosForEpochNano returns the number of zeros to be appended to the epoch time for converting it to nanoseconds
|
|
func getZerosForEpochNano(epoch int64) int64 {
|
|
count := 0
|
|
if epoch == 0 {
|
|
count = 1
|
|
} else {
|
|
for epoch != 0 {
|
|
epoch /= 10
|
|
count++
|
|
}
|
|
}
|
|
return int64(math.Pow(10, float64(19-count)))
|
|
}
|
|
|
|
func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) (string, error) {
|
|
var conditions []string
|
|
|
|
if fs != nil && len(fs.Items) != 0 {
|
|
for _, item := range fs.Items {
|
|
val := item.Value
|
|
// generate the key
|
|
columnName := getColumnName(item.Key, keys)
|
|
var fmtVal string
|
|
key := enrichKeyWithMetadata(item.Key, keys)
|
|
if item.Operator != v3.FilterOperatorExists && item.Operator != v3.FilterOperatorNotExists {
|
|
var err error
|
|
val, err = utils.ValidateAndCastValue(val, key.DataType)
|
|
if err != nil {
|
|
return "", fmt.Errorf("invalid value for key %s: %v", item.Key.Key, err)
|
|
}
|
|
}
|
|
fmtVal = utils.ClickHouseFormattedValue(val)
|
|
if operator, ok := tracesOperatorMappingV3[item.Operator]; ok {
|
|
switch item.Operator {
|
|
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
|
|
conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, item.Value))
|
|
|
|
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
|
|
if key.IsColumn {
|
|
subQuery, err := existsSubQueryForFixedColumn(key, item.Operator)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
conditions = append(conditions, subQuery)
|
|
} else {
|
|
columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key)
|
|
conditions = append(conditions, fmt.Sprintf(operator, columnDataType, columnType, key.Key))
|
|
}
|
|
|
|
default:
|
|
conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal))
|
|
}
|
|
} else {
|
|
return "", fmt.Errorf("unsupported operator %s", item.Operator)
|
|
}
|
|
}
|
|
}
|
|
queryString := strings.Join(conditions, " AND ")
|
|
|
|
if len(queryString) > 0 {
|
|
queryString = " AND " + queryString
|
|
}
|
|
return queryString, nil
|
|
}
|
|
|
|
func existsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) {
|
|
if key.DataType == v3.AttributeKeyDataTypeString {
|
|
if op == v3.FilterOperatorExists {
|
|
return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorNotEqual]), nil
|
|
} else {
|
|
return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorEqual]), nil
|
|
}
|
|
} else {
|
|
return "", fmt.Errorf("unsupported operation, exists and not exists can only be applied on custom attributes or string type columns")
|
|
}
|
|
}
|
|
|
|
func handleEmptyValuesInGroupBy(keys map[string]v3.AttributeKey, groupBy []v3.AttributeKey) (string, error) {
|
|
filterItems := []v3.FilterItem{}
|
|
if len(groupBy) != 0 {
|
|
for _, item := range groupBy {
|
|
key := enrichKeyWithMetadata(item, keys)
|
|
if !key.IsColumn {
|
|
filterItems = append(filterItems, v3.FilterItem{
|
|
Key: item,
|
|
Operator: v3.FilterOperatorExists,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
if len(filterItems) != 0 {
|
|
filterSet := v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: filterItems,
|
|
}
|
|
return buildTracesFilterQuery(&filterSet, keys)
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey) (string, error) {
|
|
|
|
filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// timerange will be sent in epoch millisecond
|
|
spanIndexTableTimeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d')", start*getZerosForEpochNano(start), end*getZerosForEpochNano(end))
|
|
|
|
selectLabels, err := getSelectLabels(mq.AggregateOperator, mq.GroupBy, keys)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
having := having(mq.Having)
|
|
if having != "" {
|
|
having = " having " + having
|
|
}
|
|
|
|
// Select the aggregate value for interval
|
|
queryTmpl :=
|
|
"SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts" + selectLabels +
|
|
", %s as value " +
|
|
"from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME +
|
|
" where " + spanIndexTableTimeFilter + "%s " +
|
|
"group by %s%s " +
|
|
"order by %sts"
|
|
|
|
emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
filterSubQuery += emptyValuesInGroupByFilter
|
|
|
|
groupBy := groupByAttributeKeyTags(keys, mq.GroupBy...)
|
|
orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
|
|
|
|
aggregationKey := ""
|
|
if mq.AggregateAttribute.Key != "" {
|
|
aggregationKey = getColumnName(mq.AggregateAttribute, keys)
|
|
}
|
|
|
|
switch mq.AggregateOperator {
|
|
case v3.AggregateOperatorRateSum,
|
|
v3.AggregateOperatorRateMax,
|
|
v3.AggregateOperatorRateAvg,
|
|
v3.AggregateOperatorRateMin,
|
|
v3.AggregateOperatorRate:
|
|
op := fmt.Sprintf("%s(%s)/%d", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, step)
|
|
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
|
|
return query, nil
|
|
case
|
|
v3.AggregateOperatorP05,
|
|
v3.AggregateOperatorP10,
|
|
v3.AggregateOperatorP20,
|
|
v3.AggregateOperatorP25,
|
|
v3.AggregateOperatorP50,
|
|
v3.AggregateOperatorP75,
|
|
v3.AggregateOperatorP90,
|
|
v3.AggregateOperatorP95,
|
|
v3.AggregateOperatorP99:
|
|
op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey)
|
|
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
|
|
return query, nil
|
|
case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax:
|
|
op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey)
|
|
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
|
|
return query, nil
|
|
case v3.AggregateOperatorCount:
|
|
if mq.AggregateAttribute.Key != "" {
|
|
key := enrichKeyWithMetadata(mq.AggregateAttribute, keys)
|
|
if key.IsColumn {
|
|
subQuery, err := existsSubQueryForFixedColumn(key, v3.FilterOperatorExists)
|
|
if err != nil {
|
|
filterSubQuery = ""
|
|
}
|
|
filterSubQuery = fmt.Sprintf(" AND %s", subQuery)
|
|
} else {
|
|
columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(key)
|
|
filterSubQuery = fmt.Sprintf("%s AND has(%s%s, '%s')", filterSubQuery, columnDataType, columnType, mq.AggregateAttribute.Key)
|
|
}
|
|
}
|
|
op := "toFloat64(count())"
|
|
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
|
|
return query, nil
|
|
case v3.AggregateOperatorCountDistinct:
|
|
op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey)
|
|
query := fmt.Sprintf(queryTmpl, step, op, filterSubQuery, groupBy, having, orderBy)
|
|
return query, nil
|
|
case v3.AggregateOperatorNoOp:
|
|
// queryTmpl := constants.TracesSQLSelect + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + " where %s %s"
|
|
// query := fmt.Sprintf(queryTmpl, spanIndexTableTimeFilter, filterSubQuery)
|
|
// return query, nil
|
|
return "", fmt.Errorf("not implemented, part of traces page")
|
|
default:
|
|
return "", fmt.Errorf("unsupported aggregate operator")
|
|
}
|
|
}
|
|
|
|
// groupBy returns a string of comma separated tags for group by clause
|
|
// `ts` is always added to the group by clause
|
|
func groupBy(tags ...string) string {
|
|
tags = append(tags, "ts")
|
|
return strings.Join(tags, ",")
|
|
}
|
|
|
|
func groupByAttributeKeyTags(keys map[string]v3.AttributeKey, tags ...v3.AttributeKey) string {
|
|
groupTags := []string{}
|
|
for _, tag := range tags {
|
|
groupTags = append(groupTags, tag.Key)
|
|
}
|
|
return groupBy(groupTags...)
|
|
}
|
|
|
|
// orderBy returns a string of comma separated tags for order by clause
|
|
// if the order is not specified, it defaults to ASC
|
|
func orderBy(items []v3.OrderBy, tags []string) string {
|
|
var orderBy []string
|
|
for _, tag := range tags {
|
|
found := false
|
|
for _, item := range items {
|
|
if item.ColumnName == tag {
|
|
found = true
|
|
orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order))
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag))
|
|
}
|
|
}
|
|
|
|
// users might want to order by value of aggreagation
|
|
for _, item := range items {
|
|
if item.ColumnName == constants.SigNozOrderByValue {
|
|
orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order))
|
|
}
|
|
}
|
|
return strings.Join(orderBy, ",")
|
|
}
|
|
|
|
func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string {
|
|
var groupTags []string
|
|
for _, tag := range tags {
|
|
groupTags = append(groupTags, tag.Key)
|
|
}
|
|
str := orderBy(items, groupTags)
|
|
if len(str) > 0 {
|
|
str = str + ","
|
|
}
|
|
return str
|
|
}
|
|
|
|
func having(items []v3.Having) string {
|
|
// aggregate something and filter on that aggregate
|
|
var having []string
|
|
for _, item := range items {
|
|
having = append(having, fmt.Sprintf("value %s %s", item.Operator, utils.ClickHouseFormattedValue(item.Value)))
|
|
}
|
|
return strings.Join(having, " AND ")
|
|
}
|
|
|
|
func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator v3.AggregateOperator) (string, error) {
|
|
|
|
var groupBy string
|
|
switch reduceTo {
|
|
case v3.ReduceToOperatorLast:
|
|
query = fmt.Sprintf("SELECT anyLast(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
|
|
case v3.ReduceToOperatorSum:
|
|
query = fmt.Sprintf("SELECT sum(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
|
|
case v3.ReduceToOperatorAvg:
|
|
query = fmt.Sprintf("SELECT avg(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
|
|
case v3.ReduceToOperatorMax:
|
|
query = fmt.Sprintf("SELECT max(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
|
|
case v3.ReduceToOperatorMin:
|
|
query = fmt.Sprintf("SELECT min(value) as value, any(ts) as ts FROM (%s) %s", query, groupBy)
|
|
default:
|
|
return "", fmt.Errorf("unsupported reduce operator")
|
|
}
|
|
return query, nil
|
|
}
|
|
|
|
func addLimitToQuery(query string, limit uint64, panelType v3.PanelType) string {
|
|
if limit == 0 {
|
|
limit = 100
|
|
}
|
|
if panelType == v3.PanelTypeList {
|
|
return fmt.Sprintf("%s LIMIT %d", query, limit)
|
|
}
|
|
return query
|
|
}
|
|
|
|
func addOffsetToQuery(query string, offset uint64) string {
|
|
return fmt.Sprintf("%s OFFSET %d", query, offset)
|
|
}
|
|
|
|
func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) {
|
|
query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if panelType == v3.PanelTypeValue {
|
|
query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator)
|
|
}
|
|
query = addLimitToQuery(query, mq.Limit, panelType)
|
|
|
|
if mq.Offset != 0 {
|
|
query = addOffsetToQuery(query, mq.Offset)
|
|
}
|
|
return query, err
|
|
}
|