package v4 import ( "fmt" "strings" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/utils" ) // buildResourceFilter builds a clickhouse filter string for resource labels func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value interface{}) string { searchKey := fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key) chFmtVal := utils.ClickHouseFormattedValue(value) switch op { case v3.FilterOperatorExists: return fmt.Sprintf("simpleJSONHas(labels, '%s')", key) case v3.FilterOperatorNotExists: return fmt.Sprintf("not simpleJSONHas(labels, '%s')", key) case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: return fmt.Sprintf(logsOp, searchKey, chFmtVal) case v3.FilterOperatorContains, v3.FilterOperatorNotContains: // this is required as clickhouseFormattedValue add's quotes to the string escapedStringValue := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", value)) return fmt.Sprintf("%s %s '%%%s%%'", searchKey, logsOp, escapedStringValue) default: return fmt.Sprintf("%s %s %s", searchKey, logsOp, chFmtVal) } } // buildIndexFilterForInOperator builds a clickhouse filter string for in operator // example:= x in a,b,c = (labels like '%x%a%' or labels like '%"x":"b"%' or labels like '%"x"="c"%') // example:= x nin a,b,c = (labels nlike '%x%a%' AND labels nlike '%"x"="b"' AND labels nlike '%"x"="c"%') func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value interface{}) string { conditions := []string{} separator := " OR " sqlOp := "like" if op == v3.FilterOperatorNotIn { separator = " AND " sqlOp = "not like" } // values is a slice of strings, we need to convert value to this type // value can be string or []interface{} values := []string{} switch value.(type) { case string: values = append(values, value.(string)) case []interface{}: for _, v := range (value).([]interface{}) { // also resources attributes are always string values strV, ok := v.(string) if !ok { continue } values = append(values, strV) } } // if there are no values to filter on, return an empty string if len(values) > 0 { for _, v := range values { value := utils.QuoteEscapedStringForContains(v) conditions = append(conditions, fmt.Sprintf("labels %s '%%\"%s\":\"%s\"%%'", sqlOp, key, value)) } return "(" + strings.Join(conditions, separator) + ")" } return "" } // buildResourceIndexFilter builds a clickhouse filter string for resource labels // example:= x like '%john%' = labels like '%x%john%' func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{}) string { // not using clickhouseFormattedValue as we don't wan't the quotes formattedValueEscaped := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", value)) // add index filters switch op { case v3.FilterOperatorContains, v3.FilterOperatorEqual, v3.FilterOperatorLike: return fmt.Sprintf("labels like '%%%s%%%s%%'", key, formattedValueEscaped) case v3.FilterOperatorNotContains, v3.FilterOperatorNotEqual, v3.FilterOperatorNotLike: return fmt.Sprintf("labels not like '%%%s%%%s%%'", key, formattedValueEscaped) case v3.FilterOperatorNotRegex: return fmt.Sprintf("labels not like '%%%s%%'", key) case v3.FilterOperatorIn, v3.FilterOperatorNotIn: return buildIndexFilterForInOperator(key, op, value) default: return fmt.Sprintf("labels like '%%%s%%'", key) } } // buildResourceFiltersFromFilterItems builds a list of clickhouse filter strings for resource labels from a FilterSet. // It skips any filter items that are not resource attributes and checks that the operator is supported and the data type is correct. func buildResourceFiltersFromFilterItems(fs *v3.FilterSet) ([]string, error) { var conditions []string if fs == nil || len(fs.Items) == 0 { return nil, nil } for _, item := range fs.Items { // skip anything other than resource attribute if item.Key.Type != v3.AttributeKeyTypeResource { continue } // since out map is in lower case we are converting it to lowercase operatorLower := strings.ToLower(string(item.Operator)) op := v3.FilterOperator(operatorLower) keyName := item.Key.Key // resource filter value data type will always be string // will be an interface if the operator is IN or NOT IN if item.Key.DataType != v3.AttributeKeyDataTypeString && (op != v3.FilterOperatorIn && op != v3.FilterOperatorNotIn) { return nil, fmt.Errorf("invalid data type for resource attribute: %s", item.Key.Key) } var value interface{} var err error if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists { // make sure to cast the value regardless of the actual type value, err = utils.ValidateAndCastValue(item.Value, item.Key.DataType) if err != nil { return nil, fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) } } if logsOp, ok := logOperators[op]; ok { // the filter if resourceFilter := buildResourceFilter(logsOp, keyName, op, value); resourceFilter != "" { conditions = append(conditions, resourceFilter) } // the additional filter for better usage of the index if resourceIndexFilter := buildResourceIndexFilter(keyName, op, value); resourceIndexFilter != "" { conditions = append(conditions, resourceIndexFilter) } } else { return nil, fmt.Errorf("unsupported operator: %s", op) } } return conditions, nil } func buildResourceFiltersFromGroupBy(groupBy []v3.AttributeKey) []string { var conditions []string for _, attr := range groupBy { if attr.Type != v3.AttributeKeyTypeResource { continue } conditions = append(conditions, fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", attr.Key, attr.Key)) } return conditions } func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeKey) string { if aggregateAttribute.Key != "" && aggregateAttribute.Type == v3.AttributeKeyTypeResource { return fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", aggregateAttribute.Key, aggregateAttribute.Key) } return "" } func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) { // BUILD THE WHERE CLAUSE var conditions []string // only add the resource attributes to the filters here rs, err := buildResourceFiltersFromFilterItems(fs) if err != nil { return "", err } conditions = append(conditions, rs...) // for aggregate attribute add exists check in resources aggregateAttributeResourceFilter := buildResourceFiltersFromAggregateAttribute(aggregateAttribute) if aggregateAttributeResourceFilter != "" { conditions = append(conditions, aggregateAttributeResourceFilter) } groupByResourceFilters := buildResourceFiltersFromGroupBy(groupBy) if len(groupByResourceFilters) > 0 { // TODO: change AND to OR once we know how to solve for group by ( i.e show values if one is not present) groupByStr := "( " + strings.Join(groupByResourceFilters, " AND ") + " )" conditions = append(conditions, groupByStr) } if len(conditions) == 0 { return "", nil } conditionStr := strings.Join(conditions, " AND ") // BUILD THE FINAL QUERY var query string if isLiveTail { query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE ", DISTRIBUTED_LOGS_V2_RESOURCE) query = "(" + query + conditionStr } else { query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd) query = "(" + query + conditionStr + ")" } return query, nil }