diff --git a/pkg/query-service/app/logs/v3/json_filter.go b/pkg/query-service/app/logs/v3/json_filter.go index 345da5a013..abb0fe6712 100644 --- a/pkg/query-service/app/logs/v3/json_filter.go +++ b/pkg/query-service/app/logs/v3/json_filter.go @@ -52,11 +52,13 @@ var jsonLogOperators = map[v3.FilterOperator]string{ v3.FilterOperatorNotRegex: "NOT match(%s, %s)", v3.FilterOperatorIn: "IN", v3.FilterOperatorNotIn: "NOT IN", + v3.FilterOperatorExists: "JSON_EXISTS(%s, '$.%s')", + v3.FilterOperatorNotExists: "NOT JSON_EXISTS(%s, '$.%s')", v3.FilterOperatorHas: "has(%s, %s)", v3.FilterOperatorNotHas: "NOT has(%s, %s)", } -func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) { +func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) { keyArr := strings.Split(key.Key, ".") if len(keyArr) < 2 { return "", fmt.Errorf("incorrect key, should contain at least 2 parts") @@ -67,6 +69,10 @@ func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) { return "", fmt.Errorf("only body can be the root key") } + if op == v3.FilterOperatorExists || op == v3.FilterOperatorNotExists { + return keyArr[0], nil + } + var dataType string var ok bool if dataType, ok = dataTypeMapping[string(key.DataType)]; !ok { @@ -99,29 +105,45 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { dataType = v3.AttributeKeyDataType(val) } - key, err := getJSONFilterKey(item.Key, isArray) + key, err := getJSONFilterKey(item.Key, item.Operator, isArray) if err != nil { return "", err } // non array - value, err := utils.ValidateAndCastValue(item.Value, dataType) - if err != nil { - return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) - } - op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) - if logsOp, ok := jsonLogOperators[op]; ok { - switch op { - case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: - fmtVal := utils.ClickHouseFormattedValue(value) - return fmt.Sprintf(logsOp, key, fmtVal), nil - case v3.FilterOperatorContains, v3.FilterOperatorNotContains: - return fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value), nil - default: - fmtVal := utils.ClickHouseFormattedValue(value) - return fmt.Sprintf("%s %s %s", key, logsOp, fmtVal), nil + + var value interface{} + if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists { + value, err = utils.ValidateAndCastValue(item.Value, dataType) + if err != nil { + return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) } } - return "", fmt.Errorf("unsupported operator: %s", op) + + var filter string + if logsOp, ok := jsonLogOperators[op]; ok { + switch op { + case v3.FilterOperatorExists, v3.FilterOperatorNotExists: + filter = fmt.Sprintf(logsOp, key, strings.Join(strings.Split(item.Key.Key, ".")[1:], ".")) + case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: + fmtVal := utils.ClickHouseFormattedValue(value) + filter = fmt.Sprintf(logsOp, key, fmtVal) + case v3.FilterOperatorContains, v3.FilterOperatorNotContains: + filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value) + default: + fmtVal := utils.ClickHouseFormattedValue(value) + filter = fmt.Sprintf("%s %s %s", key, logsOp, fmtVal) + } + } else { + return "", fmt.Errorf("unsupported operator: %s", op) + } + + // add exists check for non array items as default values of int/float/bool will corrupt the results + if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) { + existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", strings.Join(strings.Split(item.Key.Key, ".")[1:], ".")) + filter = fmt.Sprintf("%s AND %s", existsFilter, filter) + } + + return filter, nil } diff --git a/pkg/query-service/app/logs/v3/json_filter_test.go b/pkg/query-service/app/logs/v3/json_filter_test.go index 455d705b1d..2f9a7f22da 100644 --- a/pkg/query-service/app/logs/v3/json_filter_test.go +++ b/pkg/query-service/app/logs/v3/json_filter_test.go @@ -12,6 +12,7 @@ var testGetJSONFilterKeyData = []struct { Key v3.AttributeKey IsArray bool ClickhouseKey string + Operator v3.FilterOperator Error bool }{ { @@ -129,7 +130,7 @@ var testGetJSONFilterKeyData = []struct { func TestGetJSONFilterKey(t *testing.T) { for _, tt := range testGetJSONFilterKeyData { Convey("testgetKey", t, func() { - columnName, err := getJSONFilterKey(tt.Key, tt.IsArray) + columnName, err := getJSONFilterKey(tt.Key, tt.Operator, tt.IsArray) if tt.Error { So(err, ShouldNotBeNil) } else { @@ -209,7 +210,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: "hello", }, - Filter: "JSON_VALUE(body, '$.message') = 'hello'", + Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') = 'hello'", }, { Name: "eq operator number", @@ -222,7 +223,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: 1, }, - Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1", + Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1", }, { Name: "neq operator number", @@ -235,7 +236,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: 1.1, }, - Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000", + Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000", }, { Name: "eq operator bool", @@ -248,7 +249,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: true, }, - Filter: "JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true", + Filter: "JSON_EXISTS(body, '$.boolkey') AND JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true", }, { Name: "greater than operator", @@ -261,7 +262,7 @@ var testGetJSONFilterData = []struct { Operator: ">", Value: 1, }, - Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1", + Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1", }, { Name: "regex operator", @@ -274,7 +275,7 @@ var testGetJSONFilterData = []struct { Operator: "regex", Value: "a*", }, - Filter: "match(JSON_VALUE(body, '$.message'), 'a*')", + Filter: "JSON_EXISTS(body, '$.message') AND match(JSON_VALUE(body, '$.message'), 'a*')", }, { Name: "contains operator", @@ -287,7 +288,20 @@ var testGetJSONFilterData = []struct { Operator: "contains", Value: "a", }, - Filter: "JSON_VALUE(body, '$.message') ILIKE '%a%'", + Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%'", + }, + { + Name: "exists", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "exists", + Value: "", + }, + Filter: "JSON_EXISTS(body, '$.message')", }, } diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index 4c23b5f6a9..8ae90e2e0f 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -908,7 +908,7 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", + ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", }, { Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy",