diff --git a/pkg/query-service/app/logs/v3/json_filter.go b/pkg/query-service/app/logs/v3/json_filter.go index abb0fe6712..bcc4b68a14 100644 --- a/pkg/query-service/app/logs/v3/json_filter.go +++ b/pkg/query-service/app/logs/v3/json_filter.go @@ -58,6 +58,18 @@ var jsonLogOperators = map[v3.FilterOperator]string{ v3.FilterOperatorNotHas: "NOT has(%s, %s)", } +func getPath(keyArr []string) string { + path := []string{} + for i := 0; i < len(keyArr); i++ { + if strings.HasSuffix(keyArr[i], "[*]") { + path = append(path, "\""+strings.TrimSuffix(keyArr[i], "[*]")+"\""+"[*]") + } else { + path = append(path, "\""+keyArr[i]+"\"") + } + } + return strings.Join(path, ".") +} + func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) { keyArr := strings.Split(key.Key, ".") if len(keyArr) < 2 { @@ -79,12 +91,14 @@ func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) ( return "", fmt.Errorf("unsupported dataType for JSON: %s", key.DataType) } + path := getPath(keyArr[1:]) + if isArray { - return fmt.Sprintf("JSONExtract(JSON_QUERY(%s, '$.%s'), '%s')", keyArr[0], strings.Join(keyArr[1:], "."), dataType), nil + return fmt.Sprintf("JSONExtract(JSON_QUERY(%s, '$.%s'), '%s')", keyArr[0], path, dataType), nil } // for non array - keyname := fmt.Sprintf("JSON_VALUE(%s, '$.%s')", keyArr[0], strings.Join(keyArr[1:], ".")) + keyname := fmt.Sprintf("JSON_VALUE(%s, '$.%s')", keyArr[0], path) if dataType != STRING { keyname = fmt.Sprintf("JSONExtract(%s, '%s')", keyname, dataType) } @@ -125,7 +139,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { if logsOp, ok := jsonLogOperators[op]; ok { switch op { case v3.FilterOperatorExists, v3.FilterOperatorNotExists: - filter = fmt.Sprintf(logsOp, key, strings.Join(strings.Split(item.Key.Key, ".")[1:], ".")) + filter = fmt.Sprintf(logsOp, key, getPath(strings.Split(item.Key.Key, ".")[1:])) case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: fmtVal := utils.ClickHouseFormattedValue(value) filter = fmt.Sprintf(logsOp, key, fmtVal) @@ -141,7 +155,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { // add exists check for non array items as default values of int/float/bool will corrupt the results if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) { - existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", strings.Join(strings.Split(item.Key.Key, ".")[1:], ".")) + existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", getPath(strings.Split(item.Key.Key, ".")[1:])) filter = fmt.Sprintf("%s AND %s", existsFilter, filter) } diff --git a/pkg/query-service/app/logs/v3/json_filter_test.go b/pkg/query-service/app/logs/v3/json_filter_test.go index 2f9a7f22da..2992b1e273 100644 --- a/pkg/query-service/app/logs/v3/json_filter_test.go +++ b/pkg/query-service/app/logs/v3/json_filter_test.go @@ -43,7 +43,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: true, - ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.requestor_list[*]'), '" + ARRAY_STRING + "')", + ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), '" + ARRAY_STRING + "')", }, { Name: "Array String Nested", @@ -53,7 +53,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: true, - ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.nested[*].key[*]'), '" + ARRAY_STRING + "')", + ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"nested\"[*].\"key\"[*]'), '" + ARRAY_STRING + "')", }, { Name: "Array Int", @@ -63,7 +63,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: true, - ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.int_numbers[*]'), '" + ARRAY_INT64 + "')", + ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"int_numbers\"[*]'), '" + ARRAY_INT64 + "')", }, { Name: "Array Float", @@ -73,7 +73,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: true, - ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.nested_num[*].float_nums[*]'), '" + ARRAY_FLOAT64 + "')", + ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"nested_num\"[*].\"float_nums\"[*]'), '" + ARRAY_FLOAT64 + "')", }, { Name: "Array Bool", @@ -83,7 +83,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: true, - ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.boolarray[*]'), '" + ARRAY_BOOL + "')", + ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"boolarray\"[*]'), '" + ARRAY_BOOL + "')", }, { Name: "String", @@ -93,7 +93,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: false, - ClickhouseKey: "JSON_VALUE(body, '$.message')", + ClickhouseKey: "JSON_VALUE(body, '$.\"message\"')", }, { Name: "Int", @@ -103,7 +103,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: false, - ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "')", + ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + INT64 + "')", }, { Name: "Float", @@ -113,7 +113,7 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: false, - ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.fraction'), '" + FLOAT64 + "')", + ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"fraction\"'), '" + FLOAT64 + "')", }, { Name: "Bool", @@ -123,7 +123,17 @@ var testGetJSONFilterKeyData = []struct { IsJSON: true, }, IsArray: false, - ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "')", + ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"boolkey\"'), '" + BOOL + "')", + }, + { + Name: "Key with dash", + Key: v3.AttributeKey{ + Key: "body.bool-key", + DataType: "bool", + IsJSON: true, + }, + IsArray: false, + ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"bool-key\"'), '" + BOOL + "')", }, } @@ -158,7 +168,7 @@ var testGetJSONFilterData = []struct { Operator: "has", Value: "index_service", }, - Filter: "has(JSONExtract(JSON_QUERY(body, '$.requestor_list[*]'), 'Array(String)'), 'index_service')", + Filter: "has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service')", }, { Name: "Array membership int64", @@ -171,7 +181,7 @@ var testGetJSONFilterData = []struct { Operator: "has", Value: 2, }, - Filter: "has(JSONExtract(JSON_QUERY(body, '$.int_numbers[*]'), '" + ARRAY_INT64 + "'), 2)", + Filter: "has(JSONExtract(JSON_QUERY(body, '$.\"int_numbers\"[*]'), '" + ARRAY_INT64 + "'), 2)", }, { Name: "Array membership float64", @@ -184,7 +194,7 @@ var testGetJSONFilterData = []struct { Operator: "nhas", Value: 2.2, }, - Filter: "NOT has(JSONExtract(JSON_QUERY(body, '$.nested_num[*].float_nums[*]'), '" + ARRAY_FLOAT64 + "'), 2.200000)", + Filter: "NOT has(JSONExtract(JSON_QUERY(body, '$.\"nested_num\"[*].\"float_nums\"[*]'), '" + ARRAY_FLOAT64 + "'), 2.200000)", }, { Name: "Array membership bool", @@ -197,7 +207,7 @@ var testGetJSONFilterData = []struct { Operator: "has", Value: true, }, - Filter: "has(JSONExtract(JSON_QUERY(body, '$.bool[*]'), '" + ARRAY_BOOL + "'), true)", + Filter: "has(JSONExtract(JSON_QUERY(body, '$.\"bool\"[*]'), '" + ARRAY_BOOL + "'), true)", }, { Name: "eq operator", @@ -210,7 +220,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: "hello", }, - Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') = 'hello'", + Filter: "JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') = 'hello'", }, { Name: "eq operator number", @@ -223,7 +233,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: 1, }, - Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1", + Filter: "JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + INT64 + "') = 1", }, { Name: "neq operator number", @@ -236,7 +246,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: 1.1, }, - Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000", + Filter: "JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + FLOAT64 + "') = 1.100000", }, { Name: "eq operator bool", @@ -249,7 +259,7 @@ var testGetJSONFilterData = []struct { Operator: "=", Value: true, }, - Filter: "JSON_EXISTS(body, '$.boolkey') AND JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true", + Filter: "JSON_EXISTS(body, '$.\"boolkey\"') AND JSONExtract(JSON_VALUE(body, '$.\"boolkey\"'), '" + BOOL + "') = true", }, { Name: "greater than operator", @@ -262,7 +272,7 @@ var testGetJSONFilterData = []struct { Operator: ">", Value: 1, }, - Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1", + Filter: "JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + INT64 + "') > 1", }, { Name: "regex operator", @@ -275,7 +285,7 @@ var testGetJSONFilterData = []struct { Operator: "regex", Value: "a*", }, - Filter: "JSON_EXISTS(body, '$.message') AND match(JSON_VALUE(body, '$.message'), 'a*')", + Filter: "JSON_EXISTS(body, '$.\"message\"') AND match(JSON_VALUE(body, '$.\"message\"'), 'a*')", }, { Name: "contains operator", @@ -288,7 +298,7 @@ var testGetJSONFilterData = []struct { Operator: "contains", Value: "a", }, - Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%'", + Filter: "JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') ILIKE '%a%'", }, { Name: "exists", @@ -301,7 +311,7 @@ var testGetJSONFilterData = []struct { Operator: "exists", Value: "", }, - Filter: "JSON_EXISTS(body, '$.message')", + Filter: "JSON_EXISTS(body, '$.\"message\"')", }, } diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index 88d98847c8..1db46e1374 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -911,7 +911,7 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", + ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", }, { Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy", @@ -945,7 +945,7 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(JSONExtract(JSON_QUERY(body, '$.requestor_list[*]'), 'Array(String)'), 'index_service') AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", + ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service') AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", }, }