GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-12 19:09:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: support for dash in path name for json query (#3861)

Browse Source
This commit is contained in:
Nityananda Gohain 2023-11-01 18:13:28 +05:30 committed by GitHub
parent 64f0ff05f9
commit b958a06ba0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 51 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/query-service/app/logs/v3/json_filter.go
View File

@ -58,6 +58,18 @@ var jsonLogOperators = map[v3.FilterOperator]string{
v3.FilterOperatorNotHas: "NOT has(%s, %s)", v3.FilterOperatorNotHas: "NOT has(%s, %s)",
} }
func getPath(keyArr []string) string {
path := []string{}
for i := 0; i < len(keyArr); i++ {
if strings.HasSuffix(keyArr[i], "[*]") {
path = append(path, "\""+strings.TrimSuffix(keyArr[i], "[*]")+"\""+"[*]")
} else {
path = append(path, "\""+keyArr[i]+"\"")
}
}
return strings.Join(path, ".")
}
func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) { func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) {
keyArr := strings.Split(key.Key, ".") keyArr := strings.Split(key.Key, ".")
if len(keyArr) < 2 { if len(keyArr) < 2 {
@ -79,12 +91,14 @@ func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (
return "", fmt.Errorf("unsupported dataType for JSON: %s", key.DataType) return "", fmt.Errorf("unsupported dataType for JSON: %s", key.DataType)
} }
path := getPath(keyArr[1:])
if isArray { if isArray {
return fmt.Sprintf("JSONExtract(JSON_QUERY(%s, '$.%s'), '%s')", keyArr[0], strings.Join(keyArr[1:], "."), dataType), nil return fmt.Sprintf("JSONExtract(JSON_QUERY(%s, '$.%s'), '%s')", keyArr[0], path, dataType), nil
} }
// for non array // for non array
keyname := fmt.Sprintf("JSON_VALUE(%s, '$.%s')", keyArr[0], strings.Join(keyArr[1:], ".")) keyname := fmt.Sprintf("JSON_VALUE(%s, '$.%s')", keyArr[0], path)
if dataType != STRING { if dataType != STRING {
keyname = fmt.Sprintf("JSONExtract(%s, '%s')", keyname, dataType) keyname = fmt.Sprintf("JSONExtract(%s, '%s')", keyname, dataType)
} }
@ -125,7 +139,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) {
if logsOp, ok := jsonLogOperators[op]; ok { if logsOp, ok := jsonLogOperators[op]; ok {
switch op { switch op {
case v3.FilterOperatorExists, v3.FilterOperatorNotExists: case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
filter = fmt.Sprintf(logsOp, key, strings.Join(strings.Split(item.Key.Key, ".")[1:], ".")) filter = fmt.Sprintf(logsOp, key, getPath(strings.Split(item.Key.Key, ".")[1:]))
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas:
fmtVal := utils.ClickHouseFormattedValue(value) fmtVal := utils.ClickHouseFormattedValue(value)
filter = fmt.Sprintf(logsOp, key, fmtVal) filter = fmt.Sprintf(logsOp, key, fmtVal)
@ -141,7 +155,7 @@ func GetJSONFilter(item v3.FilterItem) (string, error) {
// add exists check for non array items as default values of int/float/bool will corrupt the results // add exists check for non array items as default values of int/float/bool will corrupt the results
if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) { if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) {
existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", strings.Join(strings.Split(item.Key.Key, ".")[1:], ".")) existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", getPath(strings.Split(item.Key.Key, ".")[1:]))
filter = fmt.Sprintf("%s AND %s", existsFilter, filter) filter = fmt.Sprintf("%s AND %s", existsFilter, filter)
} }

52
pkg/query-service/app/logs/v3/json_filter_test.go
View File

@ -43,7 +43,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: true, IsArray: true,
ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.requestor_list[*]'), '" + ARRAY_STRING + "')", ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), '" + ARRAY_STRING + "')",
}, },
{ {
Name: "Array String Nested", Name: "Array String Nested",
@ -53,7 +53,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: true, IsArray: true,
ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.nested[*].key[*]'), '" + ARRAY_STRING + "')", ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"nested\"[*].\"key\"[*]'), '" + ARRAY_STRING + "')",
}, },
{ {
Name: "Array Int", Name: "Array Int",
@ -63,7 +63,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: true, IsArray: true,
ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.int_numbers[*]'), '" + ARRAY_INT64 + "')", ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"int_numbers\"[*]'), '" + ARRAY_INT64 + "')",
}, },
{ {
Name: "Array Float", Name: "Array Float",
@ -73,7 +73,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: true, IsArray: true,
ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.nested_num[*].float_nums[*]'), '" + ARRAY_FLOAT64 + "')", ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"nested_num\"[*].\"float_nums\"[*]'), '" + ARRAY_FLOAT64 + "')",
}, },
{ {
Name: "Array Bool", Name: "Array Bool",
@ -83,7 +83,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: true, IsArray: true,
ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.boolarray[*]'), '" + ARRAY_BOOL + "')", ClickhouseKey: "JSONExtract(JSON_QUERY(body, '$.\"boolarray\"[*]'), '" + ARRAY_BOOL + "')",
}, },
{ {
Name: "String", Name: "String",
@ -93,7 +93,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: false, IsArray: false,
ClickhouseKey: "JSON_VALUE(body, '$.message')", ClickhouseKey: "JSON_VALUE(body, '$.\"message\"')",
}, },
{ {
Name: "Int", Name: "Int",
@ -103,7 +103,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: false, IsArray: false,
ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "')", ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + INT64 + "')",
}, },
{ {
Name: "Float", Name: "Float",
@ -113,7 +113,7 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: false, IsArray: false,
ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.fraction'), '" + FLOAT64 + "')", ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"fraction\"'), '" + FLOAT64 + "')",
}, },
{ {
Name: "Bool", Name: "Bool",
@ -123,7 +123,17 @@ var testGetJSONFilterKeyData = []struct {
IsJSON: true, IsJSON: true,
}, },
IsArray: false, IsArray: false,
ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "')", ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"boolkey\"'), '" + BOOL + "')",
},
{
Name: "Key with dash",
Key: v3.AttributeKey{
Key: "body.bool-key",
DataType: "bool",
IsJSON: true,
},
IsArray: false,
ClickhouseKey: "JSONExtract(JSON_VALUE(body, '$.\"bool-key\"'), '" + BOOL + "')",
}, },
} }
@ -158,7 +168,7 @@ var testGetJSONFilterData = []struct {
Operator: "has", Operator: "has",
Value: "index_service", Value: "index_service",
}, },
Filter: "has(JSONExtract(JSON_QUERY(body, '$.requestor_list[*]'), 'Array(String)'), 'index_service')", Filter: "has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service')",
}, },
{ {
Name: "Array membership int64", Name: "Array membership int64",
@ -171,7 +181,7 @@ var testGetJSONFilterData = []struct {
Operator: "has", Operator: "has",
Value: 2, Value: 2,
}, },
Filter: "has(JSONExtract(JSON_QUERY(body, '$.int_numbers[*]'), '" + ARRAY_INT64 + "'), 2)", Filter: "has(JSONExtract(JSON_QUERY(body, '$.\"int_numbers\"[*]'), '" + ARRAY_INT64 + "'), 2)",
}, },
{ {
Name: "Array membership float64", Name: "Array membership float64",
@ -184,7 +194,7 @@ var testGetJSONFilterData = []struct {
Operator: "nhas", Operator: "nhas",
Value: 2.2, Value: 2.2,
}, },
Filter: "NOT has(JSONExtract(JSON_QUERY(body, '$.nested_num[*].float_nums[*]'), '" + ARRAY_FLOAT64 + "'), 2.200000)", Filter: "NOT has(JSONExtract(JSON_QUERY(body, '$.\"nested_num\"[*].\"float_nums\"[*]'), '" + ARRAY_FLOAT64 + "'), 2.200000)",
}, },
{ {
Name: "Array membership bool", Name: "Array membership bool",
@ -197,7 +207,7 @@ var testGetJSONFilterData = []struct {
Operator: "has", Operator: "has",
Value: true, Value: true,
}, },
Filter: "has(JSONExtract(JSON_QUERY(body, '$.bool[*]'), '" + ARRAY_BOOL + "'), true)", Filter: "has(JSONExtract(JSON_QUERY(body, '$.\"bool\"[*]'), '" + ARRAY_BOOL + "'), true)",
}, },
{ {
Name: "eq operator", Name: "eq operator",
@ -210,7 +220,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: "hello", Value: "hello",
}, },
Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') = 'hello'", Filter: "JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') = 'hello'",
}, },
{ {
Name: "eq operator number", Name: "eq operator number",
@ -223,7 +233,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: 1, Value: 1,
}, },
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1", Filter: "JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + INT64 + "') = 1",
}, },
{ {
Name: "neq operator number", Name: "neq operator number",
@ -236,7 +246,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: 1.1, Value: 1.1,
}, },
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000", Filter: "JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + FLOAT64 + "') = 1.100000",
}, },
{ {
Name: "eq operator bool", Name: "eq operator bool",
@ -249,7 +259,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: true, Value: true,
}, },
Filter: "JSON_EXISTS(body, '$.boolkey') AND JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true", Filter: "JSON_EXISTS(body, '$.\"boolkey\"') AND JSONExtract(JSON_VALUE(body, '$.\"boolkey\"'), '" + BOOL + "') = true",
}, },
{ {
Name: "greater than operator", Name: "greater than operator",
@ -262,7 +272,7 @@ var testGetJSONFilterData = []struct {
Operator: ">", Operator: ">",
Value: 1, Value: 1,
}, },
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1", Filter: "JSON_EXISTS(body, '$.\"status\"') AND JSONExtract(JSON_VALUE(body, '$.\"status\"'), '" + INT64 + "') > 1",
}, },
{ {
Name: "regex operator", Name: "regex operator",
@ -275,7 +285,7 @@ var testGetJSONFilterData = []struct {
Operator: "regex", Operator: "regex",
Value: "a*", Value: "a*",
}, },
Filter: "JSON_EXISTS(body, '$.message') AND match(JSON_VALUE(body, '$.message'), 'a*')", Filter: "JSON_EXISTS(body, '$.\"message\"') AND match(JSON_VALUE(body, '$.\"message\"'), 'a*')",
}, },
{ {
Name: "contains operator", Name: "contains operator",
@ -288,7 +298,7 @@ var testGetJSONFilterData = []struct {
Operator: "contains", Operator: "contains",
Value: "a", Value: "a",
}, },
Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%'", Filter: "JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') ILIKE '%a%'",
}, },
{ {
Name: "exists", Name: "exists",
@ -301,7 +311,7 @@ var testGetJSONFilterData = []struct {
Operator: "exists", Operator: "exists",
Value: "", Value: "",
}, },
Filter: "JSON_EXISTS(body, '$.message')", Filter: "JSON_EXISTS(body, '$.\"message\"')",
}, },
} }

4
pkg/query-service/app/logs/v3/query_builder_test.go
View File

@ -911,7 +911,7 @@ var testBuildLogsQueryData = []struct {
}, },
}, },
TableName: "logs", TableName: "logs",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
}, },
{ {
Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy", Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy",
@ -945,7 +945,7 @@ var testBuildLogsQueryData = []struct {
}, },
}, },
TableName: "logs", TableName: "logs",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(JSONExtract(JSON_QUERY(body, '$.requestor_list[*]'), 'Array(String)'), 'index_service') AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(JSONExtract(JSON_QUERY(body, '$.\"requestor_list\"[*]'), 'Array(String)'), 'index_service') AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
}, },
} }