GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-14 03:46:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: exists check for json filters added (#3675)

Browse Source 
* fix: exists check for json filters added

* fix: comment updated
This commit is contained in:
Nityananda Gohain 2023-10-06 09:26:37 +05:30 committed by GitHub
parent a306fb64cb
commit abed60bdfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 63 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pkg/query-service/app/logs/v3/json_filter.go
View File

@ -52,11 +52,13 @@ var jsonLogOperators = map[v3.FilterOperator]string{
v3.FilterOperatorNotRegex: "NOT match(%s, %s)", v3.FilterOperatorNotRegex: "NOT match(%s, %s)",
v3.FilterOperatorIn: "IN", v3.FilterOperatorIn: "IN",
v3.FilterOperatorNotIn: "NOT IN", v3.FilterOperatorNotIn: "NOT IN",
v3.FilterOperatorExists: "JSON_EXISTS(%s, '$.%s')",
v3.FilterOperatorNotExists: "NOT JSON_EXISTS(%s, '$.%s')",
v3.FilterOperatorHas: "has(%s, %s)", v3.FilterOperatorHas: "has(%s, %s)",
v3.FilterOperatorNotHas: "NOT has(%s, %s)", v3.FilterOperatorNotHas: "NOT has(%s, %s)",
} }
func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) { func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) {
keyArr := strings.Split(key.Key, ".") keyArr := strings.Split(key.Key, ".")
if len(keyArr) < 2 { if len(keyArr) < 2 {
return "", fmt.Errorf("incorrect key, should contain at least 2 parts") return "", fmt.Errorf("incorrect key, should contain at least 2 parts")
@ -67,6 +69,10 @@ func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) {
return "", fmt.Errorf("only body can be the root key") return "", fmt.Errorf("only body can be the root key")
} }
if op == v3.FilterOperatorExists || op == v3.FilterOperatorNotExists {
return keyArr[0], nil
}
var dataType string var dataType string
var ok bool var ok bool
if dataType, ok = dataTypeMapping[string(key.DataType)]; !ok { if dataType, ok = dataTypeMapping[string(key.DataType)]; !ok {
@ -99,29 +105,45 @@ func GetJSONFilter(item v3.FilterItem) (string, error) {
dataType = v3.AttributeKeyDataType(val) dataType = v3.AttributeKeyDataType(val)
} }
key, err := getJSONFilterKey(item.Key, isArray) key, err := getJSONFilterKey(item.Key, item.Operator, isArray)
if err != nil { if err != nil {
return "", err return "", err
} }
// non array // non array
value, err := utils.ValidateAndCastValue(item.Value, dataType)
if err != nil {
return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
}
op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator))))
if logsOp, ok := jsonLogOperators[op]; ok {
switch op { var value interface{}
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas: if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists {
fmtVal := utils.ClickHouseFormattedValue(value) value, err = utils.ValidateAndCastValue(item.Value, dataType)
return fmt.Sprintf(logsOp, key, fmtVal), nil if err != nil {
case v3.FilterOperatorContains, v3.FilterOperatorNotContains: return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
return fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value), nil
default:
fmtVal := utils.ClickHouseFormattedValue(value)
return fmt.Sprintf("%s %s %s", key, logsOp, fmtVal), nil
} }
} }
return "", fmt.Errorf("unsupported operator: %s", op)
var filter string
if logsOp, ok := jsonLogOperators[op]; ok {
switch op {
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
filter = fmt.Sprintf(logsOp, key, strings.Join(strings.Split(item.Key.Key, ".")[1:], "."))
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas:
fmtVal := utils.ClickHouseFormattedValue(value)
filter = fmt.Sprintf(logsOp, key, fmtVal)
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value)
default:
fmtVal := utils.ClickHouseFormattedValue(value)
filter = fmt.Sprintf("%s %s %s", key, logsOp, fmtVal)
}
} else {
return "", fmt.Errorf("unsupported operator: %s", op)
}
// add exists check for non array items as default values of int/float/bool will corrupt the results
if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) {
existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", strings.Join(strings.Split(item.Key.Key, ".")[1:], "."))
filter = fmt.Sprintf("%s AND %s", existsFilter, filter)
}
return filter, nil
} }

30
pkg/query-service/app/logs/v3/json_filter_test.go
View File

@ -12,6 +12,7 @@ var testGetJSONFilterKeyData = []struct {
Key v3.AttributeKey Key v3.AttributeKey
IsArray bool IsArray bool
ClickhouseKey string ClickhouseKey string
Operator v3.FilterOperator
Error bool Error bool
}{ }{
{ {
@ -129,7 +130,7 @@ var testGetJSONFilterKeyData = []struct {
func TestGetJSONFilterKey(t *testing.T) { func TestGetJSONFilterKey(t *testing.T) {
for _, tt := range testGetJSONFilterKeyData { for _, tt := range testGetJSONFilterKeyData {
Convey("testgetKey", t, func() { Convey("testgetKey", t, func() {
columnName, err := getJSONFilterKey(tt.Key, tt.IsArray) columnName, err := getJSONFilterKey(tt.Key, tt.Operator, tt.IsArray)
if tt.Error { if tt.Error {
So(err, ShouldNotBeNil) So(err, ShouldNotBeNil)
} else { } else {
@ -209,7 +210,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: "hello", Value: "hello",
}, },
Filter: "JSON_VALUE(body, '$.message') = 'hello'", Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') = 'hello'",
}, },
{ {
Name: "eq operator number", Name: "eq operator number",
@ -222,7 +223,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: 1, Value: 1,
}, },
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1", Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1",
}, },
{ {
Name: "neq operator number", Name: "neq operator number",
@ -235,7 +236,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: 1.1, Value: 1.1,
}, },
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000", Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000",
}, },
{ {
Name: "eq operator bool", Name: "eq operator bool",
@ -248,7 +249,7 @@ var testGetJSONFilterData = []struct {
Operator: "=", Operator: "=",
Value: true, Value: true,
}, },
Filter: "JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true", Filter: "JSON_EXISTS(body, '$.boolkey') AND JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true",
}, },
{ {
Name: "greater than operator", Name: "greater than operator",
@ -261,7 +262,7 @@ var testGetJSONFilterData = []struct {
Operator: ">", Operator: ">",
Value: 1, Value: 1,
}, },
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1", Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1",
}, },
{ {
Name: "regex operator", Name: "regex operator",
@ -274,7 +275,7 @@ var testGetJSONFilterData = []struct {
Operator: "regex", Operator: "regex",
Value: "a*", Value: "a*",
}, },
Filter: "match(JSON_VALUE(body, '$.message'), 'a*')", Filter: "JSON_EXISTS(body, '$.message') AND match(JSON_VALUE(body, '$.message'), 'a*')",
}, },
{ {
Name: "contains operator", Name: "contains operator",
@ -287,7 +288,20 @@ var testGetJSONFilterData = []struct {
Operator: "contains", Operator: "contains",
Value: "a", Value: "a",
}, },
Filter: "JSON_VALUE(body, '$.message') ILIKE '%a%'", Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%'",
},
{
Name: "exists",
FilterItem: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.message",
DataType: "string",
IsJSON: true,
},
Operator: "exists",
Value: "",
},
Filter: "JSON_EXISTS(body, '$.message')",
}, },
} }

2
pkg/query-service/app/logs/v3/query_builder_test.go
View File

@ -908,7 +908,7 @@ var testBuildLogsQueryData = []struct {
}, },
}, },
TableName: "logs", TableName: "logs",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC", ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
}, },
{ {
Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy", Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy",