GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-12 17:19:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: exists check for json filters added (#3675)

Browse Source 
* fix: exists check for json filters added

* fix: comment updated
This commit is contained in:
Nityananda Gohain 2023-10-06 09:26:37 +05:30 committed by GitHub
parent a306fb64cb
commit abed60bdfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 63 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pkg/query-service/app/logs/v3/json_filter.go
View File

@ -52,11 +52,13 @@ var jsonLogOperators = map[v3.FilterOperator]string{
v3.FilterOperatorNotRegex: "NOT match(%s, %s)",
v3.FilterOperatorIn: "IN",
v3.FilterOperatorNotIn: "NOT IN",
v3.FilterOperatorExists: "JSON_EXISTS(%s, '$.%s')",
v3.FilterOperatorNotExists: "NOT JSON_EXISTS(%s, '$.%s')",
v3.FilterOperatorHas: "has(%s, %s)",
v3.FilterOperatorNotHas: "NOT has(%s, %s)",
}
func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) {
func getJSONFilterKey(key v3.AttributeKey, op v3.FilterOperator, isArray bool) (string, error) {
keyArr := strings.Split(key.Key, ".")
if len(keyArr) < 2 {
return "", fmt.Errorf("incorrect key, should contain at least 2 parts")
@ -67,6 +69,10 @@ func getJSONFilterKey(key v3.AttributeKey, isArray bool) (string, error) {
return "", fmt.Errorf("only body can be the root key")
}
if op == v3.FilterOperatorExists || op == v3.FilterOperatorNotExists {
return keyArr[0], nil
}
var dataType string
var ok bool
if dataType, ok = dataTypeMapping[string(key.DataType)]; !ok {
@ -99,29 +105,45 @@ func GetJSONFilter(item v3.FilterItem) (string, error) {
dataType = v3.AttributeKeyDataType(val)
}
key, err := getJSONFilterKey(item.Key, isArray)
key, err := getJSONFilterKey(item.Key, item.Operator, isArray)
if err != nil {
return "", err
}
// non array
value, err := utils.ValidateAndCastValue(item.Value, dataType)
if err != nil {
return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
}
op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator))))
if logsOp, ok := jsonLogOperators[op]; ok {
switch op {
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas:
fmtVal := utils.ClickHouseFormattedValue(value)
return fmt.Sprintf(logsOp, key, fmtVal), nil
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
return fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value), nil
default:
fmtVal := utils.ClickHouseFormattedValue(value)
return fmt.Sprintf("%s %s %s", key, logsOp, fmtVal), nil
var value interface{}
if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists {
value, err = utils.ValidateAndCastValue(item.Value, dataType)
if err != nil {
return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
}
}
return "", fmt.Errorf("unsupported operator: %s", op)
var filter string
if logsOp, ok := jsonLogOperators[op]; ok {
switch op {
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
filter = fmt.Sprintf(logsOp, key, strings.Join(strings.Split(item.Key.Key, ".")[1:], "."))
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex, v3.FilterOperatorHas, v3.FilterOperatorNotHas:
fmtVal := utils.ClickHouseFormattedValue(value)
filter = fmt.Sprintf(logsOp, key, fmtVal)
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value)
default:
fmtVal := utils.ClickHouseFormattedValue(value)
filter = fmt.Sprintf("%s %s %s", key, logsOp, fmtVal)
}
} else {
return "", fmt.Errorf("unsupported operator: %s", op)
}
// add exists check for non array items as default values of int/float/bool will corrupt the results
if !isArray && !(item.Operator == v3.FilterOperatorExists || item.Operator == v3.FilterOperatorNotExists) {
existsFilter := fmt.Sprintf("JSON_EXISTS(body, '$.%s')", strings.Join(strings.Split(item.Key.Key, ".")[1:], "."))
filter = fmt.Sprintf("%s AND %s", existsFilter, filter)
}
return filter, nil
}

30
pkg/query-service/app/logs/v3/json_filter_test.go
View File

@ -12,6 +12,7 @@ var testGetJSONFilterKeyData = []struct {
Key v3.AttributeKey
IsArray bool
ClickhouseKey string
Operator v3.FilterOperator
Error bool
}{
{
@ -129,7 +130,7 @@ var testGetJSONFilterKeyData = []struct {
func TestGetJSONFilterKey(t *testing.T) {
for _, tt := range testGetJSONFilterKeyData {
Convey("testgetKey", t, func() {
columnName, err := getJSONFilterKey(tt.Key, tt.IsArray)
columnName, err := getJSONFilterKey(tt.Key, tt.Operator, tt.IsArray)
if tt.Error {
So(err, ShouldNotBeNil)
} else {
@ -209,7 +210,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: "hello",
},
Filter: "JSON_VALUE(body, '$.message') = 'hello'",
Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') = 'hello'",
},
{
Name: "eq operator number",
@ -222,7 +223,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: 1,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1",
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') = 1",
},
{
Name: "neq operator number",
@ -235,7 +236,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: 1.1,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000",
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + FLOAT64 + "') = 1.100000",
},
{
Name: "eq operator bool",
@ -248,7 +249,7 @@ var testGetJSONFilterData = []struct {
Operator: "=",
Value: true,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true",
Filter: "JSON_EXISTS(body, '$.boolkey') AND JSONExtract(JSON_VALUE(body, '$.boolkey'), '" + BOOL + "') = true",
},
{
Name: "greater than operator",
@ -261,7 +262,7 @@ var testGetJSONFilterData = []struct {
Operator: ">",
Value: 1,
},
Filter: "JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1",
Filter: "JSON_EXISTS(body, '$.status') AND JSONExtract(JSON_VALUE(body, '$.status'), '" + INT64 + "') > 1",
},
{
Name: "regex operator",
@ -274,7 +275,7 @@ var testGetJSONFilterData = []struct {
Operator: "regex",
Value: "a*",
},
Filter: "match(JSON_VALUE(body, '$.message'), 'a*')",
Filter: "JSON_EXISTS(body, '$.message') AND match(JSON_VALUE(body, '$.message'), 'a*')",
},
{
Name: "contains operator",
@ -287,7 +288,20 @@ var testGetJSONFilterData = []struct {
Operator: "contains",
Value: "a",
},
Filter: "JSON_VALUE(body, '$.message') ILIKE '%a%'",
Filter: "JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%'",
},
{
Name: "exists",
FilterItem: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.message",
DataType: "string",
IsJSON: true,
},
Operator: "exists",
Value: "",
},
Filter: "JSON_EXISTS(body, '$.message')",
},
}

2
pkg/query-service/app/logs/v3/query_builder_test.go
View File

@ -908,7 +908,7 @@ var testBuildLogsQueryData = []struct {
},
},
TableName: "logs",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as name, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND JSON_EXISTS(body, '$.message') AND JSON_VALUE(body, '$.message') ILIKE '%a%' AND indexOf(attributes_string_key, 'name') > 0 group by name order by name DESC",
},
{
Name: "TABLE: Test count with JSON Filter Array, groupBy, orderBy",