From ab444af8e66ac36b07acb61fdcfb1408fa518e7c Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Mon, 27 May 2024 15:18:49 +0530 Subject: [PATCH] fix: escape string for contains and ncontains (#5083) * fix: escape string for contains and ncontains * fix: add changes to json and traces builder --------- Co-authored-by: Srikanth Chekuri --- pkg/query-service/app/logs/v3/json_filter.go | 3 ++- pkg/query-service/app/logs/v3/json_filter_test.go | 13 +++++++++++++ pkg/query-service/app/logs/v3/query_builder.go | 3 ++- pkg/query-service/app/logs/v3/query_builder_test.go | 8 +++++++- pkg/query-service/app/traces/v3/query_builder.go | 5 +++-- .../app/traces/v3/query_builder_test.go | 7 +++++++ pkg/query-service/utils/format.go | 6 +++--- 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/pkg/query-service/app/logs/v3/json_filter.go b/pkg/query-service/app/logs/v3/json_filter.go index bcc4b68a14..a9acdeaab3 100644 --- a/pkg/query-service/app/logs/v3/json_filter.go +++ b/pkg/query-service/app/logs/v3/json_filter.go @@ -144,7 +144,8 @@ func GetJSONFilter(item v3.FilterItem) (string, error) { fmtVal := utils.ClickHouseFormattedValue(value) filter = fmt.Sprintf(logsOp, key, fmtVal) case v3.FilterOperatorContains, v3.FilterOperatorNotContains: - filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, item.Value) + val := utils.QuoteEscapedString(fmt.Sprintf("%v", item.Value)) + filter = fmt.Sprintf("%s %s '%%%s%%'", key, logsOp, val) default: fmtVal := utils.ClickHouseFormattedValue(value) filter = fmt.Sprintf("%s %s %s", key, logsOp, fmtVal) diff --git a/pkg/query-service/app/logs/v3/json_filter_test.go b/pkg/query-service/app/logs/v3/json_filter_test.go index 2992b1e273..ac9d8edbf4 100644 --- a/pkg/query-service/app/logs/v3/json_filter_test.go +++ b/pkg/query-service/app/logs/v3/json_filter_test.go @@ -300,6 +300,19 @@ var testGetJSONFilterData = []struct { }, Filter: "JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') ILIKE '%a%'", }, + { + Name: "contains operator with quotes", + FilterItem: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.message", + DataType: "string", + IsJSON: true, + }, + Operator: "contains", + Value: "hello 'world'", + }, + Filter: "JSON_EXISTS(body, '$.\"message\"') AND JSON_VALUE(body, '$.\"message\"') ILIKE '%hello \\'world\\'%'", + }, { Name: "exists", FilterItem: v3.FilterItem{ diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index 75cacaa2ed..06cacc8f60 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -192,7 +192,8 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey, conditions = append(conditions, fmt.Sprintf(logsOp, columnName, fmtVal)) case v3.FilterOperatorContains, v3.FilterOperatorNotContains: columnName := getClickhouseColumnName(item.Key) - conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, logsOp, item.Value)) + val := utils.QuoteEscapedString(fmt.Sprintf("%v", item.Value)) + conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, logsOp, val)) default: columnName := getClickhouseColumnName(item.Key) fmtVal := utils.ClickHouseFormattedValue(value) diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index 7a8a997be4..606ccffeef 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -187,6 +187,13 @@ var timeSeriesFilterQueryData = []struct { }}, ExpectedFilter: "attributes_string_value[indexOf(attributes_string_key, 'host')] ILIKE '%102.%'", }, + { + Name: "Test contains with single quotes", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "hello 'world'", Operator: "contains"}, + }}, + ExpectedFilter: "attributes_string_value[indexOf(attributes_string_key, 'message')] ILIKE '%hello \\'world\\'%'", + }, { Name: "Test not contains", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -280,7 +287,6 @@ var timeSeriesFilterQueryData = []struct { }}, ExpectedFilter: "`attribute_int64_status_exists`=false", }, - // add new tests } func TestBuildLogsTimeSeriesFilterQuery(t *testing.T) { diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index efbc4d8872..41f64548ad 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -174,7 +174,8 @@ func buildTracesFilterQuery(fs *v3.FilterSet, keys map[string]v3.AttributeKey) ( if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { switch item.Operator { case v3.FilterOperatorContains, v3.FilterOperatorNotContains: - conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, item.Value)) + val = utils.QuoteEscapedString(fmt.Sprintf("%v", item.Value)) + conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, val)) case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal)) case v3.FilterOperatorExists, v3.FilterOperatorNotExists: @@ -263,7 +264,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str queryTmpl = "SELECT now() as ts," // step or aggregate interval is whole time period in case of table panel - step = (end*getZerosForEpochNano(end) - start*getZerosForEpochNano(start))/1000000000 + step = (end*getZerosForEpochNano(end) - start*getZerosForEpochNano(start)) / 1000000000 } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { // Select the aggregate value for interval queryTmpl = diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index b4b1a2574c..cf4eb21cfe 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -100,6 +100,13 @@ var buildFilterQueryData = []struct { }}, ExpectedFilter: " AND stringTagMap['host'] ILIKE '%102.%'", }, + { + Name: "Test contains with quotes", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "Hello 'world'", Operator: "contains"}, + }}, + ExpectedFilter: " AND stringTagMap['message'] ILIKE '%Hello \\'world\\'%'", + }, { Name: "Test not contains", FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go index 0adaebff4a..481590e761 100644 --- a/pkg/query-service/utils/format.go +++ b/pkg/query-service/utils/format.go @@ -143,7 +143,7 @@ func ValidateAndCastValue(v interface{}, dataType v3.AttributeKeyDataType) (inte } } -func quoteEscapedString(str string) string { +func QuoteEscapedString(str string) string { // https://clickhouse.com/docs/en/sql-reference/syntax#string str = strings.ReplaceAll(str, `\`, `\\`) str = strings.ReplaceAll(str, `'`, `\'`) @@ -161,7 +161,7 @@ func ClickHouseFormattedValue(v interface{}) string { case float32, float64: return fmt.Sprintf("%f", x) case string: - return fmt.Sprintf("'%s'", quoteEscapedString(x)) + return fmt.Sprintf("'%s'", QuoteEscapedString(x)) case bool: return fmt.Sprintf("%v", x) @@ -173,7 +173,7 @@ func ClickHouseFormattedValue(v interface{}) string { case string: str := "[" for idx, sVal := range x { - str += fmt.Sprintf("'%s'", quoteEscapedString(sVal.(string))) + str += fmt.Sprintf("'%s'", QuoteEscapedString(sVal.(string))) if idx != len(x)-1 { str += "," }