From a4a285c074b13087e17a1c5fa96c2118bca97a67 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Tue, 26 Sep 2023 20:10:39 +0530 Subject: [PATCH] feat: add support for freehand json query (#3625) * feat: freehand json search * feat: support for freehand json query * fix: minor updates * fix: minor refactor --- pkg/query-service/app/logs/v3/enrich_query.go | 64 +++++++- .../app/logs/v3/enrich_query_test.go | 155 +++++++++++++++++- 2 files changed, 214 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/logs/v3/enrich_query.go b/pkg/query-service/app/logs/v3/enrich_query.go index c9a133ce0f..fe8ed8b9ed 100644 --- a/pkg/query-service/app/logs/v3/enrich_query.go +++ b/pkg/query-service/app/logs/v3/enrich_query.go @@ -1,6 +1,10 @@ package v3 import ( + "fmt" + "strconv" + "strings" + "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) @@ -27,9 +31,6 @@ func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool { // check filter attribute if query.Filters != nil && len(query.Filters.Items) != 0 { for _, item := range query.Filters.Items { - if item.Key.IsJSON { - continue - } if !isEnriched(item.Key) { return true } @@ -100,6 +101,7 @@ func enrichLogsQuery(query *v3.BuilderQuery, fields map[string]v3.AttributeKey) // enrich filter attribute if query.Filters != nil && len(query.Filters.Items) != 0 { for i := 0; i < len(query.Filters.Items); i++ { + query.Filters.Items[i] = jsonFilterEnrich(query.Filters.Items[i]) if query.Filters.Items[i].Key.IsJSON { continue } @@ -149,3 +151,59 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu field.DataType = v3.AttributeKeyDataTypeString return field } + +func jsonFilterEnrich(filter v3.FilterItem) v3.FilterItem { + // check if it is a json request + if !strings.HasPrefix(filter.Key.Key, "body.") { + return filter + } + + // check if the value is a int, float, string, bool + valueType := "" + switch filter.Value.(type) { + case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64: + valueType = "int64" + case float32, float64: + valueType = "float64" + case string: + valueType, filter.Value = parseStrValue(filter.Value.(string), filter.Operator) + case bool: + valueType = "bool" + } + + // check if it is array + if strings.HasSuffix(filter.Key.Key, "[*]") { + valueType = fmt.Sprintf("array(%s)", valueType) + } + + filter.Key.DataType = v3.AttributeKeyDataType(valueType) + filter.Key.IsJSON = true + return filter +} + +func parseStrValue(valueStr string, operator v3.FilterOperator) (string, interface{}) { + + valueType := "string" + + // for the following operators it will always be string + if operator == v3.FilterOperatorContains || operator == v3.FilterOperatorNotContains || + operator == v3.FilterOperatorRegex || operator == v3.FilterOperatorNotRegex || + operator == v3.FilterOperatorLike || operator == v3.FilterOperatorNotLike { + return valueType, valueStr + } + + var err error + var parsedValue interface{} + if parsedValue, err = strconv.ParseBool(valueStr); err == nil { + valueType = "bool" + } else if parsedValue, err = strconv.ParseInt(valueStr, 10, 64); err == nil { + valueType = "int64" + } else if parsedValue, err = strconv.ParseFloat(valueStr, 64); err == nil { + valueType = "float64" + } else { + parsedValue = valueStr + valueType = "string" + } + + return valueType, parsedValue +} diff --git a/pkg/query-service/app/logs/v3/enrich_query_test.go b/pkg/query-service/app/logs/v3/enrich_query_test.go index 6d1ba3953b..c1556534e1 100644 --- a/pkg/query-service/app/logs/v3/enrich_query_test.go +++ b/pkg/query-service/app/logs/v3/enrich_query_test.go @@ -96,13 +96,13 @@ var testEnrichmentRequiredData = []struct { Expression: "test", DataSource: v3.DataSourceLogs, Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "user_name", IsJSON: true}, Value: "john", Operator: "="}, + {Key: v3.AttributeKey{Key: "body.xyz", IsJSON: true, DataType: v3.AttributeKeyDataTypeString}, Value: "john", Operator: "="}, }}, }, }, }, }, - EnrichmentRequired: false, + EnrichmentRequired: true, }, { Name: "groupBy enrichment not required", @@ -289,3 +289,154 @@ func TestEnrichParams(t *testing.T) { }) } } + +var testJSONFilterEnrichData = []struct { + Name string + Filter v3.FilterItem + Result v3.FilterItem +}{ + { + Name: "array string", + Filter: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.requestor_list[*]", + DataType: v3.AttributeKeyDataTypeUnspecified, + Type: v3.AttributeKeyTypeUnspecified, + }, + Operator: "has", + Value: "index_service", + }, + Result: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.requestor_list[*]", + DataType: v3.AttributeKeyDataTypeArrayString, + Type: v3.AttributeKeyTypeUnspecified, + IsJSON: true, + }, + Operator: "has", + Value: "index_service", + }, + }, + { + Name: "int64", + Filter: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.intx", + DataType: v3.AttributeKeyDataTypeUnspecified, + Type: v3.AttributeKeyTypeUnspecified, + }, + Operator: "=", + Value: 10, + }, + Result: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.intx", + DataType: v3.AttributeKeyDataTypeInt64, + Type: v3.AttributeKeyTypeUnspecified, + IsJSON: true, + }, + Operator: "=", + Value: 10, + }, + }, + { + Name: "float64", + Filter: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.float64[*]", + DataType: v3.AttributeKeyDataTypeArrayFloat64, + Type: v3.AttributeKeyTypeUnspecified, + }, + Operator: "!=", + Value: 10.0, + }, + Result: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.float64[*]", + DataType: v3.AttributeKeyDataTypeArrayFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsJSON: true, + }, + Operator: "!=", + Value: 10.0, + }, + }, + { + Name: "float64x", + Filter: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.float64x", + DataType: v3.AttributeKeyDataTypeUnspecified, + Type: v3.AttributeKeyTypeUnspecified, + }, + Operator: "!=", + Value: "10.0", + }, + Result: v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body.float64x", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsJSON: true, + }, + Operator: "!=", + Value: 10.0, + }, + }, +} + +func TestJsonEnrich(t *testing.T) { + for _, tt := range testJSONFilterEnrichData { + Convey(tt.Name, t, func() { + res := jsonFilterEnrich(tt.Filter) + So(res, ShouldResemble, tt.Result) + }) + } +} + +var testParseStrValueData = []struct { + Name string + Operator v3.FilterOperator + Value interface{} + ResultType string + Result interface{} +}{ + { + Name: "bool", + Value: "true", + Operator: v3.FilterOperatorEqual, + ResultType: "bool", + Result: true, + }, + { + Name: "int", + Value: "10", + Operator: v3.FilterOperatorNotEqual, + ResultType: "int64", + Result: 10, + }, + { + Name: "float", + Value: "10.0", + Operator: v3.FilterOperatorGreaterThan, + ResultType: "float64", + Result: 10.0, + }, + { + Name: "string", + Value: "hello", + Operator: v3.FilterOperatorLessThan, + ResultType: "string", + Result: "hello", + }, +} + +func TestParseStrValue(t *testing.T) { + for _, tt := range testParseStrValueData { + Convey(tt.Name, t, func() { + vtype, value := parseStrValue(tt.Value.(string), tt.Operator) + So(vtype, ShouldEqual, tt.ResultType) + So(value, ShouldEqual, tt.Result) + }) + } +}