GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-14 06:15:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add support for freehand json query (#3625)

Browse Source 
* feat: freehand json search

* feat: support for freehand json query

* fix: minor updates

* fix: minor refactor
This commit is contained in:
Nityananda Gohain 2023-09-26 20:10:39 +05:30 committed by GitHub
parent a8f8580606
commit a4a285c074
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 214 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
pkg/query-service/app/logs/v3/enrich_query.go
View File

@ -1,6 +1,10 @@
package v3
import (
"fmt"
"strconv"
"strings"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
@ -27,9 +31,6 @@ func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool {
// check filter attribute
if query.Filters != nil && len(query.Filters.Items) != 0 {
for _, item := range query.Filters.Items {
if item.Key.IsJSON {
continue
}
if !isEnriched(item.Key) {
return true
}
@ -100,6 +101,7 @@ func enrichLogsQuery(query *v3.BuilderQuery, fields map[string]v3.AttributeKey)
// enrich filter attribute
if query.Filters != nil && len(query.Filters.Items) != 0 {
for i := 0; i < len(query.Filters.Items); i++ {
query.Filters.Items[i] = jsonFilterEnrich(query.Filters.Items[i])
if query.Filters.Items[i].Key.IsJSON {
continue
}
@ -149,3 +151,59 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu
field.DataType = v3.AttributeKeyDataTypeString
return field
}
func jsonFilterEnrich(filter v3.FilterItem) v3.FilterItem {
// check if it is a json request
if !strings.HasPrefix(filter.Key.Key, "body.") {
return filter
}
// check if the value is a int, float, string, bool
valueType := ""
switch filter.Value.(type) {
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
valueType = "int64"
case float32, float64:
valueType = "float64"
case string:
valueType, filter.Value = parseStrValue(filter.Value.(string), filter.Operator)
case bool:
valueType = "bool"
}
// check if it is array
if strings.HasSuffix(filter.Key.Key, "[*]") {
valueType = fmt.Sprintf("array(%s)", valueType)
}
filter.Key.DataType = v3.AttributeKeyDataType(valueType)
filter.Key.IsJSON = true
return filter
}
func parseStrValue(valueStr string, operator v3.FilterOperator) (string, interface{}) {
valueType := "string"
// for the following operators it will always be string
if operator == v3.FilterOperatorContains || operator == v3.FilterOperatorNotContains ||
operator == v3.FilterOperatorRegex || operator == v3.FilterOperatorNotRegex ||
operator == v3.FilterOperatorLike || operator == v3.FilterOperatorNotLike {
return valueType, valueStr
}
var err error
var parsedValue interface{}
if parsedValue, err = strconv.ParseBool(valueStr); err == nil {
valueType = "bool"
} else if parsedValue, err = strconv.ParseInt(valueStr, 10, 64); err == nil {
valueType = "int64"
} else if parsedValue, err = strconv.ParseFloat(valueStr, 64); err == nil {
valueType = "float64"
} else {
parsedValue = valueStr
valueType = "string"
}
return valueType, parsedValue
}

155
pkg/query-service/app/logs/v3/enrich_query_test.go
View File

@ -96,13 +96,13 @@ var testEnrichmentRequiredData = []struct {
Expression: "test",
DataSource: v3.DataSourceLogs,
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "user_name", IsJSON: true}, Value: "john", Operator: "="},
{Key: v3.AttributeKey{Key: "body.xyz", IsJSON: true, DataType: v3.AttributeKeyDataTypeString}, Value: "john", Operator: "="},
}},
},
},
},
},
EnrichmentRequired: false,
EnrichmentRequired: true,
},
{
Name: "groupBy enrichment not required",
@ -289,3 +289,154 @@ func TestEnrichParams(t *testing.T) {
})
}
}
var testJSONFilterEnrichData = []struct {
Name string
Filter v3.FilterItem
Result v3.FilterItem
}{
{
Name: "array string",
Filter: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.requestor_list[*]",
DataType: v3.AttributeKeyDataTypeUnspecified,
Type: v3.AttributeKeyTypeUnspecified,
},
Operator: "has",
Value: "index_service",
},
Result: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.requestor_list[*]",
DataType: v3.AttributeKeyDataTypeArrayString,
Type: v3.AttributeKeyTypeUnspecified,
IsJSON: true,
},
Operator: "has",
Value: "index_service",
},
},
{
Name: "int64",
Filter: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.intx",
DataType: v3.AttributeKeyDataTypeUnspecified,
Type: v3.AttributeKeyTypeUnspecified,
},
Operator: "=",
Value: 10,
},
Result: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.intx",
DataType: v3.AttributeKeyDataTypeInt64,
Type: v3.AttributeKeyTypeUnspecified,
IsJSON: true,
},
Operator: "=",
Value: 10,
},
},
{
Name: "float64",
Filter: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.float64[*]",
DataType: v3.AttributeKeyDataTypeArrayFloat64,
Type: v3.AttributeKeyTypeUnspecified,
},
Operator: "!=",
Value: 10.0,
},
Result: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.float64[*]",
DataType: v3.AttributeKeyDataTypeArrayFloat64,
Type: v3.AttributeKeyTypeUnspecified,
IsJSON: true,
},
Operator: "!=",
Value: 10.0,
},
},
{
Name: "float64x",
Filter: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.float64x",
DataType: v3.AttributeKeyDataTypeUnspecified,
Type: v3.AttributeKeyTypeUnspecified,
},
Operator: "!=",
Value: "10.0",
},
Result: v3.FilterItem{
Key: v3.AttributeKey{
Key: "body.float64x",
DataType: v3.AttributeKeyDataTypeFloat64,
Type: v3.AttributeKeyTypeUnspecified,
IsJSON: true,
},
Operator: "!=",
Value: 10.0,
},
},
}
func TestJsonEnrich(t *testing.T) {
for _, tt := range testJSONFilterEnrichData {
Convey(tt.Name, t, func() {
res := jsonFilterEnrich(tt.Filter)
So(res, ShouldResemble, tt.Result)
})
}
}
var testParseStrValueData = []struct {
Name string
Operator v3.FilterOperator
Value interface{}
ResultType string
Result interface{}
}{
{
Name: "bool",
Value: "true",
Operator: v3.FilterOperatorEqual,
ResultType: "bool",
Result: true,
},
{
Name: "int",
Value: "10",
Operator: v3.FilterOperatorNotEqual,
ResultType: "int64",
Result: 10,
},
{
Name: "float",
Value: "10.0",
Operator: v3.FilterOperatorGreaterThan,
ResultType: "float64",
Result: 10.0,
},
{
Name: "string",
Value: "hello",
Operator: v3.FilterOperatorLessThan,
ResultType: "string",
Result: "hello",
},
}
func TestParseStrValue(t *testing.T) {
for _, tt := range testParseStrValueData {
Convey(tt.Name, t, func() {
vtype, value := parseStrValue(tt.Value.(string), tt.Operator)
So(vtype, ShouldEqual, tt.ResultType)
So(value, ShouldEqual, tt.Result)
})
}
}