GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-15 12:36:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: support for exists and nexists in existing parser (#2873)

Browse Source 
* fix: support for exists and nexists in existing parser

* fix: comment updated

* feat: handle static log fields in exists and nexists
This commit is contained in:
Nityananda Gohain 2023-06-13 09:49:23 +05:30 committed by GitHub
parent ef74ef3526
commit 316cbe484b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 85 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
pkg/query-service/app/logs/parser.go
View File

@ -36,8 +36,8 @@ const (
DESC = "desc"
)
var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+( )+(in|nin)( )+\([^(]+\))|([\w.]+( )+(gt|lt|gte|lte)( )+(')?[\S]+(')?)|([\w.]+( )+(contains|ncontains))( )+[^\\]?'(.*?[^\\])')`)
var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`)
var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+( )+(in|nin)( )+\([^(]+\))|([\w.]+( )+(gt|lt|gte|lte)( )+(')?[\S]+(')?)|([\w.]+( )+(contains|ncontains))( )+[^\\]?'(.*?[^\\])'|([\w.]+( )+(exists|nexists)( )?))`)
var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gte|lte|gt|lt|contains|ncontains|exists|nexists)(?: )?`)
func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) {
res := model.LogsFilterParams{
@ -191,13 +191,18 @@ func parseLogQuery(query string) ([]string, error) {
sqlQueryTokens = append(sqlQueryTokens, f)
} else {
symbol := operatorMapping[strings.ToLower(op)]
sqlExpr := strings.Replace(v, " "+op+" ", " "+symbol+" ", 1)
splittedExpr := strings.Split(sqlExpr, symbol)
if len(splittedExpr) != 2 {
return nil, fmt.Errorf("error while splitting expression: %s", sqlExpr)
if symbol != "" {
sqlExpr := strings.Replace(v, " "+op+" ", " "+symbol+" ", 1)
splittedExpr := strings.Split(sqlExpr, symbol)
if len(splittedExpr) != 2 {
return nil, fmt.Errorf("error while splitting expression: %s", sqlExpr)
}
trimmedSqlExpr := fmt.Sprintf("%s %s %s ", strings.Join(strings.Fields(splittedExpr[0]), " "), symbol, strings.TrimSpace(splittedExpr[1]))
sqlQueryTokens = append(sqlQueryTokens, trimmedSqlExpr)
} else {
// for exists|nexists don't process it here since we don't have metadata
sqlQueryTokens = append(sqlQueryTokens, v)
}
trimmedSqlExpr := fmt.Sprintf("%s %s %s ", strings.Join(strings.Fields(splittedExpr[0]), " "), symbol, strings.TrimSpace(splittedExpr[1]))
sqlQueryTokens = append(sqlQueryTokens, trimmedSqlExpr)
}
}
@ -209,9 +214,6 @@ func parseColumn(s string) (*string, error) {
// if has and/or as prefix
filter := strings.Split(s, " ")
if len(filter) < 3 {
return nil, fmt.Errorf("incorrect filter")
}
first := strings.ToLower(filter[0])
if first == AND || first == OR {
@ -247,6 +249,9 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens []
}
func replaceFieldInToken(queryToken string, selectedFieldsLookup map[string]model.LogField, interestingFieldLookup map[string]model.LogField) (string, error) {
op := strings.TrimSpace(operatorRegex.FindString(queryToken))
opLower := strings.ToLower(op)
col, err := parseColumn(queryToken)
if err != nil {
return "", err
@ -254,6 +259,42 @@ func replaceFieldInToken(queryToken string, selectedFieldsLookup map[string]mode
sqlColName := *col
lowerColName := strings.ToLower(*col)
if opLower == "exists" || opLower == "nexists" {
var result string
// handle static fields which are columns, timestamp and id is not required but added them regardless
defaultValue := ""
if lowerColName == "trace_id" || lowerColName == "span_id" || lowerColName == "severity_text" || lowerColName == "id" {
defaultValue = "''"
}
if lowerColName == "trace_flags" || lowerColName == "severity_number" || lowerColName == "timestamp" {
defaultValue = "0"
}
if defaultValue != "" {
if opLower == "exists" {
result = fmt.Sprintf("%s != %s", sqlColName, defaultValue)
} else {
result = fmt.Sprintf("%s = %s", sqlColName, defaultValue)
}
} else {
// creating the query token here as we have the metadata
field := model.LogField{}
if sfield, ok := selectedFieldsLookup[sqlColName]; ok {
field = sfield
} else if ifield, ok := interestingFieldLookup[sqlColName]; ok {
field = ifield
}
result = fmt.Sprintf("has(%s_%s_key, '%s')", field.Type, strings.ToLower(field.DataType), field.Name)
if opLower == "nexists" {
result = "NOT " + result
}
}
return strings.Replace(queryToken, sqlColName+" "+op, result, 1), nil
}
if lowerColName != "body" {
if _, ok := selectedFieldsLookup[sqlColName]; !ok {
if field, ok := interestingFieldLookup[sqlColName]; ok {

33
pkg/query-service/app/logs/parser_test.go
View File

@ -102,6 +102,11 @@ var correctQueriesTest = []struct {
`userIdentifier in ('user') and userIdentifier contains 'user'`,
[]string{`userIdentifier IN ('user') `, `AND userIdentifier ILIKE '%user%' `},
},
{
`filters with for exists`,
`userIdentifier exists and user nexists`,
[]string{`userIdentifier exists `, `and user nexists`},
},
}
func TestParseLogQueryCorrect(t *testing.T) {
@ -206,6 +211,16 @@ var parseCorrectColumns = []struct {
`AND body ILIKE '%searchstring%' `,
"body",
},
{
"column with exists",
`AND user exists`,
"user",
},
{
"column with nexists",
`AND user nexists `,
"user",
},
}
func TestParseColumn(t *testing.T) {
@ -374,6 +389,24 @@ var generateSQLQueryTestCases = []struct {
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( field1 < 100 and attributes_int64_value[indexOf(attributes_int64_key, 'FielD1')] > 50 and Field2 > 10 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
},
{
Name: "Check exists and not exists",
Filter: model.LogsFilterParams{
Query: "field1 exists and Field2 nexists and Field2 gt 10",
TimestampStart: uint64(1657689292000),
TimestampEnd: uint64(1657689294000),
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( has(attributes_int64_key, 'field1') and NOT has(attributes_double64_key, 'Field2') and Field2 > 10 ) ",
},
{
Name: "Check exists and not exists on top level keys",
Filter: model.LogsFilterParams{
Query: "trace_id exists and span_id nexists and trace_flags exists and severity_number nexists",
TimestampStart: uint64(1657689292000),
TimestampEnd: uint64(1657689294000),
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( trace_id != '' and span_id = '' and trace_flags != 0 and severity_number = 0) ",
},
}
func TestGenerateSQLQuery(t *testing.T) {