diff --git a/pkg/telemetrylogs/condition_builder.go b/pkg/telemetrylogs/condition_builder.go new file mode 100644 index 0000000000..762f3af508 --- /dev/null +++ b/pkg/telemetrylogs/condition_builder.go @@ -0,0 +1,275 @@ +package telemetrylogs + +import ( + "context" + "fmt" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + + "github.com/huandu/go-sqlbuilder" +) + +var ( + logsV2Columns = map[string]*schema.Column{ + "ts_bucket_start": {Name: "ts_bucket_start", Type: schema.ColumnTypeUInt64}, + "resource_fingerprint": {Name: "resource_fingerprint", Type: schema.ColumnTypeString}, + + "timestamp": {Name: "timestamp", Type: schema.ColumnTypeUInt64}, + "observed_timestamp": {Name: "observed_timestamp", Type: schema.ColumnTypeUInt64}, + "id": {Name: "id", Type: schema.ColumnTypeString}, + "trace_id": {Name: "trace_id", Type: schema.ColumnTypeString}, + "span_id": {Name: "span_id", Type: schema.ColumnTypeString}, + "trace_flags": {Name: "trace_flags", Type: schema.ColumnTypeUInt32}, + "severity_text": {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}}, + "severity_number": {Name: "severity_number", Type: schema.ColumnTypeUInt8}, + "body": {Name: "body", Type: schema.ColumnTypeString}, + "attributes_string": {Name: "attributes_string", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }}, + "attributes_number": {Name: "attributes_number", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeFloat64, + }}, + "attributes_bool": {Name: "attributes_bool", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeBool, + }}, + "resources_string": {Name: "resources_string", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }}, + "scope_name": {Name: "scope_name", Type: schema.ColumnTypeString}, + "scope_version": {Name: "scope_version", Type: schema.ColumnTypeString}, + "scope_string": {Name: "scope_string", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }}, + } +) + +var _ qbtypes.ConditionBuilder = &conditionBuilder{} + +type conditionBuilder struct { +} + +func NewConditionBuilder() qbtypes.ConditionBuilder { + return &conditionBuilder{} +} + +func (c *conditionBuilder) GetColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) { + + switch key.FieldContext { + case telemetrytypes.FieldContextResource: + return logsV2Columns["resources_string"], nil + case telemetrytypes.FieldContextScope: + switch key.Name { + case "name", "scope.name", "scope_name": + return logsV2Columns["scope_name"], nil + case "version", "scope.version", "scope_version": + return logsV2Columns["scope_version"], nil + } + return logsV2Columns["scope_string"], nil + case telemetrytypes.FieldContextAttribute: + switch key.FieldDataType { + case telemetrytypes.FieldDataTypeString: + return logsV2Columns["attributes_string"], nil + case telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeNumber: + return logsV2Columns["attributes_number"], nil + case telemetrytypes.FieldDataTypeBool: + return logsV2Columns["attributes_bool"], nil + } + case telemetrytypes.FieldContextLog: + col, ok := logsV2Columns[key.Name] + if !ok { + return nil, qbtypes.ErrColumnNotFound + } + return col, nil + } + + return nil, qbtypes.ErrColumnNotFound +} + +func (c *conditionBuilder) GetTableFieldName(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) { + column, err := c.GetColumn(ctx, key) + if err != nil { + return "", err + } + + switch column.Type { + case schema.ColumnTypeString, + schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + schema.ColumnTypeUInt64, + schema.ColumnTypeUInt32, + schema.ColumnTypeUInt8: + return column.Name, nil + case schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }: + // a key could have been materialized, if so return the materialized column name + if key.Materialized { + return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil + } + return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil + case schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeFloat64, + }: + // a key could have been materialized, if so return the materialized column name + if key.Materialized { + return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil + } + return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil + case schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeBool, + }: + // a key could have been materialized, if so return the materialized column name + if key.Materialized { + return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil + } + return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil + } + // should not reach here + return column.Name, nil +} + +func (c *conditionBuilder) GetCondition( + ctx context.Context, + key *telemetrytypes.TelemetryFieldKey, + operator qbtypes.FilterOperator, + value any, + sb *sqlbuilder.SelectBuilder, +) (string, error) { + column, err := c.GetColumn(ctx, key) + if err != nil { + return "", err + } + + tblFieldName, err := c.GetTableFieldName(ctx, key) + if err != nil { + return "", err + } + + tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName) + + // regular operators + switch operator { + // regular operators + case qbtypes.FilterOperatorEqual: + return sb.E(tblFieldName, value), nil + case qbtypes.FilterOperatorNotEqual: + return sb.NE(tblFieldName, value), nil + case qbtypes.FilterOperatorGreaterThan: + return sb.G(tblFieldName, value), nil + case qbtypes.FilterOperatorGreaterThanOrEq: + return sb.GE(tblFieldName, value), nil + case qbtypes.FilterOperatorLessThan: + return sb.LT(tblFieldName, value), nil + case qbtypes.FilterOperatorLessThanOrEq: + return sb.LE(tblFieldName, value), nil + + // like and not like + case qbtypes.FilterOperatorLike: + return sb.Like(tblFieldName, value), nil + case qbtypes.FilterOperatorNotLike: + return sb.NotLike(tblFieldName, value), nil + case qbtypes.FilterOperatorILike: + return sb.ILike(tblFieldName, value), nil + case qbtypes.FilterOperatorNotILike: + return sb.NotILike(tblFieldName, value), nil + + case qbtypes.FilterOperatorContains: + return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil + case qbtypes.FilterOperatorNotContains: + return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil + + case qbtypes.FilterOperatorRegexp: + exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value)) + return sb.And(exp), nil + case qbtypes.FilterOperatorNotRegexp: + exp := fmt.Sprintf(`not match(%s, %s)`, tblFieldName, sb.Var(value)) + return sb.And(exp), nil + // between and not between + case qbtypes.FilterOperatorBetween: + values, ok := value.([]any) + if !ok { + return "", qbtypes.ErrBetweenValues + } + if len(values) != 2 { + return "", qbtypes.ErrBetweenValues + } + return sb.Between(tblFieldName, values[0], values[1]), nil + case qbtypes.FilterOperatorNotBetween: + values, ok := value.([]any) + if !ok { + return "", qbtypes.ErrBetweenValues + } + if len(values) != 2 { + return "", qbtypes.ErrBetweenValues + } + return sb.NotBetween(tblFieldName, values[0], values[1]), nil + + // in and not in + case qbtypes.FilterOperatorIn: + values, ok := value.([]any) + if !ok { + return "", qbtypes.ErrInValues + } + return sb.In(tblFieldName, values...), nil + case qbtypes.FilterOperatorNotIn: + values, ok := value.([]any) + if !ok { + return "", qbtypes.ErrInValues + } + return sb.NotIn(tblFieldName, values...), nil + + // exists and not exists + // but how could you live and have no story to tell + // in the UI based query builder, `exists` and `not exists` are used for + // key membership checks, so depending on the column type, the condition changes + case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists: + var value any + switch column.Type { + case schema.ColumnTypeString, schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}: + value = "" + if operator == qbtypes.FilterOperatorExists { + return sb.NE(tblFieldName, value), nil + } else { + return sb.E(tblFieldName, value), nil + } + case schema.ColumnTypeUInt64, schema.ColumnTypeUInt32, schema.ColumnTypeUInt8: + value = 0 + if operator == qbtypes.FilterOperatorExists { + return sb.NE(tblFieldName, value), nil + } else { + return sb.E(tblFieldName, value), nil + } + case schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }, schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeBool, + }, schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeFloat64, + }: + leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) + if key.Materialized { + leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key) + } + if operator == qbtypes.FilterOperatorExists { + return sb.E(leftOperand, true), nil + } else { + return sb.NE(leftOperand, true), nil + } + default: + return "", fmt.Errorf("exists operator is not supported for column type %s", column.Type) + } + } + return "", fmt.Errorf("unsupported operator: %v", operator) +} diff --git a/pkg/telemetrylogs/condition_builder_test.go b/pkg/telemetrylogs/condition_builder_test.go new file mode 100644 index 0000000000..1ad643ab62 --- /dev/null +++ b/pkg/telemetrylogs/condition_builder_test.go @@ -0,0 +1,620 @@ +package telemetrylogs + +import ( + "context" + "testing" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetColumn(t *testing.T) { + ctx := context.Background() + conditionBuilder := NewConditionBuilder() + + testCases := []struct { + name string + key telemetrytypes.TelemetryFieldKey + expectedCol *schema.Column + expectedError error + }{ + { + name: "Resource field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "service.name", + FieldContext: telemetrytypes.FieldContextResource, + }, + expectedCol: logsV2Columns["resources_string"], + expectedError: nil, + }, + { + name: "Scope field - scope name", + key: telemetrytypes.TelemetryFieldKey{ + Name: "name", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: logsV2Columns["scope_name"], + expectedError: nil, + }, + { + name: "Scope field - scope.name", + key: telemetrytypes.TelemetryFieldKey{ + Name: "scope.name", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: logsV2Columns["scope_name"], + expectedError: nil, + }, + { + name: "Scope field - scope_name", + key: telemetrytypes.TelemetryFieldKey{ + Name: "scope_name", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: logsV2Columns["scope_name"], + expectedError: nil, + }, + { + name: "Scope field - version", + key: telemetrytypes.TelemetryFieldKey{ + Name: "version", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: logsV2Columns["scope_version"], + expectedError: nil, + }, + { + name: "Scope field - other scope field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "custom.scope.field", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: logsV2Columns["scope_string"], + expectedError: nil, + }, + { + name: "Attribute field - string type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + expectedCol: logsV2Columns["attributes_string"], + expectedError: nil, + }, + { + name: "Attribute field - number type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.size", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + expectedCol: logsV2Columns["attributes_number"], + expectedError: nil, + }, + { + name: "Attribute field - int64 type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.duration", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeInt64, + }, + expectedCol: logsV2Columns["attributes_number"], + expectedError: nil, + }, + { + name: "Attribute field - float64 type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "cpu.utilization", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeFloat64, + }, + expectedCol: logsV2Columns["attributes_number"], + expectedError: nil, + }, + { + name: "Attribute field - bool type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.success", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeBool, + }, + expectedCol: logsV2Columns["attributes_bool"], + expectedError: nil, + }, + { + name: "Log field - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedCol: logsV2Columns["timestamp"], + expectedError: nil, + }, + { + name: "Log field - body", + key: telemetrytypes.TelemetryFieldKey{ + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedCol: logsV2Columns["body"], + expectedError: nil, + }, + { + name: "Log field - nonexistent", + key: telemetrytypes.TelemetryFieldKey{ + Name: "nonexistent_field", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + { + name: "did_user_login", + key: telemetrytypes.TelemetryFieldKey{ + Name: "did_user_login", + Signal: telemetrytypes.SignalLogs, + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeBool, + }, + expectedCol: logsV2Columns["attributes_bool"], + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + col, err := conditionBuilder.GetColumn(ctx, &tc.key) + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedCol, col) + } + }) + } +} + +func TestGetFieldKeyName(t *testing.T) { + ctx := context.Background() + conditionBuilder := &conditionBuilder{} + + testCases := []struct { + name string + key telemetrytypes.TelemetryFieldKey + expectedResult string + expectedError error + }{ + { + name: "Simple column type - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedResult: "timestamp", + expectedError: nil, + }, + { + name: "Map column type - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + expectedResult: "attributes_string['user.id']", + expectedError: nil, + }, + { + name: "Map column type - number attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.size", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + expectedResult: "attributes_number['request.size']", + expectedError: nil, + }, + { + name: "Map column type - bool attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.success", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeBool, + }, + expectedResult: "attributes_bool['request.success']", + expectedError: nil, + }, + { + name: "Map column type - resource attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "service.name", + FieldContext: telemetrytypes.FieldContextResource, + }, + expectedResult: "resources_string['service.name']", + expectedError: nil, + }, + { + name: "Non-existent column", + key: telemetrytypes.TelemetryFieldKey{ + Name: "nonexistent_field", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedResult: "", + expectedError: qbtypes.ErrColumnNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := conditionBuilder.GetTableFieldName(ctx, &tc.key) + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedResult, result) + } + }) + } +} + +func TestGetCondition(t *testing.T) { + ctx := context.Background() + conditionBuilder := NewConditionBuilder() + + testCases := []struct { + name string + key telemetrytypes.TelemetryFieldKey + operator qbtypes.FilterOperator + value any + expectedSQL string + expectedError error + }{ + { + name: "Equal operator - string", + key: telemetrytypes.TelemetryFieldKey{ + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorEqual, + value: "error message", + expectedSQL: "body = ?", + expectedError: nil, + }, + { + name: "Not Equal operator - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorNotEqual, + value: uint64(1617979338000000000), + expectedSQL: "timestamp <> ?", + expectedError: nil, + }, + { + name: "Greater Than operator - number attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.duration", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + operator: qbtypes.FilterOperatorGreaterThan, + value: float64(100), + expectedSQL: "attributes_number['request.duration'] > ?", + expectedError: nil, + }, + { + name: "Less Than operator - number attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.size", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + operator: qbtypes.FilterOperatorLessThan, + value: float64(1024), + expectedSQL: "attributes_number['request.size'] < ?", + expectedError: nil, + }, + { + name: "Greater Than Or Equal operator - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorGreaterThanOrEq, + value: uint64(1617979338000000000), + expectedSQL: "timestamp >= ?", + expectedError: nil, + }, + { + name: "Less Than Or Equal operator - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorLessThanOrEq, + value: uint64(1617979338000000000), + expectedSQL: "timestamp <= ?", + expectedError: nil, + }, + { + name: "Like operator - body", + key: telemetrytypes.TelemetryFieldKey{ + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorLike, + value: "%error%", + expectedSQL: "body LIKE ?", + expectedError: nil, + }, + { + name: "Not Like operator - body", + key: telemetrytypes.TelemetryFieldKey{ + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorNotLike, + value: "%error%", + expectedSQL: "body NOT LIKE ?", + expectedError: nil, + }, + { + name: "ILike operator - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorILike, + value: "%admin%", + expectedSQL: "WHERE LOWER(attributes_string['user.id']) LIKE LOWER(?)", + expectedError: nil, + }, + { + name: "Not ILike operator - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorNotILike, + value: "%admin%", + expectedSQL: "WHERE LOWER(attributes_string['user.id']) NOT LIKE LOWER(?)", + expectedError: nil, + }, + { + name: "Contains operator - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorContains, + value: "admin", + expectedSQL: "WHERE LOWER(attributes_string['user.id']) LIKE LOWER(?)", + expectedError: nil, + }, + { + name: "Between operator - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorBetween, + value: []any{uint64(1617979338000000000), uint64(1617979348000000000)}, + expectedSQL: "timestamp BETWEEN ? AND ?", + expectedError: nil, + }, + { + name: "Between operator - invalid value", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorBetween, + value: "invalid", + expectedSQL: "", + expectedError: qbtypes.ErrBetweenValues, + }, + { + name: "Between operator - insufficient values", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorBetween, + value: []any{uint64(1617979338000000000)}, + expectedSQL: "", + expectedError: qbtypes.ErrBetweenValues, + }, + { + name: "Not Between operator - timestamp", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorNotBetween, + value: []any{uint64(1617979338000000000), uint64(1617979348000000000)}, + expectedSQL: "timestamp NOT BETWEEN ? AND ?", + expectedError: nil, + }, + { + name: "In operator - severity_text", + key: telemetrytypes.TelemetryFieldKey{ + Name: "severity_text", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorIn, + value: []any{"error", "fatal", "critical"}, + expectedSQL: "severity_text IN (?, ?, ?)", + expectedError: nil, + }, + { + name: "In operator - invalid value", + key: telemetrytypes.TelemetryFieldKey{ + Name: "severity_text", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorIn, + value: "error", + expectedSQL: "", + expectedError: qbtypes.ErrInValues, + }, + { + name: "Not In operator - severity_text", + key: telemetrytypes.TelemetryFieldKey{ + Name: "severity_text", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorNotIn, + value: []any{"debug", "info", "trace"}, + expectedSQL: "severity_text NOT IN (?, ?, ?)", + expectedError: nil, + }, + { + name: "Exists operator - string field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorExists, + value: nil, + expectedSQL: "body <> ?", + expectedError: nil, + }, + { + name: "Not Exists operator - string field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorNotExists, + value: nil, + expectedSQL: "body = ?", + expectedError: nil, + }, + { + name: "Exists operator - number field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "timestamp", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorExists, + value: nil, + expectedSQL: "timestamp <> ?", + expectedError: nil, + }, + { + name: "Exists operator - map field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorExists, + value: nil, + expectedSQL: "mapContains(attributes_string, 'user.id') = ?", + expectedError: nil, + }, + { + name: "Not Exists operator - map field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorNotExists, + value: nil, + expectedSQL: "mapContains(attributes_string, 'user.id') <> ?", + expectedError: nil, + }, + { + name: "Non-existent column", + key: telemetrytypes.TelemetryFieldKey{ + Name: "nonexistent_field", + FieldContext: telemetrytypes.FieldContextLog, + }, + operator: qbtypes.FilterOperatorEqual, + value: "value", + expectedSQL: "", + expectedError: qbtypes.ErrColumnNotFound, + }, + } + + for _, tc := range testCases { + sb := sqlbuilder.NewSelectBuilder() + t.Run(tc.name, func(t *testing.T) { + cond, err := conditionBuilder.GetCondition(ctx, &tc.key, tc.operator, tc.value, sb) + sb.Where(cond) + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, tc.expectedSQL) + } + }) + } +} + +func TestGetConditionMultiple(t *testing.T) { + ctx := context.Background() + conditionBuilder := NewConditionBuilder() + + testCases := []struct { + name string + keys []*telemetrytypes.TelemetryFieldKey + operator qbtypes.FilterOperator + value any + expectedSQL string + expectedError error + }{ + { + name: "Equal operator - string", + keys: []*telemetrytypes.TelemetryFieldKey{ + { + Name: "body", + FieldContext: telemetrytypes.FieldContextLog, + }, + { + Name: "severity_text", + FieldContext: telemetrytypes.FieldContextLog, + }, + }, + operator: qbtypes.FilterOperatorEqual, + value: "error message", + expectedSQL: "body = ? AND severity_text = ?", + expectedError: nil, + }, + } + + for _, tc := range testCases { + sb := sqlbuilder.NewSelectBuilder() + t.Run(tc.name, func(t *testing.T) { + var err error + for _, key := range tc.keys { + cond, err := conditionBuilder.GetCondition(ctx, key, tc.operator, tc.value, sb) + sb.Where(cond) + if err != nil { + t.Fatalf("Error getting condition for key %s: %v", key.Name, err) + } + } + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, tc.expectedSQL) + } + }) + } +} diff --git a/pkg/telemetrylogs/tables.go b/pkg/telemetrylogs/tables.go new file mode 100644 index 0000000000..1d0c35fbc6 --- /dev/null +++ b/pkg/telemetrylogs/tables.go @@ -0,0 +1,9 @@ +package telemetrylogs + +const ( + DBName = "signoz_logs" + LogsV2TableName = "distributed_logs_v2" + LogsV2LocalTableName = "logs_v2" + TagAttributesV2TableName = "distributed_tag_attributes_v2" + TagAttributesV2LocalTableName = "tag_attributes_v2" +) diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go index 9c1118977f..f4f7aefa7b 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go @@ -4,10 +4,17 @@ import ( "context" schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/huandu/go-sqlbuilder" ) +var ( + ErrColumnNotFound = errors.Newf(errors.TypeNotFound, errors.CodeNotFound, "column not found") + ErrBetweenValues = errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "(not) between operator requires two values") + ErrInValues = errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "(not) in operator requires a list of values") +) + // FilterOperator is the operator for the filter. type FilterOperator int diff --git a/pkg/types/telemetrytypes/field.go b/pkg/types/telemetrytypes/field.go index 5f435b4f7d..1fd7e3272f 100644 --- a/pkg/types/telemetrytypes/field.go +++ b/pkg/types/telemetrytypes/field.go @@ -86,11 +86,11 @@ func GetFieldKeyFromKeyText(key string) TelemetryFieldKey { return fieldKeySelector } -func FieldKeyToMaterializedColumnName(key TelemetryFieldKey) string { +func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string { return fmt.Sprintf("%s_%s_%s", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$")) } -func FieldKeyToMaterializedColumnNameForExists(key TelemetryFieldKey) string { +func FieldKeyToMaterializedColumnNameForExists(key *TelemetryFieldKey) string { return fmt.Sprintf("%s_%s_%s_exists", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$")) } @@ -123,3 +123,52 @@ type FieldValueSelector struct { Value string `json:"value"` Limit int `json:"limit"` } + +func DataTypeCollisionHandledFieldName(key *TelemetryFieldKey, value any, tblFieldName string) (string, any) { + // This block of code exists to handle the data type collisions + // We don't want to fail the requests when there is a key with more than one data type + // Let's take an example of `http.status_code`, and consider user sent a string value and number value + // When they search for `http.status_code=200`, we will search across both the number columns and string columns + // and return the results from both the columns + // While we expect user not to send the mixed data types, it inevitably happens + // So we handle the data type collisions here + switch key.FieldDataType { + case FieldDataTypeString: + switch value.(type) { + case float64: + // try to convert the string value to to number + tblFieldName = fmt.Sprintf(`toFloat64OrNull(%s)`, tblFieldName) + case []any: + areFloats := true + for _, v := range value.([]any) { + if _, ok := v.(float64); !ok { + areFloats = false + break + } + } + if areFloats { + tblFieldName = fmt.Sprintf(`toFloat64OrNull(%s)`, tblFieldName) + } + case bool: + // we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string + value = fmt.Sprintf("%t", value) + case string: + // nothing to do + } + case FieldDataTypeFloat64, FieldDataTypeInt64, FieldDataTypeNumber: + switch value.(type) { + case string: + // try to convert the string value to to number + tblFieldName = fmt.Sprintf(`toString(%s)`, tblFieldName) + case float64: + // nothing to do + } + case FieldDataTypeBool: + switch value.(type) { + case string: + // try to convert the string value to to number + tblFieldName = fmt.Sprintf(`toString(%s)`, tblFieldName) + } + } + return tblFieldName, value +}