diff --git a/pkg/telemetrymetadata/condition_builder.go b/pkg/telemetrymetadata/condition_builder.go new file mode 100644 index 0000000000..a8f5ce8f4b --- /dev/null +++ b/pkg/telemetrymetadata/condition_builder.go @@ -0,0 +1,149 @@ +package telemetrymetadata + +import ( + "context" + "fmt" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" +) + +var ( + attributeMetadataColumns = map[string]*schema.Column{ + "resource_attributes": {Name: "resource_attributes", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }}, + "attributes": {Name: "attributes", Type: schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }}, + } +) + +type conditionBuilder struct { +} + +func NewConditionBuilder() qbtypes.ConditionBuilder { + return &conditionBuilder{} +} + +func (c *conditionBuilder) GetColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) { + switch key.FieldContext { + case telemetrytypes.FieldContextResource: + return attributeMetadataColumns["resource_attributes"], nil + case telemetrytypes.FieldContextAttribute: + return attributeMetadataColumns["attributes"], nil + } + return nil, qbtypes.ErrColumnNotFound +} + +func (c *conditionBuilder) GetTableFieldName(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) { + column, err := c.GetColumn(ctx, key) + if err != nil { + return "", err + } + + switch column.Type { + case schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }: + return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil + } + return column.Name, nil +} + +func (c *conditionBuilder) GetCondition( + ctx context.Context, + key *telemetrytypes.TelemetryFieldKey, + operator qbtypes.FilterOperator, + value any, + sb *sqlbuilder.SelectBuilder, +) (string, error) { + column, err := c.GetColumn(ctx, key) + if err != nil { + // if we don't have a column, we can't build a condition for related values + return "", nil + } + + tblFieldName, err := c.GetTableFieldName(ctx, key) + if err != nil { + // if we don't have a table field name, we can't build a condition for related values + return "", nil + } + + if key.FieldDataType != telemetrytypes.FieldDataTypeString { + // if the field data type is not string, we can't build a condition for related values + return "", nil + } + + // key must exists to apply main filter + containsExp := fmt.Sprintf("mapContains(%s, %s)", column.Name, sb.Var(key.Name)) + + // regular operators + switch operator { + // regular operators + case qbtypes.FilterOperatorEqual: + return sb.And(containsExp, sb.E(tblFieldName, value)), nil + case qbtypes.FilterOperatorNotEqual: + return sb.And(containsExp, sb.NE(tblFieldName, value)), nil + + // like and not like + case qbtypes.FilterOperatorLike: + return sb.And(containsExp, sb.Like(tblFieldName, value)), nil + case qbtypes.FilterOperatorNotLike: + return sb.And(containsExp, sb.NotLike(tblFieldName, value)), nil + case qbtypes.FilterOperatorILike: + return sb.And(containsExp, sb.ILike(tblFieldName, value)), nil + case qbtypes.FilterOperatorNotILike: + return sb.And(containsExp, sb.NotILike(tblFieldName, value)), nil + + case qbtypes.FilterOperatorContains: + return sb.And(containsExp, sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value))), nil + case qbtypes.FilterOperatorNotContains: + return sb.And(containsExp, sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value))), nil + + case qbtypes.FilterOperatorRegexp: + exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value)) + return sb.And(containsExp, exp), nil + case qbtypes.FilterOperatorNotRegexp: + exp := fmt.Sprintf(`not match(%s, %s)`, tblFieldName, sb.Var(value)) + return sb.And(containsExp, exp), nil + + // in and not in + case qbtypes.FilterOperatorIn: + values, ok := value.([]any) + if !ok { + return "", qbtypes.ErrInValues + } + return sb.And(containsExp, sb.In(tblFieldName, values...)), nil + case qbtypes.FilterOperatorNotIn: + values, ok := value.([]any) + if !ok { + return "", qbtypes.ErrInValues + } + return sb.And(containsExp, sb.NotIn(tblFieldName, values...)), nil + + // exists and not exists + // in the query builder, `exists` and `not exists` are used for + // key membership checks, so depending on the column type, the condition changes + case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists: + switch column.Type { + case schema.MapColumnType{ + KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, + ValueType: schema.ColumnTypeString, + }: + leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) + if operator == qbtypes.FilterOperatorExists { + return sb.E(leftOperand, true), nil + } else { + return sb.NE(leftOperand, true), nil + } + } + } + + return "", nil +} diff --git a/pkg/telemetrymetadata/condition_builder_test.go b/pkg/telemetrymetadata/condition_builder_test.go new file mode 100644 index 0000000000..abd971146a --- /dev/null +++ b/pkg/telemetrymetadata/condition_builder_test.go @@ -0,0 +1,272 @@ +package telemetrymetadata + +import ( + "context" + "testing" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetColumn(t *testing.T) { + ctx := context.Background() + conditionBuilder := NewConditionBuilder() + + testCases := []struct { + name string + key telemetrytypes.TelemetryFieldKey + expectedCol *schema.Column + expectedError error + }{ + { + name: "Resource field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "service.name", + FieldContext: telemetrytypes.FieldContextResource, + }, + expectedCol: attributeMetadataColumns["resource_attributes"], + expectedError: nil, + }, + { + name: "Scope field - scope name", + key: telemetrytypes.TelemetryFieldKey{ + Name: "name", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + { + name: "Scope field - scope.name", + key: telemetrytypes.TelemetryFieldKey{ + Name: "scope.name", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + { + name: "Scope field - scope_name", + key: telemetrytypes.TelemetryFieldKey{ + Name: "scope_name", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + { + name: "Scope field - version", + key: telemetrytypes.TelemetryFieldKey{ + Name: "version", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + { + name: "Scope field - other scope field", + key: telemetrytypes.TelemetryFieldKey{ + Name: "custom.scope.field", + FieldContext: telemetrytypes.FieldContextScope, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + { + name: "Attribute field - string type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + expectedCol: attributeMetadataColumns["attributes"], + expectedError: nil, + }, + { + name: "Attribute field - number type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.size", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + expectedCol: attributeMetadataColumns["attributes"], + expectedError: nil, + }, + { + name: "Attribute field - int64 type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.duration", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeInt64, + }, + expectedCol: attributeMetadataColumns["attributes"], + expectedError: nil, + }, + { + name: "Attribute field - float64 type", + key: telemetrytypes.TelemetryFieldKey{ + Name: "cpu.utilization", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeFloat64, + }, + expectedCol: attributeMetadataColumns["attributes"], + expectedError: nil, + }, + { + name: "Log field - nonexistent", + key: telemetrytypes.TelemetryFieldKey{ + Name: "nonexistent_field", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedCol: nil, + expectedError: qbtypes.ErrColumnNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + col, err := conditionBuilder.GetColumn(ctx, &tc.key) + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedCol, col) + } + }) + } +} + +func TestGetFieldKeyName(t *testing.T) { + ctx := context.Background() + conditionBuilder := &conditionBuilder{} + + testCases := []struct { + name string + key telemetrytypes.TelemetryFieldKey + expectedResult string + expectedError error + }{ + { + name: "Map column type - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + expectedResult: "attributes['user.id']", + expectedError: nil, + }, + { + name: "Map column type - number attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.size", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeNumber, + }, + expectedResult: "attributes['request.size']", + expectedError: nil, + }, + { + name: "Map column type - bool attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "request.success", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeBool, + }, + expectedResult: "attributes['request.success']", + expectedError: nil, + }, + { + name: "Map column type - resource attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "service.name", + FieldContext: telemetrytypes.FieldContextResource, + }, + expectedResult: "resource_attributes['service.name']", + expectedError: nil, + }, + { + name: "Non-existent column", + key: telemetrytypes.TelemetryFieldKey{ + Name: "nonexistent_field", + FieldContext: telemetrytypes.FieldContextLog, + }, + expectedResult: "", + expectedError: qbtypes.ErrColumnNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := conditionBuilder.GetTableFieldName(ctx, &tc.key) + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expectedResult, result) + } + }) + } +} + +func TestGetCondition(t *testing.T) { + ctx := context.Background() + conditionBuilder := NewConditionBuilder() + + testCases := []struct { + name string + key telemetrytypes.TelemetryFieldKey + operator qbtypes.FilterOperator + value any + expectedSQL string + expectedError error + }{ + + { + name: "ILike operator - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorILike, + value: "%admin%", + expectedSQL: "WHERE (mapContains(attributes, ?) AND LOWER(attributes['user.id']) LIKE LOWER(?))", + expectedError: nil, + }, + { + name: "Not ILike operator - string attribute", + key: telemetrytypes.TelemetryFieldKey{ + Name: "user.id", + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }, + operator: qbtypes.FilterOperatorNotILike, + value: "%admin%", + expectedSQL: "WHERE (mapContains(attributes, ?) AND LOWER(attributes['user.id']) NOT LIKE LOWER(?))", + expectedError: nil, + }, + } + + for _, tc := range testCases { + sb := sqlbuilder.NewSelectBuilder() + t.Run(tc.name, func(t *testing.T) { + cond, err := conditionBuilder.GetCondition(ctx, &tc.key, tc.operator, tc.value, sb) + sb.Where(cond) + + if tc.expectedError != nil { + assert.Equal(t, tc.expectedError, err) + } else { + require.NoError(t, err) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, tc.expectedSQL) + } + }) + } +} diff --git a/pkg/telemetrytraces/condition_builder.go b/pkg/telemetrytraces/condition_builder.go index c08264849a..c161802990 100644 --- a/pkg/telemetrytraces/condition_builder.go +++ b/pkg/telemetrytraces/condition_builder.go @@ -252,9 +252,9 @@ func (c *conditionBuilder) GetCondition( return sb.NotILike(tblFieldName, value), nil case qbtypes.FilterOperatorContains: - return sb.ILike(tblFieldName, value), nil + return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil case qbtypes.FilterOperatorNotContains: - return sb.NotILike(tblFieldName, value), nil + return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil case qbtypes.FilterOperatorRegexp: exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value))