chore: add condition builder attributes metadata (#7558)

pkg/telemetrymetadata/condition_builder.go

@@ -0,0 +1,149 @@
+package telemetrymetadata
+
+import (
+	"context"
+	"fmt"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	attributeMetadataColumns = map[string]*schema.Column{
+		"resource_attributes": {Name: "resource_attributes", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}},
+		"attributes": {Name: "attributes", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}},
+	}
+)
+
+type conditionBuilder struct {
+}
+
+func NewConditionBuilder() qbtypes.ConditionBuilder {
+	return &conditionBuilder{}
+}
+
+func (c *conditionBuilder) GetColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
+	switch key.FieldContext {
+	case telemetrytypes.FieldContextResource:
+		return attributeMetadataColumns["resource_attributes"], nil
+	case telemetrytypes.FieldContextAttribute:
+		return attributeMetadataColumns["attributes"], nil
+	}
+	return nil, qbtypes.ErrColumnNotFound
+}
+
+func (c *conditionBuilder) GetTableFieldName(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
+	column, err := c.GetColumn(ctx, key)
+	if err != nil {
+		return "", err
+	}
+
+	switch column.Type {
+	case schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeString,
+	}:
+		return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
+	}
+	return column.Name, nil
+}
+
+func (c *conditionBuilder) GetCondition(
+	ctx context.Context,
+	key *telemetrytypes.TelemetryFieldKey,
+	operator qbtypes.FilterOperator,
+	value any,
+	sb *sqlbuilder.SelectBuilder,
+) (string, error) {
+	column, err := c.GetColumn(ctx, key)
+	if err != nil {
+		// if we don't have a column, we can't build a condition for related values
+		return "", nil
+	}
+
+	tblFieldName, err := c.GetTableFieldName(ctx, key)
+	if err != nil {
+		// if we don't have a table field name, we can't build a condition for related values
+		return "", nil
+	}
+
+	if key.FieldDataType != telemetrytypes.FieldDataTypeString {
+		// if the field data type is not string, we can't build a condition for related values
+		return "", nil
+	}
+
+	// key must exists to apply main filter
+	containsExp := fmt.Sprintf("mapContains(%s, %s)", column.Name, sb.Var(key.Name))
+
+	// regular operators
+	switch operator {
+	// regular operators
+	case qbtypes.FilterOperatorEqual:
+		return sb.And(containsExp, sb.E(tblFieldName, value)), nil
+	case qbtypes.FilterOperatorNotEqual:
+		return sb.And(containsExp, sb.NE(tblFieldName, value)), nil
+
+	// like and not like
+	case qbtypes.FilterOperatorLike:
+		return sb.And(containsExp, sb.Like(tblFieldName, value)), nil
+	case qbtypes.FilterOperatorNotLike:
+		return sb.And(containsExp, sb.NotLike(tblFieldName, value)), nil
+	case qbtypes.FilterOperatorILike:
+		return sb.And(containsExp, sb.ILike(tblFieldName, value)), nil
+	case qbtypes.FilterOperatorNotILike:
+		return sb.And(containsExp, sb.NotILike(tblFieldName, value)), nil
+
+	case qbtypes.FilterOperatorContains:
+		return sb.And(containsExp, sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value))), nil
+	case qbtypes.FilterOperatorNotContains:
+		return sb.And(containsExp, sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value))), nil
+
+	case qbtypes.FilterOperatorRegexp:
+		exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value))
+		return sb.And(containsExp, exp), nil
+	case qbtypes.FilterOperatorNotRegexp:
+		exp := fmt.Sprintf(`not match(%s, %s)`, tblFieldName, sb.Var(value))
+		return sb.And(containsExp, exp), nil
+
+	// in and not in
+	case qbtypes.FilterOperatorIn:
+		values, ok := value.([]any)
+		if !ok {
+			return "", qbtypes.ErrInValues
+		}
+		return sb.And(containsExp, sb.In(tblFieldName, values...)), nil
+	case qbtypes.FilterOperatorNotIn:
+		values, ok := value.([]any)
+		if !ok {
+			return "", qbtypes.ErrInValues
+		}
+		return sb.And(containsExp, sb.NotIn(tblFieldName, values...)), nil
+
+	// exists and not exists
+	// in the query builder, `exists` and `not exists` are used for
+	// key membership checks, so depending on the column type, the condition changes
+	case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
+		switch column.Type {
+		case schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}:
+			leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
+			if operator == qbtypes.FilterOperatorExists {
+				return sb.E(leftOperand, true), nil
+			} else {
+				return sb.NE(leftOperand, true), nil
+			}
+		}
+	}
+
+	return "", nil
+}

pkg/telemetrymetadata/condition_builder_test.go

@@ -0,0 +1,272 @@
+package telemetrymetadata
+
+import (
+	"context"
+	"testing"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/huandu/go-sqlbuilder"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetColumn(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := NewConditionBuilder()
+
+	testCases := []struct {
+		name          string
+		key           telemetrytypes.TelemetryFieldKey
+		expectedCol   *schema.Column
+		expectedError error
+	}{
+		{
+			name: "Resource field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+			},
+			expectedCol:   attributeMetadataColumns["resource_attributes"],
+			expectedError: nil,
+		},
+		{
+			name: "Scope field - scope name",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "name",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+		{
+			name: "Scope field - scope.name",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "scope.name",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+		{
+			name: "Scope field - scope_name",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "scope_name",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+		{
+			name: "Scope field - version",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "version",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+		{
+			name: "Scope field - other scope field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "custom.scope.field",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+		{
+			name: "Attribute field - string type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			expectedCol:   attributeMetadataColumns["attributes"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - number type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.size",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			},
+			expectedCol:   attributeMetadataColumns["attributes"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - int64 type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.duration",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeInt64,
+			},
+			expectedCol:   attributeMetadataColumns["attributes"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - float64 type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "cpu.utilization",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+			},
+			expectedCol:   attributeMetadataColumns["attributes"],
+			expectedError: nil,
+		},
+		{
+			name: "Log field - nonexistent",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "nonexistent_field",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			col, err := conditionBuilder.GetColumn(ctx, &tc.key)
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectedCol, col)
+			}
+		})
+	}
+}
+
+func TestGetFieldKeyName(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := &conditionBuilder{}
+
+	testCases := []struct {
+		name           string
+		key            telemetrytypes.TelemetryFieldKey
+		expectedResult string
+		expectedError  error
+	}{
+		{
+			name: "Map column type - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			expectedResult: "attributes['user.id']",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - number attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.size",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			},
+			expectedResult: "attributes['request.size']",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - bool attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.success",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeBool,
+			},
+			expectedResult: "attributes['request.success']",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - resource attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+			},
+			expectedResult: "resource_attributes['service.name']",
+			expectedError:  nil,
+		},
+		{
+			name: "Non-existent column",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "nonexistent_field",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedResult: "",
+			expectedError:  qbtypes.ErrColumnNotFound,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := conditionBuilder.GetTableFieldName(ctx, &tc.key)
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectedResult, result)
+			}
+		})
+	}
+}
+
+func TestGetCondition(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := NewConditionBuilder()
+
+	testCases := []struct {
+		name          string
+		key           telemetrytypes.TelemetryFieldKey
+		operator      qbtypes.FilterOperator
+		value         any
+		expectedSQL   string
+		expectedError error
+	}{
+
+		{
+			name: "ILike operator - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorILike,
+			value:         "%admin%",
+			expectedSQL:   "WHERE (mapContains(attributes, ?) AND LOWER(attributes['user.id']) LIKE LOWER(?))",
+			expectedError: nil,
+		},
+		{
+			name: "Not ILike operator - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorNotILike,
+			value:         "%admin%",
+			expectedSQL:   "WHERE (mapContains(attributes, ?) AND LOWER(attributes['user.id']) NOT LIKE LOWER(?))",
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		sb := sqlbuilder.NewSelectBuilder()
+		t.Run(tc.name, func(t *testing.T) {
+			cond, err := conditionBuilder.GetCondition(ctx, &tc.key, tc.operator, tc.value, sb)
+			sb.Where(cond)
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+				assert.Contains(t, sql, tc.expectedSQL)
+			}
+		})
+	}
+}

pkg/telemetrytraces/condition_builder.go

@@ -252,9 +252,9 @@ func (c *conditionBuilder) GetCondition(
 		return sb.NotILike(tblFieldName, value), nil
 
 	case qbtypes.FilterOperatorContains:
-		return sb.ILike(tblFieldName, value), nil
+		return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
 	case qbtypes.FilterOperatorNotContains:
-		return sb.NotILike(tblFieldName, value), nil
+		return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
 
 	case qbtypes.FilterOperatorRegexp:
 		exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value))