chore: add non-json condition builder for logs v2 (#7555)

pkg/telemetrylogs/condition_builder.go

@@ -0,0 +1,275 @@
+package telemetrylogs
+
+import (
+	"context"
+	"fmt"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	logsV2Columns = map[string]*schema.Column{
+		"ts_bucket_start":      {Name: "ts_bucket_start", Type: schema.ColumnTypeUInt64},
+		"resource_fingerprint": {Name: "resource_fingerprint", Type: schema.ColumnTypeString},
+
+		"timestamp":          {Name: "timestamp", Type: schema.ColumnTypeUInt64},
+		"observed_timestamp": {Name: "observed_timestamp", Type: schema.ColumnTypeUInt64},
+		"id":                 {Name: "id", Type: schema.ColumnTypeString},
+		"trace_id":           {Name: "trace_id", Type: schema.ColumnTypeString},
+		"span_id":            {Name: "span_id", Type: schema.ColumnTypeString},
+		"trace_flags":        {Name: "trace_flags", Type: schema.ColumnTypeUInt32},
+		"severity_text":      {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}},
+		"severity_number":    {Name: "severity_number", Type: schema.ColumnTypeUInt8},
+		"body":               {Name: "body", Type: schema.ColumnTypeString},
+		"attributes_string": {Name: "attributes_string", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}},
+		"attributes_number": {Name: "attributes_number", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeFloat64,
+		}},
+		"attributes_bool": {Name: "attributes_bool", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeBool,
+		}},
+		"resources_string": {Name: "resources_string", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}},
+		"scope_name":    {Name: "scope_name", Type: schema.ColumnTypeString},
+		"scope_version": {Name: "scope_version", Type: schema.ColumnTypeString},
+		"scope_string": {Name: "scope_string", Type: schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}},
+	}
+)
+
+var _ qbtypes.ConditionBuilder = &conditionBuilder{}
+
+type conditionBuilder struct {
+}
+
+func NewConditionBuilder() qbtypes.ConditionBuilder {
+	return &conditionBuilder{}
+}
+
+func (c *conditionBuilder) GetColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
+
+	switch key.FieldContext {
+	case telemetrytypes.FieldContextResource:
+		return logsV2Columns["resources_string"], nil
+	case telemetrytypes.FieldContextScope:
+		switch key.Name {
+		case "name", "scope.name", "scope_name":
+			return logsV2Columns["scope_name"], nil
+		case "version", "scope.version", "scope_version":
+			return logsV2Columns["scope_version"], nil
+		}
+		return logsV2Columns["scope_string"], nil
+	case telemetrytypes.FieldContextAttribute:
+		switch key.FieldDataType {
+		case telemetrytypes.FieldDataTypeString:
+			return logsV2Columns["attributes_string"], nil
+		case telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeNumber:
+			return logsV2Columns["attributes_number"], nil
+		case telemetrytypes.FieldDataTypeBool:
+			return logsV2Columns["attributes_bool"], nil
+		}
+	case telemetrytypes.FieldContextLog:
+		col, ok := logsV2Columns[key.Name]
+		if !ok {
+			return nil, qbtypes.ErrColumnNotFound
+		}
+		return col, nil
+	}
+
+	return nil, qbtypes.ErrColumnNotFound
+}
+
+func (c *conditionBuilder) GetTableFieldName(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
+	column, err := c.GetColumn(ctx, key)
+	if err != nil {
+		return "", err
+	}
+
+	switch column.Type {
+	case schema.ColumnTypeString,
+		schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		schema.ColumnTypeUInt64,
+		schema.ColumnTypeUInt32,
+		schema.ColumnTypeUInt8:
+		return column.Name, nil
+	case schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeString,
+	}:
+		// a key could have been materialized, if so return the materialized column name
+		if key.Materialized {
+			return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
+		}
+		return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
+	case schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeFloat64,
+	}:
+		// a key could have been materialized, if so return the materialized column name
+		if key.Materialized {
+			return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
+		}
+		return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
+	case schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeBool,
+	}:
+		// a key could have been materialized, if so return the materialized column name
+		if key.Materialized {
+			return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
+		}
+		return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
+	}
+	// should not reach here
+	return column.Name, nil
+}
+
+func (c *conditionBuilder) GetCondition(
+	ctx context.Context,
+	key *telemetrytypes.TelemetryFieldKey,
+	operator qbtypes.FilterOperator,
+	value any,
+	sb *sqlbuilder.SelectBuilder,
+) (string, error) {
+	column, err := c.GetColumn(ctx, key)
+	if err != nil {
+		return "", err
+	}
+
+	tblFieldName, err := c.GetTableFieldName(ctx, key)
+	if err != nil {
+		return "", err
+	}
+
+	tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName)
+
+	// regular operators
+	switch operator {
+	// regular operators
+	case qbtypes.FilterOperatorEqual:
+		return sb.E(tblFieldName, value), nil
+	case qbtypes.FilterOperatorNotEqual:
+		return sb.NE(tblFieldName, value), nil
+	case qbtypes.FilterOperatorGreaterThan:
+		return sb.G(tblFieldName, value), nil
+	case qbtypes.FilterOperatorGreaterThanOrEq:
+		return sb.GE(tblFieldName, value), nil
+	case qbtypes.FilterOperatorLessThan:
+		return sb.LT(tblFieldName, value), nil
+	case qbtypes.FilterOperatorLessThanOrEq:
+		return sb.LE(tblFieldName, value), nil
+
+	// like and not like
+	case qbtypes.FilterOperatorLike:
+		return sb.Like(tblFieldName, value), nil
+	case qbtypes.FilterOperatorNotLike:
+		return sb.NotLike(tblFieldName, value), nil
+	case qbtypes.FilterOperatorILike:
+		return sb.ILike(tblFieldName, value), nil
+	case qbtypes.FilterOperatorNotILike:
+		return sb.NotILike(tblFieldName, value), nil
+
+	case qbtypes.FilterOperatorContains:
+		return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
+	case qbtypes.FilterOperatorNotContains:
+		return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
+
+	case qbtypes.FilterOperatorRegexp:
+		exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value))
+		return sb.And(exp), nil
+	case qbtypes.FilterOperatorNotRegexp:
+		exp := fmt.Sprintf(`not match(%s, %s)`, tblFieldName, sb.Var(value))
+		return sb.And(exp), nil
+	// between and not between
+	case qbtypes.FilterOperatorBetween:
+		values, ok := value.([]any)
+		if !ok {
+			return "", qbtypes.ErrBetweenValues
+		}
+		if len(values) != 2 {
+			return "", qbtypes.ErrBetweenValues
+		}
+		return sb.Between(tblFieldName, values[0], values[1]), nil
+	case qbtypes.FilterOperatorNotBetween:
+		values, ok := value.([]any)
+		if !ok {
+			return "", qbtypes.ErrBetweenValues
+		}
+		if len(values) != 2 {
+			return "", qbtypes.ErrBetweenValues
+		}
+		return sb.NotBetween(tblFieldName, values[0], values[1]), nil
+
+	// in and not in
+	case qbtypes.FilterOperatorIn:
+		values, ok := value.([]any)
+		if !ok {
+			return "", qbtypes.ErrInValues
+		}
+		return sb.In(tblFieldName, values...), nil
+	case qbtypes.FilterOperatorNotIn:
+		values, ok := value.([]any)
+		if !ok {
+			return "", qbtypes.ErrInValues
+		}
+		return sb.NotIn(tblFieldName, values...), nil
+
+	// exists and not exists
+	// but how could you live and have no story to tell
+	// in the UI based query builder, `exists` and `not exists` are used for
+	// key membership checks, so depending on the column type, the condition changes
+	case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
+		var value any
+		switch column.Type {
+		case schema.ColumnTypeString, schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}:
+			value = ""
+			if operator == qbtypes.FilterOperatorExists {
+				return sb.NE(tblFieldName, value), nil
+			} else {
+				return sb.E(tblFieldName, value), nil
+			}
+		case schema.ColumnTypeUInt64, schema.ColumnTypeUInt32, schema.ColumnTypeUInt8:
+			value = 0
+			if operator == qbtypes.FilterOperatorExists {
+				return sb.NE(tblFieldName, value), nil
+			} else {
+				return sb.E(tblFieldName, value), nil
+			}
+		case schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeString,
+		}, schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeBool,
+		}, schema.MapColumnType{
+			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+			ValueType: schema.ColumnTypeFloat64,
+		}:
+			leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
+			if key.Materialized {
+				leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
+			}
+			if operator == qbtypes.FilterOperatorExists {
+				return sb.E(leftOperand, true), nil
+			} else {
+				return sb.NE(leftOperand, true), nil
+			}
+		default:
+			return "", fmt.Errorf("exists operator is not supported for column type %s", column.Type)
+		}
+	}
+	return "", fmt.Errorf("unsupported operator: %v", operator)
+}

pkg/telemetrylogs/condition_builder_test.go

@@ -0,0 +1,620 @@
+package telemetrylogs
+
+import (
+	"context"
+	"testing"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/huandu/go-sqlbuilder"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetColumn(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := NewConditionBuilder()
+
+	testCases := []struct {
+		name          string
+		key           telemetrytypes.TelemetryFieldKey
+		expectedCol   *schema.Column
+		expectedError error
+	}{
+		{
+			name: "Resource field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+			},
+			expectedCol:   logsV2Columns["resources_string"],
+			expectedError: nil,
+		},
+		{
+			name: "Scope field - scope name",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "name",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   logsV2Columns["scope_name"],
+			expectedError: nil,
+		},
+		{
+			name: "Scope field - scope.name",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "scope.name",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   logsV2Columns["scope_name"],
+			expectedError: nil,
+		},
+		{
+			name: "Scope field - scope_name",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "scope_name",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   logsV2Columns["scope_name"],
+			expectedError: nil,
+		},
+		{
+			name: "Scope field - version",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "version",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   logsV2Columns["scope_version"],
+			expectedError: nil,
+		},
+		{
+			name: "Scope field - other scope field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "custom.scope.field",
+				FieldContext: telemetrytypes.FieldContextScope,
+			},
+			expectedCol:   logsV2Columns["scope_string"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - string type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			expectedCol:   logsV2Columns["attributes_string"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - number type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.size",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			},
+			expectedCol:   logsV2Columns["attributes_number"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - int64 type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.duration",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeInt64,
+			},
+			expectedCol:   logsV2Columns["attributes_number"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - float64 type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "cpu.utilization",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+			},
+			expectedCol:   logsV2Columns["attributes_number"],
+			expectedError: nil,
+		},
+		{
+			name: "Attribute field - bool type",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.success",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeBool,
+			},
+			expectedCol:   logsV2Columns["attributes_bool"],
+			expectedError: nil,
+		},
+		{
+			name: "Log field - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedCol:   logsV2Columns["timestamp"],
+			expectedError: nil,
+		},
+		{
+			name: "Log field - body",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "body",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedCol:   logsV2Columns["body"],
+			expectedError: nil,
+		},
+		{
+			name: "Log field - nonexistent",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "nonexistent_field",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedCol:   nil,
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+		{
+			name: "did_user_login",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "did_user_login",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeBool,
+			},
+			expectedCol:   logsV2Columns["attributes_bool"],
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			col, err := conditionBuilder.GetColumn(ctx, &tc.key)
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectedCol, col)
+			}
+		})
+	}
+}
+
+func TestGetFieldKeyName(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := &conditionBuilder{}
+
+	testCases := []struct {
+		name           string
+		key            telemetrytypes.TelemetryFieldKey
+		expectedResult string
+		expectedError  error
+	}{
+		{
+			name: "Simple column type - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedResult: "timestamp",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			expectedResult: "attributes_string['user.id']",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - number attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.size",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			},
+			expectedResult: "attributes_number['request.size']",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - bool attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.success",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeBool,
+			},
+			expectedResult: "attributes_bool['request.success']",
+			expectedError:  nil,
+		},
+		{
+			name: "Map column type - resource attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+			},
+			expectedResult: "resources_string['service.name']",
+			expectedError:  nil,
+		},
+		{
+			name: "Non-existent column",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "nonexistent_field",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			expectedResult: "",
+			expectedError:  qbtypes.ErrColumnNotFound,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := conditionBuilder.GetTableFieldName(ctx, &tc.key)
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.expectedResult, result)
+			}
+		})
+	}
+}
+
+func TestGetCondition(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := NewConditionBuilder()
+
+	testCases := []struct {
+		name          string
+		key           telemetrytypes.TelemetryFieldKey
+		operator      qbtypes.FilterOperator
+		value         any
+		expectedSQL   string
+		expectedError error
+	}{
+		{
+			name: "Equal operator - string",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "body",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorEqual,
+			value:         "error message",
+			expectedSQL:   "body = ?",
+			expectedError: nil,
+		},
+		{
+			name: "Not Equal operator - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorNotEqual,
+			value:         uint64(1617979338000000000),
+			expectedSQL:   "timestamp <> ?",
+			expectedError: nil,
+		},
+		{
+			name: "Greater Than operator - number attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.duration",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			},
+			operator:      qbtypes.FilterOperatorGreaterThan,
+			value:         float64(100),
+			expectedSQL:   "attributes_number['request.duration'] > ?",
+			expectedError: nil,
+		},
+		{
+			name: "Less Than operator - number attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "request.size",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			},
+			operator:      qbtypes.FilterOperatorLessThan,
+			value:         float64(1024),
+			expectedSQL:   "attributes_number['request.size'] < ?",
+			expectedError: nil,
+		},
+		{
+			name: "Greater Than Or Equal operator - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorGreaterThanOrEq,
+			value:         uint64(1617979338000000000),
+			expectedSQL:   "timestamp >= ?",
+			expectedError: nil,
+		},
+		{
+			name: "Less Than Or Equal operator - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorLessThanOrEq,
+			value:         uint64(1617979338000000000),
+			expectedSQL:   "timestamp <= ?",
+			expectedError: nil,
+		},
+		{
+			name: "Like operator - body",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "body",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorLike,
+			value:         "%error%",
+			expectedSQL:   "body LIKE ?",
+			expectedError: nil,
+		},
+		{
+			name: "Not Like operator - body",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "body",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorNotLike,
+			value:         "%error%",
+			expectedSQL:   "body NOT LIKE ?",
+			expectedError: nil,
+		},
+		{
+			name: "ILike operator - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorILike,
+			value:         "%admin%",
+			expectedSQL:   "WHERE LOWER(attributes_string['user.id']) LIKE LOWER(?)",
+			expectedError: nil,
+		},
+		{
+			name: "Not ILike operator - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorNotILike,
+			value:         "%admin%",
+			expectedSQL:   "WHERE LOWER(attributes_string['user.id']) NOT LIKE LOWER(?)",
+			expectedError: nil,
+		},
+		{
+			name: "Contains operator - string attribute",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorContains,
+			value:         "admin",
+			expectedSQL:   "WHERE LOWER(attributes_string['user.id']) LIKE LOWER(?)",
+			expectedError: nil,
+		},
+		{
+			name: "Between operator - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorBetween,
+			value:         []any{uint64(1617979338000000000), uint64(1617979348000000000)},
+			expectedSQL:   "timestamp BETWEEN ? AND ?",
+			expectedError: nil,
+		},
+		{
+			name: "Between operator - invalid value",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorBetween,
+			value:         "invalid",
+			expectedSQL:   "",
+			expectedError: qbtypes.ErrBetweenValues,
+		},
+		{
+			name: "Between operator - insufficient values",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorBetween,
+			value:         []any{uint64(1617979338000000000)},
+			expectedSQL:   "",
+			expectedError: qbtypes.ErrBetweenValues,
+		},
+		{
+			name: "Not Between operator - timestamp",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorNotBetween,
+			value:         []any{uint64(1617979338000000000), uint64(1617979348000000000)},
+			expectedSQL:   "timestamp NOT BETWEEN ? AND ?",
+			expectedError: nil,
+		},
+		{
+			name: "In operator - severity_text",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "severity_text",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorIn,
+			value:         []any{"error", "fatal", "critical"},
+			expectedSQL:   "severity_text IN (?, ?, ?)",
+			expectedError: nil,
+		},
+		{
+			name: "In operator - invalid value",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "severity_text",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorIn,
+			value:         "error",
+			expectedSQL:   "",
+			expectedError: qbtypes.ErrInValues,
+		},
+		{
+			name: "Not In operator - severity_text",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "severity_text",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorNotIn,
+			value:         []any{"debug", "info", "trace"},
+			expectedSQL:   "severity_text NOT IN (?, ?, ?)",
+			expectedError: nil,
+		},
+		{
+			name: "Exists operator - string field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "body",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorExists,
+			value:         nil,
+			expectedSQL:   "body <> ?",
+			expectedError: nil,
+		},
+		{
+			name: "Not Exists operator - string field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "body",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorNotExists,
+			value:         nil,
+			expectedSQL:   "body = ?",
+			expectedError: nil,
+		},
+		{
+			name: "Exists operator - number field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "timestamp",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorExists,
+			value:         nil,
+			expectedSQL:   "timestamp <> ?",
+			expectedError: nil,
+		},
+		{
+			name: "Exists operator - map field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorExists,
+			value:         nil,
+			expectedSQL:   "mapContains(attributes_string, 'user.id') = ?",
+			expectedError: nil,
+		},
+		{
+			name: "Not Exists operator - map field",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "user.id",
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			},
+			operator:      qbtypes.FilterOperatorNotExists,
+			value:         nil,
+			expectedSQL:   "mapContains(attributes_string, 'user.id') <> ?",
+			expectedError: nil,
+		},
+		{
+			name: "Non-existent column",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "nonexistent_field",
+				FieldContext: telemetrytypes.FieldContextLog,
+			},
+			operator:      qbtypes.FilterOperatorEqual,
+			value:         "value",
+			expectedSQL:   "",
+			expectedError: qbtypes.ErrColumnNotFound,
+		},
+	}
+
+	for _, tc := range testCases {
+		sb := sqlbuilder.NewSelectBuilder()
+		t.Run(tc.name, func(t *testing.T) {
+			cond, err := conditionBuilder.GetCondition(ctx, &tc.key, tc.operator, tc.value, sb)
+			sb.Where(cond)
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+				assert.Contains(t, sql, tc.expectedSQL)
+			}
+		})
+	}
+}
+
+func TestGetConditionMultiple(t *testing.T) {
+	ctx := context.Background()
+	conditionBuilder := NewConditionBuilder()
+
+	testCases := []struct {
+		name          string
+		keys          []*telemetrytypes.TelemetryFieldKey
+		operator      qbtypes.FilterOperator
+		value         any
+		expectedSQL   string
+		expectedError error
+	}{
+		{
+			name: "Equal operator - string",
+			keys: []*telemetrytypes.TelemetryFieldKey{
+				{
+					Name:         "body",
+					FieldContext: telemetrytypes.FieldContextLog,
+				},
+				{
+					Name:         "severity_text",
+					FieldContext: telemetrytypes.FieldContextLog,
+				},
+			},
+			operator:      qbtypes.FilterOperatorEqual,
+			value:         "error message",
+			expectedSQL:   "body = ? AND severity_text = ?",
+			expectedError: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		sb := sqlbuilder.NewSelectBuilder()
+		t.Run(tc.name, func(t *testing.T) {
+			var err error
+			for _, key := range tc.keys {
+				cond, err := conditionBuilder.GetCondition(ctx, key, tc.operator, tc.value, sb)
+				sb.Where(cond)
+				if err != nil {
+					t.Fatalf("Error getting condition for key %s: %v", key.Name, err)
+				}
+			}
+
+			if tc.expectedError != nil {
+				assert.Equal(t, tc.expectedError, err)
+			} else {
+				require.NoError(t, err)
+				sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+				assert.Contains(t, sql, tc.expectedSQL)
+			}
+		})
+	}
+}

pkg/telemetrylogs/tables.go

@@ -0,0 +1,9 @@
+package telemetrylogs
+
+const (
+	DBName                        = "signoz_logs"
+	LogsV2TableName               = "distributed_logs_v2"
+	LogsV2LocalTableName          = "logs_v2"
+	TagAttributesV2TableName      = "distributed_tag_attributes_v2"
+	TagAttributesV2LocalTableName = "tag_attributes_v2"
+)

pkg/types/querybuildertypes/querybuildertypesv5/qb.go

@@ -4,10 +4,17 @@ import (
 	"context"
 
 	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/huandu/go-sqlbuilder"
 )
 
+var (
+	ErrColumnNotFound = errors.Newf(errors.TypeNotFound, errors.CodeNotFound, "column not found")
+	ErrBetweenValues  = errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "(not) between operator requires two values")
+	ErrInValues       = errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "(not) in operator requires a list of values")
+)
+
 // FilterOperator is the operator for the filter.
 type FilterOperator int
 

pkg/types/telemetrytypes/field.go

@@ -86,11 +86,11 @@ func GetFieldKeyFromKeyText(key string) TelemetryFieldKey {
 	return fieldKeySelector
 }
 
-func FieldKeyToMaterializedColumnName(key TelemetryFieldKey) string {
+func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
 	return fmt.Sprintf("%s_%s_%s", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$"))
 }
 
-func FieldKeyToMaterializedColumnNameForExists(key TelemetryFieldKey) string {
+func FieldKeyToMaterializedColumnNameForExists(key *TelemetryFieldKey) string {
 	return fmt.Sprintf("%s_%s_%s_exists", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$"))
 }
 
@@ -123,3 +123,52 @@ type FieldValueSelector struct {
 	Value         string `json:"value"`
 	Limit         int    `json:"limit"`
 }
+
+func DataTypeCollisionHandledFieldName(key *TelemetryFieldKey, value any, tblFieldName string) (string, any) {
+	// This block of code exists to handle the data type collisions
+	// We don't want to fail the requests when there is a key with more than one data type
+	// Let's take an example of `http.status_code`, and consider user sent a string value and number value
+	// When they search for `http.status_code=200`, we will search across both the number columns and string columns
+	// and return the results from both the columns
+	// While we expect user not to send the mixed data types, it inevitably happens
+	// So we handle the data type collisions here
+	switch key.FieldDataType {
+	case FieldDataTypeString:
+		switch value.(type) {
+		case float64:
+			// try to convert the string value to to number
+			tblFieldName = fmt.Sprintf(`toFloat64OrNull(%s)`, tblFieldName)
+		case []any:
+			areFloats := true
+			for _, v := range value.([]any) {
+				if _, ok := v.(float64); !ok {
+					areFloats = false
+					break
+				}
+			}
+			if areFloats {
+				tblFieldName = fmt.Sprintf(`toFloat64OrNull(%s)`, tblFieldName)
+			}
+		case bool:
+			// we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string
+			value = fmt.Sprintf("%t", value)
+		case string:
+			// nothing to do
+		}
+	case FieldDataTypeFloat64, FieldDataTypeInt64, FieldDataTypeNumber:
+		switch value.(type) {
+		case string:
+			// try to convert the string value to to number
+			tblFieldName = fmt.Sprintf(`toString(%s)`, tblFieldName)
+		case float64:
+			// nothing to do
+		}
+	case FieldDataTypeBool:
+		switch value.(type) {
+		case string:
+			// try to convert the string value to to number
+			tblFieldName = fmt.Sprintf(`toString(%s)`, tblFieldName)
+		}
+	}
+	return tblFieldName, value
+}