chore: use attributes table for metric keys and values (#7680)

pkg/apis/fields/api.go

@@ -28,8 +28,7 @@ func NewAPI(telemetryStore telemetrystore.TelemetryStore) *API {
 		telemetrytraces.TagAttributesV2TableName,
 		telemetrytraces.SpanIndexV3TableName,
 		telemetrymetrics.DBName,
-		telemetrymetrics.TimeseriesV41weekTableName,
-		telemetrymetrics.TimeseriesV41weekLocalTableName,
+		telemetrymetrics.AttributesMetadataTableName,
 		telemetrylogs.DBName,
 		telemetrylogs.LogsV2TableName,
 		telemetrylogs.TagAttributesV2TableName,

pkg/parser/grammar/where_clause_visitor.go

@@ -20,18 +20,18 @@ import (
 type WhereClauseVisitor struct {
 	conditionBuilder qbtypes.ConditionBuilder
 	warnings         []error
-	fieldKeys        map[string][]telemetrytypes.TelemetryFieldKey
+	fieldKeys        map[string][]*telemetrytypes.TelemetryFieldKey
 	errors           []error
 	builder          *sqlbuilder.SelectBuilder
-	fullTextColumn   telemetrytypes.TelemetryFieldKey
+	fullTextColumn   *telemetrytypes.TelemetryFieldKey
 }
 
 // NewWhereClauseVisitor creates a new WhereClauseVisitor
 func NewWhereClauseVisitor(
 	conditionBuilder qbtypes.ConditionBuilder,
-	fieldKeys map[string][]telemetrytypes.TelemetryFieldKey,
+	fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
 	builder *sqlbuilder.SelectBuilder,
-	fullTextColumn telemetrytypes.TelemetryFieldKey,
+	fullTextColumn *telemetrytypes.TelemetryFieldKey,
 ) *WhereClauseVisitor {
 	return &WhereClauseVisitor{
 		conditionBuilder: conditionBuilder,
@@ -72,10 +72,10 @@ func (l *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol
 // PrepareWhereClause generates a ClickHouse compatible WHERE clause from the filter query
 func PrepareWhereClause(
 	query string,
-	fieldKeys map[string][]telemetrytypes.TelemetryFieldKey,
+	fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
 	conditionBuilder qbtypes.ConditionBuilder,
-	fullTextColumn telemetrytypes.TelemetryFieldKey,
-) (string, []any, []error, error) {
+	fullTextColumn *telemetrytypes.TelemetryFieldKey,
+) (*sqlbuilder.WhereClause, []error, error) {
 	// Setup the ANTLR parsing pipeline
 	input := antlr.NewInputStream(query)
 	lexer := NewFilterQueryLexer(input)
@@ -107,7 +107,7 @@ func PrepareWhereClause(
 			len(parserErrorListener.Errors),
 			parserErrorListener.Errors,
 		)
-		return "", nil, nil, combinedErrors
+		return nil, nil, combinedErrors
 	}
 
 	// Visit the parse tree with our ClickHouse visitor
@@ -122,12 +122,12 @@ func PrepareWhereClause(
 			len(visitor.errors),
 			visitor.errors,
 		)
-		return "", nil, nil, combinedErrors
+		return nil, nil, combinedErrors
 	}
 
-	whereClause, args := visitor.builder.Where(cond).BuildWithFlavor(sqlbuilder.ClickHouse)
+	whereClause := sqlbuilder.NewWhereClause().AddWhereExpr(visitor.builder.Args, cond)
 
-	return whereClause, args, visitor.warnings, nil
+	return whereClause, visitor.warnings, nil
 }
 
 // Visit dispatches to the specific visit method based on node type
@@ -250,7 +250,7 @@ func (v *WhereClauseVisitor) VisitPrimary(ctx *PrimaryContext) any {
 		if keyCtx, ok := child.(*KeyContext); ok {
 			// create a full text search condition on the body field
 			keyText := keyCtx.GetText()
-			cond, err := v.conditionBuilder.GetCondition(context.Background(), &v.fullTextColumn, qbtypes.FilterOperatorRegexp, keyText, v.builder)
+			cond, err := v.conditionBuilder.GetCondition(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, keyText, v.builder)
 			if err != nil {
 				return ""
 			}
@@ -263,7 +263,7 @@ func (v *WhereClauseVisitor) VisitPrimary(ctx *PrimaryContext) any {
 
 // VisitComparison handles all comparison operators
 func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any {
-	keys := v.Visit(ctx.Key()).([]telemetrytypes.TelemetryFieldKey)
+	keys := v.Visit(ctx.Key()).([]*telemetrytypes.TelemetryFieldKey)
 
 	// Handle EXISTS specially
 	if ctx.EXISTS() != nil {
@@ -273,7 +273,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any {
 		}
 		var conds []string
 		for _, key := range keys {
-			condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, nil, v.builder)
+			condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, nil, v.builder)
 			if err != nil {
 				return ""
 			}
@@ -291,7 +291,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any {
 		}
 		var conds []string
 		for _, key := range keys {
-			condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, values, v.builder)
+			condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, values, v.builder)
 			if err != nil {
 				return ""
 			}
@@ -317,7 +317,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any {
 
 		var conds []string
 		for _, key := range keys {
-			condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, []any{value1, value2}, v.builder)
+			condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, []any{value1, value2}, v.builder)
 			if err != nil {
 				return ""
 			}
@@ -366,7 +366,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any {
 
 		var conds []string
 		for _, key := range keys {
-			condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, value, v.builder)
+			condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, value, v.builder)
 			if err != nil {
 				return ""
 			}
@@ -404,7 +404,7 @@ func (v *WhereClauseVisitor) VisitValueList(ctx *ValueListContext) any {
 func (v *WhereClauseVisitor) VisitFullText(ctx *FullTextContext) any {
 	// remove quotes from the quotedText
 	quotedText := strings.Trim(ctx.QUOTED_TEXT().GetText(), "\"'")
-	cond, err := v.conditionBuilder.GetCondition(context.Background(), &v.fullTextColumn, qbtypes.FilterOperatorRegexp, quotedText, v.builder)
+	cond, err := v.conditionBuilder.GetCondition(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, quotedText, v.builder)
 	if err != nil {
 		return ""
 	}
@@ -443,7 +443,7 @@ func (v *WhereClauseVisitor) VisitFunctionCall(ctx *FunctionCallContext) any {
 		return ""
 	}
 
-	keys, ok := params[0].([]telemetrytypes.TelemetryFieldKey)
+	keys, ok := params[0].([]*telemetrytypes.TelemetryFieldKey)
 	if !ok {
 		v.errors = append(v.errors, errors.Newf(
 			errors.TypeInvalidInput,
@@ -459,9 +459,9 @@ func (v *WhereClauseVisitor) VisitFunctionCall(ctx *FunctionCallContext) any {
 		var fieldName string
 
 		if strings.HasPrefix(key.Name, telemetrylogs.BodyJSONStringSearchPrefix) {
-			fieldName, _ = telemetrylogs.GetBodyJSONKey(context.Background(), &key, qbtypes.FilterOperatorUnknown, value)
+			fieldName, _ = telemetrylogs.GetBodyJSONKey(context.Background(), key, qbtypes.FilterOperatorUnknown, value)
 		} else {
-			fieldName, _ = v.conditionBuilder.GetTableFieldName(context.Background(), &key)
+			fieldName, _ = v.conditionBuilder.GetTableFieldName(context.Background(), key)
 		}
 
 		var cond string
@@ -557,7 +557,7 @@ func (v *WhereClauseVisitor) VisitKey(ctx *KeyContext) any {
 	// Since it will ORed with the fieldKeysForName, it will not result empty
 	// when either of them have values
 	if strings.HasPrefix(fieldKey.Name, telemetrylogs.BodyJSONStringSearchPrefix) {
-		fieldKeysForName = append(fieldKeysForName, fieldKey)
+		fieldKeysForName = append(fieldKeysForName, &fieldKey)
 	}
 
 	// TODO(srikanthccv): do we want to return an error here?

pkg/parser/grammar/where_clause_visitor_test.go

@@ -8,19 +8,20 @@ import (
 	"github.com/SigNoz/signoz/pkg/telemetrylogs"
 	"github.com/SigNoz/signoz/pkg/telemetrytraces"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	sqlbuilder "github.com/huandu/go-sqlbuilder"
 )
 
 func TestConvertToClickHouseLogsQuery(t *testing.T) {
 	cases := []struct {
 		name                 string
-		fieldKeys            map[string][]telemetrytypes.TelemetryFieldKey
+		fieldKeys            map[string][]*telemetrytypes.TelemetryFieldKey
 		query                string
 		expectedSearchString string
 		expectedSearchArgs   []any
 	}{
 		{
 			name: "test-simple-service-name-filter",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"service.name": {
 					{
 						Name:          "service.name",
@@ -36,7 +37,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "test-simple-service-name-filter-with-materialised-column",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"service.name": {
 					{
 						Name:          "service.name",
@@ -53,7 +54,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "http-status-code-multiple-data-types",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"http.status_code": {
 					{
 						Name:          "http.status_code",
@@ -75,7 +76,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "http-status-code-multiple-data-types-between-operator",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"http.status_code": {
 					{
 						Name:          "http.status_code",
@@ -97,7 +98,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "response-body-multiple-data-types-string-contains",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"response.body": {
 					{
 						Name:          "response.body",
@@ -119,7 +120,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "search-on-top-level-key",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"severity_text": {
 					{
 						Name:          "severity_text",
@@ -135,7 +136,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "search-on-top-level-key-conflict-with-attribute",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"severity_text": {
 					{
 						Name:          "severity_text",
@@ -157,7 +158,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "collision-with-attribute-field-and-resource-attribute",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"k8s.namespace.name": {
 					{
 						Name:          "k8s.namespace.name",
@@ -179,7 +180,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "collision-with-attribute-field-and-resource-attribute-materialised-column",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"k8s.namespace.name": {
 					{
 						Name:          "k8s.namespace.name",
@@ -202,7 +203,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "boolean-collision-with-attribute-field-and-data-type-boolean",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"did_user_login": {
 					{
 						Name:          "did_user_login",
@@ -224,7 +225,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name: "regexp-search",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"k8s.namespace.name": {
 					{
 						Name:          "k8s.namespace.name",
@@ -248,42 +249,42 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name:                 "full-text-search-multiple-words",
-			fieldKeys:            map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:            map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                "waiting for response",
 			expectedSearchString: "WHERE ((match(body, ?)) AND (match(body, ?)) AND (match(body, ?)))",
 			expectedSearchArgs:   []any{"waiting", "for", "response"},
 		},
 		{
 			name:                 "full-text-search-with-phrase-search",
-			fieldKeys:            map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:            map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                `"waiting for response"`,
 			expectedSearchString: "WHERE (match(body, ?))",
 			expectedSearchArgs:   []any{"waiting for response"},
 		},
 		{
 			name:                 "full-text-search-with-word-and-not-word",
-			fieldKeys:            map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:            map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                "error NOT buggy_app",
 			expectedSearchString: "WHERE ((match(body, ?)) AND NOT ((match(body, ?))))",
 			expectedSearchArgs:   []any{"error", "buggy_app"},
 		},
 		{
 			name:                 "full-text-search-with-word-and-not-word-and-not-word",
-			fieldKeys:            map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:            map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                "error NOT buggy_app NOT redis",
 			expectedSearchString: "WHERE ((match(body, ?)) AND NOT ((match(body, ?))) AND NOT ((match(body, ?))))",
 			expectedSearchArgs:   []any{"error", "buggy_app", "redis"},
 		},
 		{
 			name:                 "full-text-search-with-word-and-not-word-and-not-word-tricky",
-			fieldKeys:            map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:            map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                "error NOT buggy_app OR redis",
 			expectedSearchString: "WHERE (((match(body, ?)) AND NOT ((match(body, ?)))) OR (match(body, ?)))",
 			expectedSearchArgs:   []any{"error", "buggy_app", "redis"},
 		},
 		{
 			name: "has-function",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"service.name": {
 					{
 						Name:         "service.name",
@@ -305,14 +306,14 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 		},
 		{
 			name:                 "has-from-list-of-values",
-			fieldKeys:            map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:            map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                "has(body.payload.user_ids[*], 'u1292')",
 			expectedSearchString: "WHERE (has(JSONExtract(JSON_QUERY(body, '$.payload.user_ids[*]'), 'Array(String)'), ?))",
 			expectedSearchArgs:   []any{"u1292"},
 		},
 		{
 			name: "body-json-search-that-also-has-attribute-with-same-name",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"http.status_code": {
 					{
 						Name:          "http.status_code",
@@ -331,15 +332,18 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 
 	for _, c := range cases {
 		t.Logf("running test %s", c.name)
-		chQuery, chQueryArgs, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrylogs.NewConditionBuilder(), telemetrytypes.TelemetryFieldKey{
+		whereClause, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrylogs.NewConditionBuilder(), &telemetrytypes.TelemetryFieldKey{
 			Name:          "body",
 			Signal:        telemetrytypes.SignalLogs,
 			FieldContext:  telemetrytypes.FieldContextLog,
 			FieldDataType: telemetrytypes.FieldDataTypeString,
 		})
+
 		if err != nil {
 			t.Errorf("Error converting query to ClickHouse: %v", err)
 		}
+		chQuery, chQueryArgs := whereClause.BuildWithFlavor(sqlbuilder.ClickHouse)
+
 		if chQuery != c.expectedSearchString {
 			t.Errorf("Expected %s, got %s", c.expectedSearchString, chQuery)
 		}
@@ -355,14 +359,14 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) {
 func TestConvertToClickHouseSpansQuery(t *testing.T) {
 	cases := []struct {
 		name                 string
-		fieldKeys            map[string][]telemetrytypes.TelemetryFieldKey
+		fieldKeys            map[string][]*telemetrytypes.TelemetryFieldKey
 		query                string
 		expectedSearchString string
 		expectedSearchArgs   []any
 	}{
 		{
 			name: "test-simple-service-name-filter",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"service.name": {
 					{
 						Name:          "service.name",
@@ -378,7 +382,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "test-simple-service-name-filter-with-materialised-column",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"service.name": {
 					{
 						Name:          "service.name",
@@ -395,7 +399,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "http-status-code-multiple-data-types",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"http.status_code": {
 					{
 						Name:          "http.status_code",
@@ -417,7 +421,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "http-status-code-multiple-data-types-between-operator",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"http.status_code": {
 					{
 						Name:          "http.status_code",
@@ -439,7 +443,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "response-body-multiple-data-types-string-contains",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"response.body": {
 					{
 						Name:          "response.body",
@@ -461,7 +465,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "collision-with-attribute-field-and-resource-attribute",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"k8s.namespace.name": {
 					{
 						Name:          "k8s.namespace.name",
@@ -483,7 +487,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "collision-with-attribute-field-and-resource-attribute-materialised-column",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"k8s.namespace.name": {
 					{
 						Name:          "k8s.namespace.name",
@@ -506,7 +510,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "boolean-collision-with-attribute-field-and-data-type-boolean",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"did_user_login": {
 					{
 						Name:          "did_user_login",
@@ -528,7 +532,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		},
 		{
 			name: "regexp-search",
-			fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{
+			fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
 				"k8s.namespace.name": {
 					{
 						Name:          "k8s.namespace.name",
@@ -553,7 +557,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		chQuery, chQueryArgs, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), telemetrytypes.TelemetryFieldKey{
+		whereClause, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), &telemetrytypes.TelemetryFieldKey{
 			Name:          "dummy",
 			Signal:        telemetrytypes.SignalTraces,
 			FieldContext:  telemetrytypes.FieldContextSpan,
@@ -562,6 +566,9 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 		if err != nil {
 			t.Errorf("Error converting query to ClickHouse: %v", err)
 		}
+
+		chQuery, chQueryArgs := whereClause.BuildWithFlavor(sqlbuilder.ClickHouse)
+
 		if chQuery != c.expectedSearchString {
 			t.Errorf("Expected %s, got %s", c.expectedSearchString, chQuery)
 		}
@@ -577,7 +584,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) {
 func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) {
 	cases := []struct {
 		name                   string
-		fieldKeys              map[string][]telemetrytypes.TelemetryFieldKey
+		fieldKeys              map[string][]*telemetrytypes.TelemetryFieldKey
 		query                  string
 		expectedSearchString   string
 		expectedSearchArgs     []any
@@ -586,7 +593,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) {
 	}{
 		{
 			name:                   "has-function-with-multiple-values",
-			fieldKeys:              map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:              map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                  "key.that.does.not.exist = 'redis'",
 			expectedSearchString:   "",
 			expectedSearchArgs:     []any{},
@@ -595,7 +602,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) {
 		},
 		{
 			name:                   "unknown-function",
-			fieldKeys:              map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:              map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                  "unknown.function()",
 			expectedSearchString:   "",
 			expectedSearchArgs:     []any{},
@@ -604,7 +611,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) {
 		},
 		{
 			name:                   "has-function-not-enough-params",
-			fieldKeys:              map[string][]telemetrytypes.TelemetryFieldKey{},
+			fieldKeys:              map[string][]*telemetrytypes.TelemetryFieldKey{},
 			query:                  "has(key.that.does.not.exist)",
 			expectedSearchString:   "",
 			expectedSearchArgs:     []any{},
@@ -614,7 +621,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		_, _, warnings, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), telemetrytypes.TelemetryFieldKey{
+		_, warnings, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), &telemetrytypes.TelemetryFieldKey{
 			Name:          "dummy",
 			Signal:        telemetrytypes.SignalTraces,
 			FieldContext:  telemetrytypes.FieldContextSpan,

pkg/telemetrymetadata/condition_builder.go

@@ -80,6 +80,8 @@ func (c *conditionBuilder) GetCondition(
 		return "", nil
 	}
 
+	tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName)
+
 	// key must exists to apply main filter
 	containsExp := fmt.Sprintf("mapContains(%s, %s)", column.Name, sb.Var(key.Name))
 

pkg/telemetrymetadata/metadata.go

@@ -3,9 +3,9 @@ package telemetrymetadata
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	parser "github.com/SigNoz/signoz/pkg/parser/grammar"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -28,7 +28,6 @@ type telemetryMetaStore struct {
 	indexV3TblName         string
 	metricsDBName          string
 	metricsFieldsTblName   string
-	timeseries1WTblName    string
 	logsDBName             string
 	logsFieldsTblName      string
 	logsV2TblName          string
@@ -45,7 +44,6 @@ func NewTelemetryMetaStore(
 	indexV3TblName string,
 	metricsDBName string,
 	metricsFieldsTblName string,
-	timeseries1WTblName string,
 	logsDBName string,
 	logsV2TblName string,
 	logsFieldsTblName string,
@@ -53,13 +51,13 @@ func NewTelemetryMetaStore(
 	relatedMetadataTblName string,
 ) telemetrytypes.MetadataStore {
 	return &telemetryMetaStore{
+
 		telemetrystore:         telemetrystore,
 		tracesDBName:           tracesDBName,
 		tracesFieldsTblName:    tracesFieldsTblName,
 		indexV3TblName:         indexV3TblName,
 		metricsDBName:          metricsDBName,
 		metricsFieldsTblName:   metricsFieldsTblName,
-		timeseries1WTblName:    timeseries1WTblName,
 		logsDBName:             logsDBName,
 		logsV2TblName:          logsV2TblName,
 		logsFieldsTblName:      logsFieldsTblName,
@@ -79,7 +77,16 @@ func (t *telemetryMetaStore) tracesTblStatementToFieldKeys(ctx context.Context)
 		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTblStatement.Error())
 	}
 
-	return ExtractFieldKeysFromTblStatement(statements[0].Statement)
+	materialisedKeys, err := ExtractFieldKeysFromTblStatement(statements[0].Statement)
+	if err != nil {
+		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTracesKeys.Error())
+	}
+
+	for idx := range materialisedKeys {
+		materialisedKeys[idx].Signal = telemetrytypes.SignalTraces
+	}
+
+	return materialisedKeys, nil
 }
 
 // getTracesKeys returns the keys from the spans that match the field selection criteria
@@ -175,6 +182,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
 		if !ok {
 			key = &telemetrytypes.TelemetryFieldKey{
 				Name:          name,
+				Signal:        telemetrytypes.SignalTraces,
 				FieldContext:  fieldContext,
 				FieldDataType: fieldDataType,
 			}
@@ -199,7 +207,16 @@ func (t *telemetryMetaStore) logsTblStatementToFieldKeys(ctx context.Context) ([
 		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTblStatement.Error())
 	}
 
-	return ExtractFieldKeysFromTblStatement(statements[0].Statement)
+	materialisedKeys, err := ExtractFieldKeysFromTblStatement(statements[0].Statement)
+	if err != nil {
+		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
+	}
+
+	for idx := range materialisedKeys {
+		materialisedKeys[idx].Signal = telemetrytypes.SignalLogs
+	}
+
+	return materialisedKeys, nil
 }
 
 // getLogsKeys returns the keys from the spans that match the field selection criteria
@@ -293,6 +310,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 		if !ok {
 			key = &telemetrytypes.TelemetryFieldKey{
 				Name:          name,
+				Signal:        telemetrytypes.SignalLogs,
 				FieldContext:  fieldContext,
 				FieldDataType: fieldDataType,
 			}
@@ -309,52 +327,60 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 }
 
 // getMetricsKeys returns the keys from the metrics that match the field selection criteria
-// TODO(srikanthccv): update the implementation after the dot metrics migration is done
 func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, error) {
 	if len(fieldKeySelectors) == 0 {
 		return nil, nil
 	}
 
-	var whereClause, innerWhereClause string
+	sb := sqlbuilder.
+		Select("attr_name as name", "attr_type as field_context", "attr_datatype as field_data_type", `
+			CASE
+				WHEN attr_type = 'resource' THEN 1
+				WHEN attr_type = 'scope' THEN 2
+				WHEN attr_type = 'point' THEN 3
+				ELSE 4
+			END as priority`).
+		From(t.metricsDBName + "." + t.metricsFieldsTblName)
+
 	var limit int
-	args := []any{}
 
+	conds := []string{}
 	for _, fieldKeySelector := range fieldKeySelectors {
-		if fieldKeySelector.MetricContext != nil {
-			innerWhereClause += "metric_name IN ? AND"
-			args = append(args, fieldKeySelector.MetricContext.MetricName)
-		}
-	}
-	innerWhereClause += " __normalized = true"
-
-	for idx, fieldKeySelector := range fieldKeySelectors {
+		fieldConds := []string{}
 		if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
-			whereClause += "(distinctTagKey = ? AND distinctTagKey NOT LIKE '\\_\\_%%')"
-			args = append(args, fieldKeySelector.Name)
+			fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
 		} else {
-			whereClause += "(distinctTagKey ILIKE ? AND distinctTagKey NOT LIKE '\\_\\_%%')"
-			args = append(args, fmt.Sprintf("%%%s%%", fieldKeySelector.Name))
+			fieldConds = append(fieldConds, sb.Like("attr_name", "%"+fieldKeySelector.Name+"%"))
 		}
-		if idx != len(fieldKeySelectors)-1 {
-			whereClause += " OR "
+
+		if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified {
+			fieldConds = append(fieldConds, sb.E("attr_type", fieldKeySelector.FieldContext.TagType()))
 		}
+
+		if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
+			fieldConds = append(fieldConds, sb.E("attr_datatype", fieldKeySelector.FieldDataType.TagDataType()))
+		}
+
+		if fieldKeySelector.MetricContext != nil {
+			fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
+		}
+
+		conds = append(conds, sb.And(fieldConds...))
 		limit += fieldKeySelector.Limit
 	}
-	args = append(args, limit)
+	sb.Where(sb.Or(conds...))
 
-	query := fmt.Sprintf(`
-		SELECT
-			arrayJoin(tagKeys) AS distinctTagKey
-		FROM (
-			SELECT JSONExtractKeys(labels) AS tagKeys
-			FROM %s.%s
-			WHERE `+innerWhereClause+`
-			GROUP BY tagKeys
-		)
-		WHERE `+whereClause+`
-		GROUP BY distinctTagKey
-		LIMIT ?
-	`, t.metricsDBName, t.timeseries1WTblName)
+	if limit == 0 {
+		limit = 1000
+	}
+
+	mainSb := sqlbuilder.Select("name", "field_context", "field_data_type", "max(priority) as priority")
+	mainSb.From(mainSb.BuilderAs(sb, "sub_query"))
+	mainSb.GroupBy("name", "field_context", "field_data_type")
+	mainSb.OrderBy("priority")
+	mainSb.Limit(limit)
+
+	query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
 
 	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
 	if err != nil {
@@ -365,16 +391,19 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
 	keys := []*telemetrytypes.TelemetryFieldKey{}
 	for rows.Next() {
 		var name string
-		err = rows.Scan(&name)
+		var fieldContext telemetrytypes.FieldContext
+		var fieldDataType telemetrytypes.FieldDataType
+		var priority uint8
+		err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
 		if err != nil {
 			return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error())
 		}
-		key := &telemetrytypes.TelemetryFieldKey{
+		keys = append(keys, &telemetrytypes.TelemetryFieldKey{
 			Name:          name,
-			FieldContext:  telemetrytypes.FieldContextAttribute,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
-		}
-		keys = append(keys, key)
+			Signal:        telemetrytypes.SignalMetrics,
+			FieldContext:  fieldContext,
+			FieldDataType: fieldDataType,
+		})
 	}
 
 	if rows.Err() != nil {
@@ -486,20 +515,10 @@ func (t *telemetryMetaStore) GetKey(ctx context.Context, fieldKeySelector *telem
 
 func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) ([]string, error) {
 
-	args := []any{}
-
-	var andConditions []string
-
-	andConditions = append(andConditions, `unix_milli >= ?`)
-	args = append(args, fieldValueSelector.StartUnixMilli)
-
-	andConditions = append(andConditions, `unix_milli <= ?`)
-	args = append(args, fieldValueSelector.EndUnixMilli)
-
-	if len(fieldValueSelector.ExistingQuery) != 0 {
-		// TODO(srikanthccv): add the existing query to the where clause
+	// nothing to return as "related" value if there is nothing to filter on
+	if fieldValueSelector.ExistingQuery == "" {
+		return nil, nil
 	}
-	whereClause := strings.Join(andConditions, " AND ")
 
 	key := telemetrytypes.TelemetryFieldKey{
 		Name:          fieldValueSelector.Name,
@@ -508,20 +527,65 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
 		FieldDataType: fieldValueSelector.FieldDataType,
 	}
 
-	// TODO(srikanthccv): add the select column
-	selectColumn, _ := t.conditionBuilder.GetTableFieldName(ctx, &key)
+	selectColumn, err := t.conditionBuilder.GetTableFieldName(ctx, &key)
 
-	args = append(args, fieldValueSelector.Limit)
-	filterSubQuery := fmt.Sprintf(
-		"SELECT DISTINCT %s FROM %s.%s WHERE %s LIMIT ?",
-		selectColumn,
-		t.relatedMetadataDBName,
-		t.relatedMetadataTblName,
-		whereClause,
-	)
-	zap.L().Debug("filterSubQuery for related values", zap.String("query", filterSubQuery), zap.Any("args", args))
+	if err != nil {
+		// we don't have a explicit column to select from the related metadata table
+		// so we will select either from resource_attributes or attributes table
+		// in that order
+		resourceColumn, _ := t.conditionBuilder.GetTableFieldName(ctx, &telemetrytypes.TelemetryFieldKey{
+			Name:          key.Name,
+			FieldContext:  telemetrytypes.FieldContextResource,
+			FieldDataType: telemetrytypes.FieldDataTypeString,
+		})
+		attributeColumn, _ := t.conditionBuilder.GetTableFieldName(ctx, &telemetrytypes.TelemetryFieldKey{
+			Name:          key.Name,
+			FieldContext:  telemetrytypes.FieldContextAttribute,
+			FieldDataType: telemetrytypes.FieldDataTypeString,
+		})
+		selectColumn = fmt.Sprintf("if(notEmpty(%s), %s, %s)", resourceColumn, resourceColumn, attributeColumn)
+	}
 
-	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, filterSubQuery, args...)
+	sb := sqlbuilder.Select("DISTINCT " + selectColumn).From(t.relatedMetadataDBName + "." + t.relatedMetadataTblName)
+
+	if len(fieldValueSelector.ExistingQuery) != 0 {
+		keysSelectors, err := parser.QueryStringToKeysSelectors(fieldValueSelector.ExistingQuery)
+
+		if err == nil {
+			for idx := range keysSelectors {
+				keysSelectors[idx].Signal = fieldValueSelector.Signal
+			}
+			keys, err := t.GetKeysMulti(ctx, keysSelectors)
+			if err == nil {
+				whereClause, _, err := parser.PrepareWhereClause(fieldValueSelector.ExistingQuery, keys, t.conditionBuilder, &telemetrytypes.TelemetryFieldKey{})
+				if err == nil {
+					sb.AddWhereClause(whereClause)
+				} else {
+					zap.L().Warn("error parsing existing query for related values", zap.Error(err))
+				}
+			}
+		}
+	}
+
+	if fieldValueSelector.StartUnixMilli != 0 {
+		sb.Where(sb.GE("unix_milli", fieldValueSelector.StartUnixMilli))
+	}
+
+	if fieldValueSelector.EndUnixMilli != 0 {
+		sb.Where(sb.LE("unix_milli", fieldValueSelector.EndUnixMilli))
+	}
+
+	if fieldValueSelector.Limit != 0 {
+		sb.Limit(fieldValueSelector.Limit)
+	} else {
+		sb.Limit(50)
+	}
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+
+	zap.L().Debug("query for related values", zap.String("query", query), zap.Any("args", args))
+
+	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
 	if err != nil {
 		return nil, ErrFailedToGetRelatedValues
 	}
@@ -668,9 +732,66 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
 	return values, nil
 }
 
-func (t *telemetryMetaStore) getMetricFieldValues(_ context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) {
-	// TODO(srikanthccv): implement this. use new tables?
-	return nil, nil
+func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) {
+	sb := sqlbuilder.
+		Select("DISTINCT attr_string_value").
+		From(t.metricsDBName + "." + t.metricsFieldsTblName)
+
+	if fieldValueSelector.Name != "" {
+		sb.Where(sb.E("attr_name", fieldValueSelector.Name))
+	}
+
+	if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
+		sb.And(sb.E("attr_type", fieldValueSelector.FieldContext.TagType()))
+	}
+
+	if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
+		sb.And(sb.E("attr_datatype", fieldValueSelector.FieldDataType.TagDataType()))
+	}
+
+	if fieldValueSelector.MetricContext != nil {
+		sb.And(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
+	}
+
+	if fieldValueSelector.StartUnixMilli > 0 {
+		sb.And(sb.GE("last_reported_unix_milli", fieldValueSelector.StartUnixMilli))
+	}
+
+	if fieldValueSelector.EndUnixMilli > 0 {
+		sb.And(sb.LE("first_reported_unix_milli", fieldValueSelector.EndUnixMilli))
+	}
+
+	if fieldValueSelector.Value != "" {
+		if fieldValueSelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
+			sb.And(sb.E("attr_string_value", fieldValueSelector.Value))
+		} else {
+			sb.And(sb.Like("attr_string_value", "%"+fieldValueSelector.Value+"%"))
+		}
+	}
+
+	if fieldValueSelector.Limit > 0 {
+		sb.Limit(fieldValueSelector.Limit)
+	} else {
+		sb.Limit(50)
+	}
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+
+	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error())
+	}
+	defer rows.Close()
+
+	values := &telemetrytypes.TelemetryFieldValues{}
+	for rows.Next() {
+		var stringValue string
+		if err := rows.Scan(&stringValue); err != nil {
+			return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error())
+		}
+		values.StringValues = append(values.StringValues, stringValue)
+	}
+	return values, nil
 }
 
 func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) {

pkg/telemetrymetadata/metadata_test.go

@@ -39,8 +39,7 @@ func TestGetKeys(t *testing.T) {
 		telemetrytraces.TagAttributesV2TableName,
 		telemetrytraces.SpanIndexV3TableName,
 		telemetrymetrics.DBName,
-		telemetrymetrics.TimeseriesV41weekTableName,
-		telemetrymetrics.TimeseriesV41weekTableName,
+		telemetrymetrics.AttributesMetadataTableName,
 		telemetrylogs.DBName,
 		telemetrylogs.LogsV2TableName,
 		telemetrylogs.TagAttributesV2TableName,

pkg/telemetrymetrics/tables.go

@@ -1,21 +1,23 @@
 package telemetrymetrics
 
 const (
-	DBName                          = "signoz_metrics"
-	SamplesV4TableName              = "distributed_samples_v4"
-	SamplesV4LocalTableName         = "samples_v4"
-	SamplesV4Agg5mTableName         = "distributed_samples_v4_agg_5m"
-	SamplesV4Agg5mLocalTableName    = "samples_v4_agg_5m"
-	SamplesV4Agg30mTableName        = "distributed_samples_v4_agg_30m"
-	SamplesV4Agg30mLocalTableName   = "samples_v4_agg_30m"
-	ExpHistogramTableName           = "distributed_exp_hist"
-	ExpHistogramLocalTableName      = "exp_hist"
-	TimeseriesV4TableName           = "distributed_time_series_v4"
-	TimeseriesV4LocalTableName      = "time_series_v4"
-	TimeseriesV46hrsTableName       = "distributed_time_series_v4_6hrs"
-	TimeseriesV46hrsLocalTableName  = "time_series_v4_6hrs"
-	TimeseriesV41dayTableName       = "distributed_time_series_v4_1day"
-	TimeseriesV41dayLocalTableName  = "time_series_v4_1day"
-	TimeseriesV41weekTableName      = "distributed_time_series_v4_1week"
-	TimeseriesV41weekLocalTableName = "time_series_v4_1week"
+	DBName                           = "signoz_metrics"
+	SamplesV4TableName               = "distributed_samples_v4"
+	SamplesV4LocalTableName          = "samples_v4"
+	SamplesV4Agg5mTableName          = "distributed_samples_v4_agg_5m"
+	SamplesV4Agg5mLocalTableName     = "samples_v4_agg_5m"
+	SamplesV4Agg30mTableName         = "distributed_samples_v4_agg_30m"
+	SamplesV4Agg30mLocalTableName    = "samples_v4_agg_30m"
+	ExpHistogramTableName            = "distributed_exp_hist"
+	ExpHistogramLocalTableName       = "exp_hist"
+	TimeseriesV4TableName            = "distributed_time_series_v4"
+	TimeseriesV4LocalTableName       = "time_series_v4"
+	TimeseriesV46hrsTableName        = "distributed_time_series_v4_6hrs"
+	TimeseriesV46hrsLocalTableName   = "time_series_v4_6hrs"
+	TimeseriesV41dayTableName        = "distributed_time_series_v4_1day"
+	TimeseriesV41dayLocalTableName   = "time_series_v4_1day"
+	TimeseriesV41weekTableName       = "distributed_time_series_v4_1week"
+	TimeseriesV41weekLocalTableName  = "time_series_v4_1week"
+	AttributesMetadataTableName      = "distributed_metadata"
+	AttributesMetadataLocalTableName = "metadata"
 )