fix: enrich attributes regardless if it is materialized (#6000)

* fix: enrich attributes regardless if it is materialized

* feat: take care of same key name with different type and datatype

* fix: makeLinks updated with new logic for logs

* fix: clean up PrepareFilters and make it generic

pkg/query-service/app/logs/v3/enrich_query.go

@@ -7,6 +7,7 @@ import (
 
 	"go.signoz.io/signoz/pkg/query-service/constants"
 	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+	"go.signoz.io/signoz/pkg/query-service/utils"
 )
 
 func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool {
@@ -61,26 +62,23 @@ func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool {
 	return false
 }
 
+// if the field is timestamp/id/value we don't need to enrich
+// if the field is static we don't need to enrich
+// for all others we need to enrich
+// an attribute/resource can be materialized/dematerialized
+// but the query should work regardless and shouldn't fail
 func isEnriched(field v3.AttributeKey) bool {
 	// if it is timestamp/id dont check
 	if field.Key == "timestamp" || field.Key == "id" || field.Key == constants.SigNozOrderByValue {
 		return true
 	}
 
-	if field.IsColumn {
+	// don't need to enrich the static fields as they will be always used a column
+	if _, ok := constants.StaticFieldsLogsV3[field.Key]; ok && field.IsColumn {
 		return true
 	}
 
-	if field.Type == v3.AttributeKeyTypeUnspecified || field.DataType == v3.AttributeKeyDataTypeUnspecified {
+	return false
-		return false
-	}
-
-	// try to enrich all attributes which doesn't have isColumn = true
-	if !field.IsColumn {
-		return false
-	}
-
-	return true
 }
 
 func Enrich(params *v3.QueryRangeParamsV3, fields map[string]v3.AttributeKey) {
@@ -144,13 +142,9 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu
 	}
 
 	// check if the field is present in the fields map
-	if existingField, ok := fields[field.Key]; ok {
+	for _, key := range utils.GenerateLogEnrichmentKeys(field) {
-		// don't update if type is not the same
+		if val, ok := fields[key]; ok {
-		if (field.Type == "" && field.DataType == "") ||
+			return val
-			(field.Type == existingField.Type && field.DataType == existingField.DataType) ||
-			(field.Type == "" && field.DataType == existingField.DataType) ||
-			(field.DataType == "" && field.Type == existingField.Type) {
-			return existingField
 		}
 	}
 

pkg/query-service/app/logs/v3/enrich_query_test.go

@@ -51,7 +51,7 @@ var testEnrichmentRequiredData = []struct {
 		EnrichmentRequired: true,
 	},
 	{
-		Name: "filter enrichment not required",
+		Name: "filter enrichment required",
 		Params: v3.QueryRangeParamsV3{
 			CompositeQuery: &v3.CompositeQuery{
 				BuilderQueries: map[string]*v3.BuilderQuery{
@@ -87,7 +87,7 @@ var testEnrichmentRequiredData = []struct {
 		EnrichmentRequired: true,
 	},
 	{
-		Name: "filter enrichment not required required json",
+		Name: "filter enrichment required required json",
 		Params: v3.QueryRangeParamsV3{
 			CompositeQuery: &v3.CompositeQuery{
 				BuilderQueries: map[string]*v3.BuilderQuery{
@@ -105,7 +105,7 @@ var testEnrichmentRequiredData = []struct {
 		EnrichmentRequired: true,
 	},
 	{
-		Name: "groupBy enrichment not required",
+		Name: "groupBy enrichment required",
 		Params: v3.QueryRangeParamsV3{
 			CompositeQuery: &v3.CompositeQuery{
 				BuilderQueries: map[string]*v3.BuilderQuery{
@@ -194,8 +194,9 @@ var testEnrichmentRequiredData = []struct {
 						QueryName:  "test",
 						Expression: "test",
 						DataSource: v3.DataSourceLogs,
-						GroupBy:    []v3.AttributeKey{{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
+						// here we have to fallback to trace_id attribute instead of column
-						OrderBy:    []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}},
+						GroupBy: []v3.AttributeKey{{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
+						OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}},
 					},
 				},
 			},
@@ -241,17 +242,17 @@ var testEnrichParamsData = []struct {
 			},
 		},
 		Fields: map[string]v3.AttributeKey{
-			"test": {
+			"test##tag##int64": {
 				Key:      "test",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeInt64,
 			},
-			"user_name": {
+			"user_name##tag##string": {
 				Key:      "user_name",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeString,
 			},
-			"response_time": {
+			"response_time##tag##int64": {
 				Key:      "response_time",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeInt64,
@@ -302,18 +303,18 @@ var testEnrichParamsData = []struct {
 			},
 		},
 		Fields: map[string]v3.AttributeKey{
-			"method.name": {
+			"method.name##tag##string": {
 				Key:      "method.name",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeString,
 				IsColumn: true,
 			},
-			"service.name": {
+			"service.name##tag##string": {
 				Key:      "service.name",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeString,
 			},
-			"host.name": {
+			"host.name##tag##string": {
 				Key:      "host.name",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeString,
@@ -365,7 +366,7 @@ var testEnrichParamsData = []struct {
 			},
 		},
 		Fields: map[string]v3.AttributeKey{
-			"test": {
+			"test##tag##string": {
 				Key:      "test",
 				Type:     v3.AttributeKeyTypeTag,
 				DataType: v3.AttributeKeyDataTypeString,
@@ -393,6 +394,71 @@ var testEnrichParamsData = []struct {
 			},
 		},
 	},
+	{
+		Name: "Enrich if an attribute/resource attribute is materialized/dematerialized",
+		Params: v3.QueryRangeParamsV3{
+			CompositeQuery: &v3.CompositeQuery{
+				BuilderQueries: map[string]*v3.BuilderQuery{
+					"test": {
+						QueryName:  "test",
+						Expression: "test",
+						DataSource: v3.DataSourceLogs,
+						AggregateAttribute: v3.AttributeKey{
+							Key:      "mat_resource",
+							Type:     v3.AttributeKeyTypeResource,
+							DataType: v3.AttributeKeyDataTypeInt64,
+							IsColumn: true,
+						},
+						Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
+							{Key: v3.AttributeKey{Key: "mat_attr", Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "test", Operator: "="},
+							{Key: v3.AttributeKey{Key: "normal_attr", DataType: v3.AttributeKeyDataTypeString, IsColumn: false}, Value: "test1", Operator: "="},
+						}},
+					},
+				},
+			},
+		},
+		Fields: map[string]v3.AttributeKey{
+			"mat_resource##resource##int64": {
+				Key:      "mat_resource",
+				Type:     v3.AttributeKeyTypeResource,
+				DataType: v3.AttributeKeyDataTypeInt64,
+				IsColumn: false,
+			},
+			"mat_attr##tag##string": {
+				Key:      "mat_attr",
+				Type:     v3.AttributeKeyTypeTag,
+				DataType: v3.AttributeKeyDataTypeString,
+				IsColumn: false,
+			},
+			"normal_attr##tag##string": {
+				Key:      "normal_attr",
+				Type:     v3.AttributeKeyTypeTag,
+				DataType: v3.AttributeKeyDataTypeString,
+				IsColumn: true,
+			},
+		},
+		Result: v3.QueryRangeParamsV3{
+			CompositeQuery: &v3.CompositeQuery{
+				BuilderQueries: map[string]*v3.BuilderQuery{
+					"test": {
+						QueryName:  "test",
+						Expression: "test",
+						DataSource: v3.DataSourceLogs,
+						AggregateAttribute: v3.AttributeKey{
+							Key:      "mat_resource",
+							Type:     v3.AttributeKeyTypeResource,
+							DataType: v3.AttributeKeyDataTypeInt64,
+							IsColumn: false,
+						},
+						Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
+							{Key: v3.AttributeKey{Key: "mat_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: false}, Value: "test", Operator: "="},
+							{Key: v3.AttributeKey{Key: "normal_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "test1", Operator: "="},
+						}},
+					},
+				},
+			},
+		},
+	},
 }
 
 func TestEnrichParams(t *testing.T) {

pkg/query-service/contextlinks/links.go

@@ -8,6 +8,7 @@ import (
 
 	tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3"
 	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+	"go.signoz.io/signoz/pkg/query-service/utils"
 )
 
 func PrepareLinksToTraces(start, end time.Time, filterItems []v3.FilterItem) string {
@@ -178,8 +179,26 @@ func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem,
 	for key, value := range labels {
 		if _, ok := added[key]; !ok {
 			// start by taking the attribute key from the keys map, if not present, create a new one
-			attributeKey, ok := keys[key]
+			var attributeKey v3.AttributeKey
-			if !ok {
+			var attrFound bool
+
+			// as of now this logic will only apply for logs
+			for _, tKey := range utils.GenerateLogEnrichmentKeys(v3.AttributeKey{Key: key}) {
+				if val, ok := keys[tKey]; ok {
+					attributeKey = val
+					attrFound = true
+					break
+				}
+			}
+
+			// check if the attribute key is directly present, as of now this will always be false for logs
+			// as for logs it will be satisfied in the condition above
+			if !attrFound {
+				attributeKey, attrFound = keys[key]
+			}
+
+			// if the attribute key is not present, create a new one
+			if !attrFound {
 				attributeKey = v3.AttributeKey{Key: key}
 			}
 

pkg/query-service/model/logs.go

@@ -49,7 +49,8 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3
 				if pass {
 					continue
 				}
-				data[selectedField.Name] = v3.AttributeKey{
+				name := selectedField.Name + "##" + fieldType.String() + "##" + strings.ToLower(selectedField.DataType)
+				data[name] = v3.AttributeKey{
 					Key:      selectedField.Name,
 					Type:     fieldType,
 					DataType: v3.AttributeKeyDataType(strings.ToLower(selectedField.DataType)),
@@ -61,12 +62,14 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3
 				if pass {
 					continue
 				}
-				data[interestingField.Name] = v3.AttributeKey{
+				name := interestingField.Name + "##" + fieldType.String() + "##" + strings.ToLower(interestingField.DataType)
+				data[name] = v3.AttributeKey{
 					Key:      interestingField.Name,
 					Type:     fieldType,
 					DataType: v3.AttributeKeyDataType(strings.ToLower(interestingField.DataType)),
 					IsColumn: false,
 				}
+
 			}
 			break
 		}

pkg/query-service/model/v3/v3.go

@@ -287,6 +287,10 @@ func (q AttributeKeyDataType) Validate() error {
 	}
 }
 
+func (q AttributeKeyDataType) String() string {
+	return string(q)
+}
+
 // FilterAttributeValueRequest is a request to fetch possible attribute values
 // for a selected aggregate operator, aggregate attribute, filter attribute key
 // and search text.
@@ -317,6 +321,10 @@ const (
 	AttributeKeyTypeResource    AttributeKeyType = "resource"
 )
 
+func (t AttributeKeyType) String() string {
+	return string(t)
+}
+
 type AttributeKey struct {
 	Key      string               `json:"key"`
 	DataType AttributeKeyDataType `json:"dataType"`

pkg/query-service/utils/logs.go

@@ -1,5 +1,7 @@
 package utils
 
+import v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+
 const HOUR_NANO = int64(3600000000000)
 
 type LogsListTsRange struct {
@@ -36,3 +38,34 @@ func GetLogsListTsRanges(start, end int64) []LogsListTsRange {
 	}
 	return result
 }
+
+// This tries to see all possible fields that it can fall back to if some meta is missing
+// check Test_GenerateLogEnrichmentKeys for example
+func GenerateLogEnrichmentKeys(field v3.AttributeKey) []string {
+	names := []string{}
+	if field.Type != v3.AttributeKeyTypeUnspecified && field.DataType != v3.AttributeKeyDataTypeUnspecified {
+		names = append(names, field.Key+"##"+field.Type.String()+"##"+field.DataType.String())
+		return names
+	}
+
+	types := []v3.AttributeKeyType{}
+	dTypes := []v3.AttributeKeyDataType{}
+	if field.Type != v3.AttributeKeyTypeUnspecified {
+		types = append(types, field.Type)
+	} else {
+		types = append(types, v3.AttributeKeyTypeTag, v3.AttributeKeyTypeResource)
+	}
+	if field.DataType != v3.AttributeKeyDataTypeUnspecified {
+		dTypes = append(dTypes, field.DataType)
+	} else {
+		dTypes = append(dTypes, v3.AttributeKeyDataTypeFloat64, v3.AttributeKeyDataTypeInt64, v3.AttributeKeyDataTypeString, v3.AttributeKeyDataTypeBool)
+	}
+
+	for _, t := range types {
+		for _, d := range dTypes {
+			names = append(names, field.Key+"##"+t.String()+"##"+d.String())
+		}
+	}
+
+	return names
+}

pkg/query-service/utils/logs_test.go

@@ -1,6 +1,11 @@
 package utils
 
-import "testing"
+import (
+	"reflect"
+	"testing"
+
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+)
 
 func TestLogsListTsRange(t *testing.T) {
 	startEndData := []struct {
@@ -47,3 +52,53 @@ func TestLogsListTsRange(t *testing.T) {
 		}
 	}
 }
+
+func Test_GenerateLogEnrichmentKeys(t *testing.T) {
+	type args struct {
+		field v3.AttributeKey
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "all are present",
+			args: args{
+				field: v3.AttributeKey{
+					Key:      "data",
+					DataType: v3.AttributeKeyDataTypeString,
+					Type:     v3.AttributeKeyTypeTag,
+				},
+			},
+			want: []string{"data##tag##string"},
+		},
+		{
+			name: "type present",
+			args: args{
+				field: v3.AttributeKey{
+					Key:  "data",
+					Type: v3.AttributeKeyTypeTag,
+				},
+			},
+			want: []string{"data##tag##float64", "data##tag##int64", "data##tag##string", "data##tag##bool"},
+		},
+		{
+			name: "dataType present",
+			args: args{
+				field: v3.AttributeKey{
+					Key:      "data",
+					DataType: v3.AttributeKeyDataTypeString,
+				},
+			},
+			want: []string{"data##tag##string", "data##resource##string"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := GenerateLogEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("generateLogEnrichmentKeys() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}