From 2230ca174086a4af3fa34757242d5964e233822f Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Wed, 9 Oct 2024 20:03:26 +0530 Subject: [PATCH] fix: enrich attributes regardless if it is materialized (#6000) * fix: enrich attributes regardless if it is materialized * feat: take care of same key name with different type and datatype * fix: makeLinks updated with new logic for logs * fix: clean up PrepareFilters and make it generic --- pkg/query-service/app/logs/v3/enrich_query.go | 30 +++---- .../app/logs/v3/enrich_query_test.go | 90 ++++++++++++++++--- pkg/query-service/contextlinks/links.go | 23 ++++- pkg/query-service/model/logs.go | 7 +- pkg/query-service/model/v3/v3.go | 8 ++ pkg/query-service/utils/logs.go | 33 +++++++ pkg/query-service/utils/logs_test.go | 57 +++++++++++- 7 files changed, 213 insertions(+), 35 deletions(-) diff --git a/pkg/query-service/app/logs/v3/enrich_query.go b/pkg/query-service/app/logs/v3/enrich_query.go index b8ed0ff801..49ff422fd5 100644 --- a/pkg/query-service/app/logs/v3/enrich_query.go +++ b/pkg/query-service/app/logs/v3/enrich_query.go @@ -7,6 +7,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" ) func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool { @@ -61,26 +62,23 @@ func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool { return false } +// if the field is timestamp/id/value we don't need to enrich +// if the field is static we don't need to enrich +// for all others we need to enrich +// an attribute/resource can be materialized/dematerialized +// but the query should work regardless and shouldn't fail func isEnriched(field v3.AttributeKey) bool { // if it is timestamp/id dont check if field.Key == "timestamp" || field.Key == "id" || field.Key == constants.SigNozOrderByValue { return true } - if field.IsColumn { + // don't need to enrich the static fields as they will be always used a column + if _, ok := constants.StaticFieldsLogsV3[field.Key]; ok && field.IsColumn { return true } - if field.Type == v3.AttributeKeyTypeUnspecified || field.DataType == v3.AttributeKeyDataTypeUnspecified { - return false - } - - // try to enrich all attributes which doesn't have isColumn = true - if !field.IsColumn { - return false - } - - return true + return false } func Enrich(params *v3.QueryRangeParamsV3, fields map[string]v3.AttributeKey) { @@ -144,13 +142,9 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu } // check if the field is present in the fields map - if existingField, ok := fields[field.Key]; ok { - // don't update if type is not the same - if (field.Type == "" && field.DataType == "") || - (field.Type == existingField.Type && field.DataType == existingField.DataType) || - (field.Type == "" && field.DataType == existingField.DataType) || - (field.DataType == "" && field.Type == existingField.Type) { - return existingField + for _, key := range utils.GenerateLogEnrichmentKeys(field) { + if val, ok := fields[key]; ok { + return val } } diff --git a/pkg/query-service/app/logs/v3/enrich_query_test.go b/pkg/query-service/app/logs/v3/enrich_query_test.go index 32d714925c..20b5d90487 100644 --- a/pkg/query-service/app/logs/v3/enrich_query_test.go +++ b/pkg/query-service/app/logs/v3/enrich_query_test.go @@ -51,7 +51,7 @@ var testEnrichmentRequiredData = []struct { EnrichmentRequired: true, }, { - Name: "filter enrichment not required", + Name: "filter enrichment required", Params: v3.QueryRangeParamsV3{ CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ @@ -87,7 +87,7 @@ var testEnrichmentRequiredData = []struct { EnrichmentRequired: true, }, { - Name: "filter enrichment not required required json", + Name: "filter enrichment required required json", Params: v3.QueryRangeParamsV3{ CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ @@ -105,7 +105,7 @@ var testEnrichmentRequiredData = []struct { EnrichmentRequired: true, }, { - Name: "groupBy enrichment not required", + Name: "groupBy enrichment required", Params: v3.QueryRangeParamsV3{ CompositeQuery: &v3.CompositeQuery{ BuilderQueries: map[string]*v3.BuilderQuery{ @@ -194,8 +194,9 @@ var testEnrichmentRequiredData = []struct { QueryName: "test", Expression: "test", DataSource: v3.DataSourceLogs, - GroupBy: []v3.AttributeKey{{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, + // here we have to fallback to trace_id attribute instead of column + GroupBy: []v3.AttributeKey{{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, }, }, }, @@ -241,17 +242,17 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "test": { + "test##tag##int64": { Key: "test", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeInt64, }, - "user_name": { + "user_name##tag##string": { Key: "user_name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, }, - "response_time": { + "response_time##tag##int64": { Key: "response_time", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeInt64, @@ -302,18 +303,18 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "method.name": { + "method.name##tag##string": { Key: "method.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "service.name": { + "service.name##tag##string": { Key: "service.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, }, - "host.name": { + "host.name##tag##string": { Key: "host.name", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, @@ -365,7 +366,7 @@ var testEnrichParamsData = []struct { }, }, Fields: map[string]v3.AttributeKey{ - "test": { + "test##tag##string": { Key: "test", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, @@ -393,6 +394,71 @@ var testEnrichParamsData = []struct { }, }, }, + { + Name: "Enrich if an attribute/resource attribute is materialized/dematerialized", + Params: v3.QueryRangeParamsV3{ + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "test": { + QueryName: "test", + Expression: "test", + DataSource: v3.DataSourceLogs, + AggregateAttribute: v3.AttributeKey{ + Key: "mat_resource", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: true, + }, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "mat_attr", Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "test", Operator: "="}, + {Key: v3.AttributeKey{Key: "normal_attr", DataType: v3.AttributeKeyDataTypeString, IsColumn: false}, Value: "test1", Operator: "="}, + }}, + }, + }, + }, + }, + Fields: map[string]v3.AttributeKey{ + "mat_resource##resource##int64": { + Key: "mat_resource", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: false, + }, + "mat_attr##tag##string": { + Key: "mat_attr", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: false, + }, + "normal_attr##tag##string": { + Key: "normal_attr", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + }, + Result: v3.QueryRangeParamsV3{ + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "test": { + QueryName: "test", + Expression: "test", + DataSource: v3.DataSourceLogs, + AggregateAttribute: v3.AttributeKey{ + Key: "mat_resource", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: false, + }, + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "mat_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: false}, Value: "test", Operator: "="}, + {Key: v3.AttributeKey{Key: "normal_attr", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "test1", Operator: "="}, + }}, + }, + }, + }, + }, + }, } func TestEnrichParams(t *testing.T) { diff --git a/pkg/query-service/contextlinks/links.go b/pkg/query-service/contextlinks/links.go index d0d8400e74..260745eda3 100644 --- a/pkg/query-service/contextlinks/links.go +++ b/pkg/query-service/contextlinks/links.go @@ -8,6 +8,7 @@ import ( tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" ) func PrepareLinksToTraces(start, end time.Time, filterItems []v3.FilterItem) string { @@ -178,8 +179,26 @@ func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, for key, value := range labels { if _, ok := added[key]; !ok { // start by taking the attribute key from the keys map, if not present, create a new one - attributeKey, ok := keys[key] - if !ok { + var attributeKey v3.AttributeKey + var attrFound bool + + // as of now this logic will only apply for logs + for _, tKey := range utils.GenerateLogEnrichmentKeys(v3.AttributeKey{Key: key}) { + if val, ok := keys[tKey]; ok { + attributeKey = val + attrFound = true + break + } + } + + // check if the attribute key is directly present, as of now this will always be false for logs + // as for logs it will be satisfied in the condition above + if !attrFound { + attributeKey, attrFound = keys[key] + } + + // if the attribute key is not present, create a new one + if !attrFound { attributeKey = v3.AttributeKey{Key: key} } diff --git a/pkg/query-service/model/logs.go b/pkg/query-service/model/logs.go index 0319581516..c6a9276008 100644 --- a/pkg/query-service/model/logs.go +++ b/pkg/query-service/model/logs.go @@ -49,7 +49,8 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3 if pass { continue } - data[selectedField.Name] = v3.AttributeKey{ + name := selectedField.Name + "##" + fieldType.String() + "##" + strings.ToLower(selectedField.DataType) + data[name] = v3.AttributeKey{ Key: selectedField.Name, Type: fieldType, DataType: v3.AttributeKeyDataType(strings.ToLower(selectedField.DataType)), @@ -61,12 +62,14 @@ func GetLogFieldsV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3 if pass { continue } - data[interestingField.Name] = v3.AttributeKey{ + name := interestingField.Name + "##" + fieldType.String() + "##" + strings.ToLower(interestingField.DataType) + data[name] = v3.AttributeKey{ Key: interestingField.Name, Type: fieldType, DataType: v3.AttributeKeyDataType(strings.ToLower(interestingField.DataType)), IsColumn: false, } + } break } diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 9770df4108..a58575cd08 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -287,6 +287,10 @@ func (q AttributeKeyDataType) Validate() error { } } +func (q AttributeKeyDataType) String() string { + return string(q) +} + // FilterAttributeValueRequest is a request to fetch possible attribute values // for a selected aggregate operator, aggregate attribute, filter attribute key // and search text. @@ -317,6 +321,10 @@ const ( AttributeKeyTypeResource AttributeKeyType = "resource" ) +func (t AttributeKeyType) String() string { + return string(t) +} + type AttributeKey struct { Key string `json:"key"` DataType AttributeKeyDataType `json:"dataType"` diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go index 2f536ef857..8efa026b52 100644 --- a/pkg/query-service/utils/logs.go +++ b/pkg/query-service/utils/logs.go @@ -1,5 +1,7 @@ package utils +import v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + const HOUR_NANO = int64(3600000000000) type LogsListTsRange struct { @@ -36,3 +38,34 @@ func GetLogsListTsRanges(start, end int64) []LogsListTsRange { } return result } + +// This tries to see all possible fields that it can fall back to if some meta is missing +// check Test_GenerateLogEnrichmentKeys for example +func GenerateLogEnrichmentKeys(field v3.AttributeKey) []string { + names := []string{} + if field.Type != v3.AttributeKeyTypeUnspecified && field.DataType != v3.AttributeKeyDataTypeUnspecified { + names = append(names, field.Key+"##"+field.Type.String()+"##"+field.DataType.String()) + return names + } + + types := []v3.AttributeKeyType{} + dTypes := []v3.AttributeKeyDataType{} + if field.Type != v3.AttributeKeyTypeUnspecified { + types = append(types, field.Type) + } else { + types = append(types, v3.AttributeKeyTypeTag, v3.AttributeKeyTypeResource) + } + if field.DataType != v3.AttributeKeyDataTypeUnspecified { + dTypes = append(dTypes, field.DataType) + } else { + dTypes = append(dTypes, v3.AttributeKeyDataTypeFloat64, v3.AttributeKeyDataTypeInt64, v3.AttributeKeyDataTypeString, v3.AttributeKeyDataTypeBool) + } + + for _, t := range types { + for _, d := range dTypes { + names = append(names, field.Key+"##"+t.String()+"##"+d.String()) + } + } + + return names +} diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go index 939fa5fa1b..e1efd813d1 100644 --- a/pkg/query-service/utils/logs_test.go +++ b/pkg/query-service/utils/logs_test.go @@ -1,6 +1,11 @@ package utils -import "testing" +import ( + "reflect" + "testing" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) func TestLogsListTsRange(t *testing.T) { startEndData := []struct { @@ -47,3 +52,53 @@ func TestLogsListTsRange(t *testing.T) { } } } + +func Test_GenerateLogEnrichmentKeys(t *testing.T) { + type args struct { + field v3.AttributeKey + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "all are present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + want: []string{"data##tag##string"}, + }, + { + name: "type present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + Type: v3.AttributeKeyTypeTag, + }, + }, + want: []string{"data##tag##float64", "data##tag##int64", "data##tag##string", "data##tag##bool"}, + }, + { + name: "dataType present", + args: args{ + field: v3.AttributeKey{ + Key: "data", + DataType: v3.AttributeKeyDataTypeString, + }, + }, + want: []string{"data##tag##string", "data##resource##string"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GenerateLogEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) { + t.Errorf("generateLogEnrichmentKeys() = %v, want %v", got, tt.want) + } + }) + } +}