From 9fbf111976a1d0b7a3e23a2872e808d2c9af4990 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Thu, 15 May 2025 19:59:40 +0530 Subject: [PATCH] chore: less strict context for fetching field values (#7807) --- pkg/telemetrymetadata/condition_builder.go | 39 +++++------ .../condition_builder_test.go | 4 +- pkg/telemetrymetadata/metadata.go | 67 ++++++++++++++++--- pkg/types/telemetrytypes/field_context.go | 1 + pkg/types/telemetrytypes/field_datatype.go | 1 + 5 files changed, 83 insertions(+), 29 deletions(-) diff --git a/pkg/telemetrymetadata/condition_builder.go b/pkg/telemetrymetadata/condition_builder.go index 6ec5492917..33fb22a1b2 100644 --- a/pkg/telemetrymetadata/condition_builder.go +++ b/pkg/telemetrymetadata/condition_builder.go @@ -75,7 +75,8 @@ func (c *conditionBuilder) GetCondition( return "", nil } - if key.FieldDataType != telemetrytypes.FieldDataTypeString { + if key.FieldDataType != telemetrytypes.FieldDataTypeString && + key.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { // if the field data type is not string, we can't build a condition for related values return "", nil } @@ -83,37 +84,37 @@ func (c *conditionBuilder) GetCondition( tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName) // key must exists to apply main filter - containsExp := fmt.Sprintf("mapContains(%s, %s)", column.Name, sb.Var(key.Name)) + expr := `if(mapContains(%s, %s), %s, true)` + + var cond string // regular operators switch operator { // regular operators case qbtypes.FilterOperatorEqual: - return sb.And(containsExp, sb.E(tblFieldName, value)), nil + cond = sb.E(tblFieldName, value) case qbtypes.FilterOperatorNotEqual: - return sb.And(containsExp, sb.NE(tblFieldName, value)), nil + cond = sb.NE(tblFieldName, value) // like and not like case qbtypes.FilterOperatorLike: - return sb.And(containsExp, sb.Like(tblFieldName, value)), nil + cond = sb.Like(tblFieldName, value) case qbtypes.FilterOperatorNotLike: - return sb.And(containsExp, sb.NotLike(tblFieldName, value)), nil + cond = sb.NotLike(tblFieldName, value) case qbtypes.FilterOperatorILike: - return sb.And(containsExp, sb.ILike(tblFieldName, value)), nil + cond = sb.ILike(tblFieldName, value) case qbtypes.FilterOperatorNotILike: - return sb.And(containsExp, sb.NotILike(tblFieldName, value)), nil + cond = sb.NotILike(tblFieldName, value) case qbtypes.FilterOperatorContains: - return sb.And(containsExp, sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value))), nil + cond = sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)) case qbtypes.FilterOperatorNotContains: - return sb.And(containsExp, sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value))), nil + cond = sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)) case qbtypes.FilterOperatorRegexp: - exp := fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value)) - return sb.And(containsExp, exp), nil + cond = fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value)) case qbtypes.FilterOperatorNotRegexp: - exp := fmt.Sprintf(`not match(%s, %s)`, tblFieldName, sb.Var(value)) - return sb.And(containsExp, exp), nil + cond = fmt.Sprintf(`not match(%s, %s)`, tblFieldName, sb.Var(value)) // in and not in case qbtypes.FilterOperatorIn: @@ -121,13 +122,13 @@ func (c *conditionBuilder) GetCondition( if !ok { return "", qbtypes.ErrInValues } - return sb.And(containsExp, sb.In(tblFieldName, values...)), nil + cond = sb.In(tblFieldName, values...) case qbtypes.FilterOperatorNotIn: values, ok := value.([]any) if !ok { return "", qbtypes.ErrInValues } - return sb.And(containsExp, sb.NotIn(tblFieldName, values...)), nil + cond = sb.NotIn(tblFieldName, values...) // exists and not exists // in the query builder, `exists` and `not exists` are used for @@ -140,12 +141,12 @@ func (c *conditionBuilder) GetCondition( }: leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) if operator == qbtypes.FilterOperatorExists { - return sb.E(leftOperand, true), nil + cond = sb.E(leftOperand, true) } else { - return sb.NE(leftOperand, true), nil + cond = sb.NE(leftOperand, true) } } } - return "", nil + return fmt.Sprintf(expr, column.Name, sb.Var(key.Name), cond), nil } diff --git a/pkg/telemetrymetadata/condition_builder_test.go b/pkg/telemetrymetadata/condition_builder_test.go index abd971146a..1a292ab98c 100644 --- a/pkg/telemetrymetadata/condition_builder_test.go +++ b/pkg/telemetrymetadata/condition_builder_test.go @@ -237,7 +237,7 @@ func TestGetCondition(t *testing.T) { }, operator: qbtypes.FilterOperatorILike, value: "%admin%", - expectedSQL: "WHERE (mapContains(attributes, ?) AND LOWER(attributes['user.id']) LIKE LOWER(?))", + expectedSQL: "WHERE if(mapContains(attributes, ?), LOWER(attributes['user.id']) LIKE LOWER(?), true)", expectedError: nil, }, { @@ -249,7 +249,7 @@ func TestGetCondition(t *testing.T) { }, operator: qbtypes.FilterOperatorNotILike, value: "%admin%", - expectedSQL: "WHERE (mapContains(attributes, ?) AND LOWER(attributes['user.id']) NOT LIKE LOWER(?))", + expectedSQL: "WHERE if(mapContains(attributes, ?), LOWER(attributes['user.id']) NOT LIKE LOWER(?), true)", expectedError: nil, }, } diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index 49190b9193..89334c2747 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -635,6 +635,12 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS } else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeNumber { sb.Where(sb.IsNotNull("number_value")) sb.Where(sb.Like("toString(number_value)", "%"+fieldValueSelector.Value+"%")) + } else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeUnspecified { + // or b/w string and number + sb.Where(sb.Or( + sb.Like("string_value", "%"+fieldValueSelector.Value+"%"), + sb.Like("toString(number_value)", "%"+fieldValueSelector.Value+"%"), + )) } } @@ -696,6 +702,12 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe } else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeNumber { sb.Where(sb.IsNotNull("number_value")) sb.Where(sb.Like("toString(number_value)", "%"+fieldValueSelector.Value+"%")) + } else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeUnspecified { + // or b/w string and number + sb.Where(sb.Or( + sb.Like("string_value", "%"+fieldValueSelector.Value+"%"), + sb.Like("toString(number_value)", "%"+fieldValueSelector.Value+"%"), + )) } } @@ -742,30 +754,30 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValu } if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified { - sb.And(sb.E("attr_type", fieldValueSelector.FieldContext.TagType())) + sb.Where(sb.E("attr_type", fieldValueSelector.FieldContext.TagType())) } if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { - sb.And(sb.E("attr_datatype", fieldValueSelector.FieldDataType.TagDataType())) + sb.Where(sb.E("attr_datatype", fieldValueSelector.FieldDataType.TagDataType())) } if fieldValueSelector.MetricContext != nil { - sb.And(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName)) + sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName)) } if fieldValueSelector.StartUnixMilli > 0 { - sb.And(sb.GE("last_reported_unix_milli", fieldValueSelector.StartUnixMilli)) + sb.Where(sb.GE("last_reported_unix_milli", fieldValueSelector.StartUnixMilli)) } if fieldValueSelector.EndUnixMilli > 0 { - sb.And(sb.LE("first_reported_unix_milli", fieldValueSelector.EndUnixMilli)) + sb.Where(sb.LE("first_reported_unix_milli", fieldValueSelector.EndUnixMilli)) } if fieldValueSelector.Value != "" { if fieldValueSelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact { - sb.And(sb.E("attr_string_value", fieldValueSelector.Value)) + sb.Where(sb.E("attr_string_value", fieldValueSelector.Value)) } else { - sb.And(sb.Like("attr_string_value", "%"+fieldValueSelector.Value+"%")) + sb.Where(sb.Like("attr_string_value", "%"+fieldValueSelector.Value+"%")) } } @@ -794,8 +806,30 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValu return values, nil } +func populateAllUnspecifiedValues(allUnspecifiedValues *telemetrytypes.TelemetryFieldValues, mapOfValues map[any]bool, mapOfRelatedValues map[any]bool, values *telemetrytypes.TelemetryFieldValues) { + for _, value := range values.StringValues { + if _, ok := mapOfValues[value]; !ok { + mapOfValues[value] = true + allUnspecifiedValues.StringValues = append(allUnspecifiedValues.StringValues, value) + } + } + for _, value := range values.NumberValues { + if _, ok := mapOfValues[value]; !ok { + mapOfValues[value] = true + allUnspecifiedValues.NumberValues = append(allUnspecifiedValues.NumberValues, value) + } + } + + for _, value := range values.RelatedValues { + if _, ok := mapOfRelatedValues[value]; !ok { + mapOfRelatedValues[value] = true + allUnspecifiedValues.RelatedValues = append(allUnspecifiedValues.RelatedValues, value) + } + } +} + func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) { - var values *telemetrytypes.TelemetryFieldValues + values := &telemetrytypes.TelemetryFieldValues{} var err error switch fieldValueSelector.Signal { case telemetrytypes.SignalTraces: @@ -804,6 +838,23 @@ func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelecto values, err = t.getLogFieldValues(ctx, fieldValueSelector) case telemetrytypes.SignalMetrics: values, err = t.getMetricFieldValues(ctx, fieldValueSelector) + case telemetrytypes.SignalUnspecified: + mapOfValues := make(map[any]bool) + mapOfRelatedValues := make(map[any]bool) + allUnspecifiedValues := &telemetrytypes.TelemetryFieldValues{} + tracesValues, err := t.getSpanFieldValues(ctx, fieldValueSelector) + if err == nil { + populateAllUnspecifiedValues(allUnspecifiedValues, mapOfValues, mapOfRelatedValues, tracesValues) + } + logsValues, err := t.getLogFieldValues(ctx, fieldValueSelector) + if err == nil { + populateAllUnspecifiedValues(allUnspecifiedValues, mapOfValues, mapOfRelatedValues, logsValues) + } + metricsValues, err := t.getMetricFieldValues(ctx, fieldValueSelector) + if err == nil { + populateAllUnspecifiedValues(allUnspecifiedValues, mapOfValues, mapOfRelatedValues, metricsValues) + } + values = allUnspecifiedValues } if err != nil { return nil, err diff --git a/pkg/types/telemetrytypes/field_context.go b/pkg/types/telemetrytypes/field_context.go index 2d25f788cf..b3e05c1625 100644 --- a/pkg/types/telemetrytypes/field_context.go +++ b/pkg/types/telemetrytypes/field_context.go @@ -62,6 +62,7 @@ var ( "resource": FieldContextResource, "scope": FieldContextScope, "tag": FieldContextAttribute, + "point": FieldContextAttribute, "attribute": FieldContextAttribute, "event": FieldContextEvent, "spanfield": FieldContextSpan, diff --git a/pkg/types/telemetrytypes/field_datatype.go b/pkg/types/telemetrytypes/field_datatype.go index 07d91b9afb..f6232d32ce 100644 --- a/pkg/types/telemetrytypes/field_datatype.go +++ b/pkg/types/telemetrytypes/field_datatype.go @@ -38,6 +38,7 @@ var ( fieldDataTypes = map[string]FieldDataType{ // String types "string": FieldDataTypeString, + "str": FieldDataTypeString, // Boolean types "bool": FieldDataTypeBool,