diff --git a/pkg/apis/fields/api.go b/pkg/apis/fields/api.go index 0468302137..e32f75ec42 100644 --- a/pkg/apis/fields/api.go +++ b/pkg/apis/fields/api.go @@ -28,8 +28,7 @@ func NewAPI(telemetryStore telemetrystore.TelemetryStore) *API { telemetrytraces.TagAttributesV2TableName, telemetrytraces.SpanIndexV3TableName, telemetrymetrics.DBName, - telemetrymetrics.TimeseriesV41weekTableName, - telemetrymetrics.TimeseriesV41weekLocalTableName, + telemetrymetrics.AttributesMetadataTableName, telemetrylogs.DBName, telemetrylogs.LogsV2TableName, telemetrylogs.TagAttributesV2TableName, diff --git a/pkg/parser/grammar/where_clause_visitor.go b/pkg/parser/grammar/where_clause_visitor.go index 3145f946c6..3f1523cf24 100644 --- a/pkg/parser/grammar/where_clause_visitor.go +++ b/pkg/parser/grammar/where_clause_visitor.go @@ -20,18 +20,18 @@ import ( type WhereClauseVisitor struct { conditionBuilder qbtypes.ConditionBuilder warnings []error - fieldKeys map[string][]telemetrytypes.TelemetryFieldKey + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey errors []error builder *sqlbuilder.SelectBuilder - fullTextColumn telemetrytypes.TelemetryFieldKey + fullTextColumn *telemetrytypes.TelemetryFieldKey } // NewWhereClauseVisitor creates a new WhereClauseVisitor func NewWhereClauseVisitor( conditionBuilder qbtypes.ConditionBuilder, - fieldKeys map[string][]telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, builder *sqlbuilder.SelectBuilder, - fullTextColumn telemetrytypes.TelemetryFieldKey, + fullTextColumn *telemetrytypes.TelemetryFieldKey, ) *WhereClauseVisitor { return &WhereClauseVisitor{ conditionBuilder: conditionBuilder, @@ -72,10 +72,10 @@ func (l *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol // PrepareWhereClause generates a ClickHouse compatible WHERE clause from the filter query func PrepareWhereClause( query string, - fieldKeys map[string][]telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, conditionBuilder qbtypes.ConditionBuilder, - fullTextColumn telemetrytypes.TelemetryFieldKey, -) (string, []any, []error, error) { + fullTextColumn *telemetrytypes.TelemetryFieldKey, +) (*sqlbuilder.WhereClause, []error, error) { // Setup the ANTLR parsing pipeline input := antlr.NewInputStream(query) lexer := NewFilterQueryLexer(input) @@ -107,7 +107,7 @@ func PrepareWhereClause( len(parserErrorListener.Errors), parserErrorListener.Errors, ) - return "", nil, nil, combinedErrors + return nil, nil, combinedErrors } // Visit the parse tree with our ClickHouse visitor @@ -122,12 +122,12 @@ func PrepareWhereClause( len(visitor.errors), visitor.errors, ) - return "", nil, nil, combinedErrors + return nil, nil, combinedErrors } - whereClause, args := visitor.builder.Where(cond).BuildWithFlavor(sqlbuilder.ClickHouse) + whereClause := sqlbuilder.NewWhereClause().AddWhereExpr(visitor.builder.Args, cond) - return whereClause, args, visitor.warnings, nil + return whereClause, visitor.warnings, nil } // Visit dispatches to the specific visit method based on node type @@ -250,7 +250,7 @@ func (v *WhereClauseVisitor) VisitPrimary(ctx *PrimaryContext) any { if keyCtx, ok := child.(*KeyContext); ok { // create a full text search condition on the body field keyText := keyCtx.GetText() - cond, err := v.conditionBuilder.GetCondition(context.Background(), &v.fullTextColumn, qbtypes.FilterOperatorRegexp, keyText, v.builder) + cond, err := v.conditionBuilder.GetCondition(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, keyText, v.builder) if err != nil { return "" } @@ -263,7 +263,7 @@ func (v *WhereClauseVisitor) VisitPrimary(ctx *PrimaryContext) any { // VisitComparison handles all comparison operators func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any { - keys := v.Visit(ctx.Key()).([]telemetrytypes.TelemetryFieldKey) + keys := v.Visit(ctx.Key()).([]*telemetrytypes.TelemetryFieldKey) // Handle EXISTS specially if ctx.EXISTS() != nil { @@ -273,7 +273,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any { } var conds []string for _, key := range keys { - condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, nil, v.builder) + condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, nil, v.builder) if err != nil { return "" } @@ -291,7 +291,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any { } var conds []string for _, key := range keys { - condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, values, v.builder) + condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, values, v.builder) if err != nil { return "" } @@ -317,7 +317,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any { var conds []string for _, key := range keys { - condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, []any{value1, value2}, v.builder) + condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, []any{value1, value2}, v.builder) if err != nil { return "" } @@ -366,7 +366,7 @@ func (v *WhereClauseVisitor) VisitComparison(ctx *ComparisonContext) any { var conds []string for _, key := range keys { - condition, err := v.conditionBuilder.GetCondition(context.Background(), &key, op, value, v.builder) + condition, err := v.conditionBuilder.GetCondition(context.Background(), key, op, value, v.builder) if err != nil { return "" } @@ -404,7 +404,7 @@ func (v *WhereClauseVisitor) VisitValueList(ctx *ValueListContext) any { func (v *WhereClauseVisitor) VisitFullText(ctx *FullTextContext) any { // remove quotes from the quotedText quotedText := strings.Trim(ctx.QUOTED_TEXT().GetText(), "\"'") - cond, err := v.conditionBuilder.GetCondition(context.Background(), &v.fullTextColumn, qbtypes.FilterOperatorRegexp, quotedText, v.builder) + cond, err := v.conditionBuilder.GetCondition(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, quotedText, v.builder) if err != nil { return "" } @@ -443,7 +443,7 @@ func (v *WhereClauseVisitor) VisitFunctionCall(ctx *FunctionCallContext) any { return "" } - keys, ok := params[0].([]telemetrytypes.TelemetryFieldKey) + keys, ok := params[0].([]*telemetrytypes.TelemetryFieldKey) if !ok { v.errors = append(v.errors, errors.Newf( errors.TypeInvalidInput, @@ -459,9 +459,9 @@ func (v *WhereClauseVisitor) VisitFunctionCall(ctx *FunctionCallContext) any { var fieldName string if strings.HasPrefix(key.Name, telemetrylogs.BodyJSONStringSearchPrefix) { - fieldName, _ = telemetrylogs.GetBodyJSONKey(context.Background(), &key, qbtypes.FilterOperatorUnknown, value) + fieldName, _ = telemetrylogs.GetBodyJSONKey(context.Background(), key, qbtypes.FilterOperatorUnknown, value) } else { - fieldName, _ = v.conditionBuilder.GetTableFieldName(context.Background(), &key) + fieldName, _ = v.conditionBuilder.GetTableFieldName(context.Background(), key) } var cond string @@ -557,7 +557,7 @@ func (v *WhereClauseVisitor) VisitKey(ctx *KeyContext) any { // Since it will ORed with the fieldKeysForName, it will not result empty // when either of them have values if strings.HasPrefix(fieldKey.Name, telemetrylogs.BodyJSONStringSearchPrefix) { - fieldKeysForName = append(fieldKeysForName, fieldKey) + fieldKeysForName = append(fieldKeysForName, &fieldKey) } // TODO(srikanthccv): do we want to return an error here? diff --git a/pkg/parser/grammar/where_clause_visitor_test.go b/pkg/parser/grammar/where_clause_visitor_test.go index 1faf1f2adc..60b0f0d6b6 100644 --- a/pkg/parser/grammar/where_clause_visitor_test.go +++ b/pkg/parser/grammar/where_clause_visitor_test.go @@ -8,19 +8,20 @@ import ( "github.com/SigNoz/signoz/pkg/telemetrylogs" "github.com/SigNoz/signoz/pkg/telemetrytraces" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + sqlbuilder "github.com/huandu/go-sqlbuilder" ) func TestConvertToClickHouseLogsQuery(t *testing.T) { cases := []struct { name string - fieldKeys map[string][]telemetrytypes.TelemetryFieldKey + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey query string expectedSearchString string expectedSearchArgs []any }{ { name: "test-simple-service-name-filter", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "service.name": { { Name: "service.name", @@ -36,7 +37,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "test-simple-service-name-filter-with-materialised-column", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "service.name": { { Name: "service.name", @@ -53,7 +54,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "http-status-code-multiple-data-types", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "http.status_code": { { Name: "http.status_code", @@ -75,7 +76,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "http-status-code-multiple-data-types-between-operator", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "http.status_code": { { Name: "http.status_code", @@ -97,7 +98,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "response-body-multiple-data-types-string-contains", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "response.body": { { Name: "response.body", @@ -119,7 +120,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "search-on-top-level-key", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "severity_text": { { Name: "severity_text", @@ -135,7 +136,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "search-on-top-level-key-conflict-with-attribute", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "severity_text": { { Name: "severity_text", @@ -157,7 +158,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "collision-with-attribute-field-and-resource-attribute", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "k8s.namespace.name": { { Name: "k8s.namespace.name", @@ -179,7 +180,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "collision-with-attribute-field-and-resource-attribute-materialised-column", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "k8s.namespace.name": { { Name: "k8s.namespace.name", @@ -202,7 +203,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "boolean-collision-with-attribute-field-and-data-type-boolean", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "did_user_login": { { Name: "did_user_login", @@ -224,7 +225,7 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "regexp-search", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "k8s.namespace.name": { { Name: "k8s.namespace.name", @@ -248,42 +249,42 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "full-text-search-multiple-words", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "waiting for response", expectedSearchString: "WHERE ((match(body, ?)) AND (match(body, ?)) AND (match(body, ?)))", expectedSearchArgs: []any{"waiting", "for", "response"}, }, { name: "full-text-search-with-phrase-search", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: `"waiting for response"`, expectedSearchString: "WHERE (match(body, ?))", expectedSearchArgs: []any{"waiting for response"}, }, { name: "full-text-search-with-word-and-not-word", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "error NOT buggy_app", expectedSearchString: "WHERE ((match(body, ?)) AND NOT ((match(body, ?))))", expectedSearchArgs: []any{"error", "buggy_app"}, }, { name: "full-text-search-with-word-and-not-word-and-not-word", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "error NOT buggy_app NOT redis", expectedSearchString: "WHERE ((match(body, ?)) AND NOT ((match(body, ?))) AND NOT ((match(body, ?))))", expectedSearchArgs: []any{"error", "buggy_app", "redis"}, }, { name: "full-text-search-with-word-and-not-word-and-not-word-tricky", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "error NOT buggy_app OR redis", expectedSearchString: "WHERE (((match(body, ?)) AND NOT ((match(body, ?)))) OR (match(body, ?)))", expectedSearchArgs: []any{"error", "buggy_app", "redis"}, }, { name: "has-function", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "service.name": { { Name: "service.name", @@ -305,14 +306,14 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { }, { name: "has-from-list-of-values", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "has(body.payload.user_ids[*], 'u1292')", expectedSearchString: "WHERE (has(JSONExtract(JSON_QUERY(body, '$.payload.user_ids[*]'), 'Array(String)'), ?))", expectedSearchArgs: []any{"u1292"}, }, { name: "body-json-search-that-also-has-attribute-with-same-name", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "http.status_code": { { Name: "http.status_code", @@ -331,15 +332,18 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { for _, c := range cases { t.Logf("running test %s", c.name) - chQuery, chQueryArgs, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrylogs.NewConditionBuilder(), telemetrytypes.TelemetryFieldKey{ + whereClause, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrylogs.NewConditionBuilder(), &telemetrytypes.TelemetryFieldKey{ Name: "body", Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString, }) + if err != nil { t.Errorf("Error converting query to ClickHouse: %v", err) } + chQuery, chQueryArgs := whereClause.BuildWithFlavor(sqlbuilder.ClickHouse) + if chQuery != c.expectedSearchString { t.Errorf("Expected %s, got %s", c.expectedSearchString, chQuery) } @@ -355,14 +359,14 @@ func TestConvertToClickHouseLogsQuery(t *testing.T) { func TestConvertToClickHouseSpansQuery(t *testing.T) { cases := []struct { name string - fieldKeys map[string][]telemetrytypes.TelemetryFieldKey + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey query string expectedSearchString string expectedSearchArgs []any }{ { name: "test-simple-service-name-filter", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "service.name": { { Name: "service.name", @@ -378,7 +382,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "test-simple-service-name-filter-with-materialised-column", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "service.name": { { Name: "service.name", @@ -395,7 +399,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "http-status-code-multiple-data-types", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "http.status_code": { { Name: "http.status_code", @@ -417,7 +421,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "http-status-code-multiple-data-types-between-operator", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "http.status_code": { { Name: "http.status_code", @@ -439,7 +443,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "response-body-multiple-data-types-string-contains", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "response.body": { { Name: "response.body", @@ -461,7 +465,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "collision-with-attribute-field-and-resource-attribute", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "k8s.namespace.name": { { Name: "k8s.namespace.name", @@ -483,7 +487,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "collision-with-attribute-field-and-resource-attribute-materialised-column", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "k8s.namespace.name": { { Name: "k8s.namespace.name", @@ -506,7 +510,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "boolean-collision-with-attribute-field-and-data-type-boolean", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "did_user_login": { { Name: "did_user_login", @@ -528,7 +532,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { }, { name: "regexp-search", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{ + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{ "k8s.namespace.name": { { Name: "k8s.namespace.name", @@ -553,7 +557,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { } for _, c := range cases { - chQuery, chQueryArgs, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), telemetrytypes.TelemetryFieldKey{ + whereClause, _, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), &telemetrytypes.TelemetryFieldKey{ Name: "dummy", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, @@ -562,6 +566,9 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { if err != nil { t.Errorf("Error converting query to ClickHouse: %v", err) } + + chQuery, chQueryArgs := whereClause.BuildWithFlavor(sqlbuilder.ClickHouse) + if chQuery != c.expectedSearchString { t.Errorf("Expected %s, got %s", c.expectedSearchString, chQuery) } @@ -577,7 +584,7 @@ func TestConvertToClickHouseSpansQuery(t *testing.T) { func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) { cases := []struct { name string - fieldKeys map[string][]telemetrytypes.TelemetryFieldKey + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey query string expectedSearchString string expectedSearchArgs []any @@ -586,7 +593,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) { }{ { name: "has-function-with-multiple-values", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "key.that.does.not.exist = 'redis'", expectedSearchString: "", expectedSearchArgs: []any{}, @@ -595,7 +602,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) { }, { name: "unknown-function", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "unknown.function()", expectedSearchString: "", expectedSearchArgs: []any{}, @@ -604,7 +611,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) { }, { name: "has-function-not-enough-params", - fieldKeys: map[string][]telemetrytypes.TelemetryFieldKey{}, + fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{}, query: "has(key.that.does.not.exist)", expectedSearchString: "", expectedSearchArgs: []any{}, @@ -614,7 +621,7 @@ func TestConvertToClickHouseSpansQueryWithErrors(t *testing.T) { } for _, c := range cases { - _, _, warnings, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), telemetrytypes.TelemetryFieldKey{ + _, warnings, err := PrepareWhereClause(c.query, c.fieldKeys, telemetrytraces.NewConditionBuilder(), &telemetrytypes.TelemetryFieldKey{ Name: "dummy", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, diff --git a/pkg/telemetrymetadata/condition_builder.go b/pkg/telemetrymetadata/condition_builder.go index a8f5ce8f4b..6ec5492917 100644 --- a/pkg/telemetrymetadata/condition_builder.go +++ b/pkg/telemetrymetadata/condition_builder.go @@ -80,6 +80,8 @@ func (c *conditionBuilder) GetCondition( return "", nil } + tblFieldName, value = telemetrytypes.DataTypeCollisionHandledFieldName(key, value, tblFieldName) + // key must exists to apply main filter containsExp := fmt.Sprintf("mapContains(%s, %s)", column.Name, sb.Var(key.Name)) diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index e0504636d9..49190b9193 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -3,9 +3,9 @@ package telemetrymetadata import ( "context" "fmt" - "strings" "github.com/SigNoz/signoz/pkg/errors" + parser "github.com/SigNoz/signoz/pkg/parser/grammar" "github.com/SigNoz/signoz/pkg/telemetrystore" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" @@ -28,7 +28,6 @@ type telemetryMetaStore struct { indexV3TblName string metricsDBName string metricsFieldsTblName string - timeseries1WTblName string logsDBName string logsFieldsTblName string logsV2TblName string @@ -45,7 +44,6 @@ func NewTelemetryMetaStore( indexV3TblName string, metricsDBName string, metricsFieldsTblName string, - timeseries1WTblName string, logsDBName string, logsV2TblName string, logsFieldsTblName string, @@ -53,13 +51,13 @@ func NewTelemetryMetaStore( relatedMetadataTblName string, ) telemetrytypes.MetadataStore { return &telemetryMetaStore{ + telemetrystore: telemetrystore, tracesDBName: tracesDBName, tracesFieldsTblName: tracesFieldsTblName, indexV3TblName: indexV3TblName, metricsDBName: metricsDBName, metricsFieldsTblName: metricsFieldsTblName, - timeseries1WTblName: timeseries1WTblName, logsDBName: logsDBName, logsV2TblName: logsV2TblName, logsFieldsTblName: logsFieldsTblName, @@ -79,7 +77,16 @@ func (t *telemetryMetaStore) tracesTblStatementToFieldKeys(ctx context.Context) return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTblStatement.Error()) } - return ExtractFieldKeysFromTblStatement(statements[0].Statement) + materialisedKeys, err := ExtractFieldKeysFromTblStatement(statements[0].Statement) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTracesKeys.Error()) + } + + for idx := range materialisedKeys { + materialisedKeys[idx].Signal = telemetrytypes.SignalTraces + } + + return materialisedKeys, nil } // getTracesKeys returns the keys from the spans that match the field selection criteria @@ -175,6 +182,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector if !ok { key = &telemetrytypes.TelemetryFieldKey{ Name: name, + Signal: telemetrytypes.SignalTraces, FieldContext: fieldContext, FieldDataType: fieldDataType, } @@ -199,7 +207,16 @@ func (t *telemetryMetaStore) logsTblStatementToFieldKeys(ctx context.Context) ([ return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTblStatement.Error()) } - return ExtractFieldKeysFromTblStatement(statements[0].Statement) + materialisedKeys, err := ExtractFieldKeysFromTblStatement(statements[0].Statement) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error()) + } + + for idx := range materialisedKeys { + materialisedKeys[idx].Signal = telemetrytypes.SignalLogs + } + + return materialisedKeys, nil } // getLogsKeys returns the keys from the spans that match the field selection criteria @@ -293,6 +310,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors if !ok { key = &telemetrytypes.TelemetryFieldKey{ Name: name, + Signal: telemetrytypes.SignalLogs, FieldContext: fieldContext, FieldDataType: fieldDataType, } @@ -309,52 +327,60 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors } // getMetricsKeys returns the keys from the metrics that match the field selection criteria -// TODO(srikanthccv): update the implementation after the dot metrics migration is done func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, error) { if len(fieldKeySelectors) == 0 { return nil, nil } - var whereClause, innerWhereClause string + sb := sqlbuilder. + Select("attr_name as name", "attr_type as field_context", "attr_datatype as field_data_type", ` + CASE + WHEN attr_type = 'resource' THEN 1 + WHEN attr_type = 'scope' THEN 2 + WHEN attr_type = 'point' THEN 3 + ELSE 4 + END as priority`). + From(t.metricsDBName + "." + t.metricsFieldsTblName) + var limit int - args := []any{} + conds := []string{} for _, fieldKeySelector := range fieldKeySelectors { - if fieldKeySelector.MetricContext != nil { - innerWhereClause += "metric_name IN ? AND" - args = append(args, fieldKeySelector.MetricContext.MetricName) - } - } - innerWhereClause += " __normalized = true" - - for idx, fieldKeySelector := range fieldKeySelectors { + fieldConds := []string{} if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact { - whereClause += "(distinctTagKey = ? AND distinctTagKey NOT LIKE '\\_\\_%%')" - args = append(args, fieldKeySelector.Name) + fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name)) } else { - whereClause += "(distinctTagKey ILIKE ? AND distinctTagKey NOT LIKE '\\_\\_%%')" - args = append(args, fmt.Sprintf("%%%s%%", fieldKeySelector.Name)) + fieldConds = append(fieldConds, sb.Like("attr_name", "%"+fieldKeySelector.Name+"%")) } - if idx != len(fieldKeySelectors)-1 { - whereClause += " OR " + + if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified { + fieldConds = append(fieldConds, sb.E("attr_type", fieldKeySelector.FieldContext.TagType())) } + + if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { + fieldConds = append(fieldConds, sb.E("attr_datatype", fieldKeySelector.FieldDataType.TagDataType())) + } + + if fieldKeySelector.MetricContext != nil { + fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName)) + } + + conds = append(conds, sb.And(fieldConds...)) limit += fieldKeySelector.Limit } - args = append(args, limit) + sb.Where(sb.Or(conds...)) - query := fmt.Sprintf(` - SELECT - arrayJoin(tagKeys) AS distinctTagKey - FROM ( - SELECT JSONExtractKeys(labels) AS tagKeys - FROM %s.%s - WHERE `+innerWhereClause+` - GROUP BY tagKeys - ) - WHERE `+whereClause+` - GROUP BY distinctTagKey - LIMIT ? - `, t.metricsDBName, t.timeseries1WTblName) + if limit == 0 { + limit = 1000 + } + + mainSb := sqlbuilder.Select("name", "field_context", "field_data_type", "max(priority) as priority") + mainSb.From(mainSb.BuilderAs(sb, "sub_query")) + mainSb.GroupBy("name", "field_context", "field_data_type") + mainSb.OrderBy("priority") + mainSb.Limit(limit) + + query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse) rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...) if err != nil { @@ -365,16 +391,19 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto keys := []*telemetrytypes.TelemetryFieldKey{} for rows.Next() { var name string - err = rows.Scan(&name) + var fieldContext telemetrytypes.FieldContext + var fieldDataType telemetrytypes.FieldDataType + var priority uint8 + err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority) if err != nil { return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error()) } - key := &telemetrytypes.TelemetryFieldKey{ + keys = append(keys, &telemetrytypes.TelemetryFieldKey{ Name: name, - FieldContext: telemetrytypes.FieldContextAttribute, - FieldDataType: telemetrytypes.FieldDataTypeString, - } - keys = append(keys, key) + Signal: telemetrytypes.SignalMetrics, + FieldContext: fieldContext, + FieldDataType: fieldDataType, + }) } if rows.Err() != nil { @@ -486,20 +515,10 @@ func (t *telemetryMetaStore) GetKey(ctx context.Context, fieldKeySelector *telem func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) ([]string, error) { - args := []any{} - - var andConditions []string - - andConditions = append(andConditions, `unix_milli >= ?`) - args = append(args, fieldValueSelector.StartUnixMilli) - - andConditions = append(andConditions, `unix_milli <= ?`) - args = append(args, fieldValueSelector.EndUnixMilli) - - if len(fieldValueSelector.ExistingQuery) != 0 { - // TODO(srikanthccv): add the existing query to the where clause + // nothing to return as "related" value if there is nothing to filter on + if fieldValueSelector.ExistingQuery == "" { + return nil, nil } - whereClause := strings.Join(andConditions, " AND ") key := telemetrytypes.TelemetryFieldKey{ Name: fieldValueSelector.Name, @@ -508,20 +527,65 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel FieldDataType: fieldValueSelector.FieldDataType, } - // TODO(srikanthccv): add the select column - selectColumn, _ := t.conditionBuilder.GetTableFieldName(ctx, &key) + selectColumn, err := t.conditionBuilder.GetTableFieldName(ctx, &key) - args = append(args, fieldValueSelector.Limit) - filterSubQuery := fmt.Sprintf( - "SELECT DISTINCT %s FROM %s.%s WHERE %s LIMIT ?", - selectColumn, - t.relatedMetadataDBName, - t.relatedMetadataTblName, - whereClause, - ) - zap.L().Debug("filterSubQuery for related values", zap.String("query", filterSubQuery), zap.Any("args", args)) + if err != nil { + // we don't have a explicit column to select from the related metadata table + // so we will select either from resource_attributes or attributes table + // in that order + resourceColumn, _ := t.conditionBuilder.GetTableFieldName(ctx, &telemetrytypes.TelemetryFieldKey{ + Name: key.Name, + FieldContext: telemetrytypes.FieldContextResource, + FieldDataType: telemetrytypes.FieldDataTypeString, + }) + attributeColumn, _ := t.conditionBuilder.GetTableFieldName(ctx, &telemetrytypes.TelemetryFieldKey{ + Name: key.Name, + FieldContext: telemetrytypes.FieldContextAttribute, + FieldDataType: telemetrytypes.FieldDataTypeString, + }) + selectColumn = fmt.Sprintf("if(notEmpty(%s), %s, %s)", resourceColumn, resourceColumn, attributeColumn) + } - rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, filterSubQuery, args...) + sb := sqlbuilder.Select("DISTINCT " + selectColumn).From(t.relatedMetadataDBName + "." + t.relatedMetadataTblName) + + if len(fieldValueSelector.ExistingQuery) != 0 { + keysSelectors, err := parser.QueryStringToKeysSelectors(fieldValueSelector.ExistingQuery) + + if err == nil { + for idx := range keysSelectors { + keysSelectors[idx].Signal = fieldValueSelector.Signal + } + keys, err := t.GetKeysMulti(ctx, keysSelectors) + if err == nil { + whereClause, _, err := parser.PrepareWhereClause(fieldValueSelector.ExistingQuery, keys, t.conditionBuilder, &telemetrytypes.TelemetryFieldKey{}) + if err == nil { + sb.AddWhereClause(whereClause) + } else { + zap.L().Warn("error parsing existing query for related values", zap.Error(err)) + } + } + } + } + + if fieldValueSelector.StartUnixMilli != 0 { + sb.Where(sb.GE("unix_milli", fieldValueSelector.StartUnixMilli)) + } + + if fieldValueSelector.EndUnixMilli != 0 { + sb.Where(sb.LE("unix_milli", fieldValueSelector.EndUnixMilli)) + } + + if fieldValueSelector.Limit != 0 { + sb.Limit(fieldValueSelector.Limit) + } else { + sb.Limit(50) + } + + query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + + zap.L().Debug("query for related values", zap.String("query", query), zap.Any("args", args)) + + rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...) if err != nil { return nil, ErrFailedToGetRelatedValues } @@ -668,9 +732,66 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe return values, nil } -func (t *telemetryMetaStore) getMetricFieldValues(_ context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) { - // TODO(srikanthccv): implement this. use new tables? - return nil, nil +func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) { + sb := sqlbuilder. + Select("DISTINCT attr_string_value"). + From(t.metricsDBName + "." + t.metricsFieldsTblName) + + if fieldValueSelector.Name != "" { + sb.Where(sb.E("attr_name", fieldValueSelector.Name)) + } + + if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified { + sb.And(sb.E("attr_type", fieldValueSelector.FieldContext.TagType())) + } + + if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { + sb.And(sb.E("attr_datatype", fieldValueSelector.FieldDataType.TagDataType())) + } + + if fieldValueSelector.MetricContext != nil { + sb.And(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName)) + } + + if fieldValueSelector.StartUnixMilli > 0 { + sb.And(sb.GE("last_reported_unix_milli", fieldValueSelector.StartUnixMilli)) + } + + if fieldValueSelector.EndUnixMilli > 0 { + sb.And(sb.LE("first_reported_unix_milli", fieldValueSelector.EndUnixMilli)) + } + + if fieldValueSelector.Value != "" { + if fieldValueSelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact { + sb.And(sb.E("attr_string_value", fieldValueSelector.Value)) + } else { + sb.And(sb.Like("attr_string_value", "%"+fieldValueSelector.Value+"%")) + } + } + + if fieldValueSelector.Limit > 0 { + sb.Limit(fieldValueSelector.Limit) + } else { + sb.Limit(50) + } + + query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + + rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error()) + } + defer rows.Close() + + values := &telemetrytypes.TelemetryFieldValues{} + for rows.Next() { + var stringValue string + if err := rows.Scan(&stringValue); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error()) + } + values.StringValues = append(values.StringValues, stringValue) + } + return values, nil } func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, error) { diff --git a/pkg/telemetrymetadata/metadata_test.go b/pkg/telemetrymetadata/metadata_test.go index 74540b678e..f5d949cc33 100644 --- a/pkg/telemetrymetadata/metadata_test.go +++ b/pkg/telemetrymetadata/metadata_test.go @@ -39,8 +39,7 @@ func TestGetKeys(t *testing.T) { telemetrytraces.TagAttributesV2TableName, telemetrytraces.SpanIndexV3TableName, telemetrymetrics.DBName, - telemetrymetrics.TimeseriesV41weekTableName, - telemetrymetrics.TimeseriesV41weekTableName, + telemetrymetrics.AttributesMetadataTableName, telemetrylogs.DBName, telemetrylogs.LogsV2TableName, telemetrylogs.TagAttributesV2TableName, diff --git a/pkg/telemetrymetrics/tables.go b/pkg/telemetrymetrics/tables.go index cc498ffe33..d8a5039f60 100644 --- a/pkg/telemetrymetrics/tables.go +++ b/pkg/telemetrymetrics/tables.go @@ -1,21 +1,23 @@ package telemetrymetrics const ( - DBName = "signoz_metrics" - SamplesV4TableName = "distributed_samples_v4" - SamplesV4LocalTableName = "samples_v4" - SamplesV4Agg5mTableName = "distributed_samples_v4_agg_5m" - SamplesV4Agg5mLocalTableName = "samples_v4_agg_5m" - SamplesV4Agg30mTableName = "distributed_samples_v4_agg_30m" - SamplesV4Agg30mLocalTableName = "samples_v4_agg_30m" - ExpHistogramTableName = "distributed_exp_hist" - ExpHistogramLocalTableName = "exp_hist" - TimeseriesV4TableName = "distributed_time_series_v4" - TimeseriesV4LocalTableName = "time_series_v4" - TimeseriesV46hrsTableName = "distributed_time_series_v4_6hrs" - TimeseriesV46hrsLocalTableName = "time_series_v4_6hrs" - TimeseriesV41dayTableName = "distributed_time_series_v4_1day" - TimeseriesV41dayLocalTableName = "time_series_v4_1day" - TimeseriesV41weekTableName = "distributed_time_series_v4_1week" - TimeseriesV41weekLocalTableName = "time_series_v4_1week" + DBName = "signoz_metrics" + SamplesV4TableName = "distributed_samples_v4" + SamplesV4LocalTableName = "samples_v4" + SamplesV4Agg5mTableName = "distributed_samples_v4_agg_5m" + SamplesV4Agg5mLocalTableName = "samples_v4_agg_5m" + SamplesV4Agg30mTableName = "distributed_samples_v4_agg_30m" + SamplesV4Agg30mLocalTableName = "samples_v4_agg_30m" + ExpHistogramTableName = "distributed_exp_hist" + ExpHistogramLocalTableName = "exp_hist" + TimeseriesV4TableName = "distributed_time_series_v4" + TimeseriesV4LocalTableName = "time_series_v4" + TimeseriesV46hrsTableName = "distributed_time_series_v4_6hrs" + TimeseriesV46hrsLocalTableName = "time_series_v4_6hrs" + TimeseriesV41dayTableName = "distributed_time_series_v4_1day" + TimeseriesV41dayLocalTableName = "time_series_v4_1day" + TimeseriesV41weekTableName = "distributed_time_series_v4_1week" + TimeseriesV41weekLocalTableName = "time_series_v4_1week" + AttributesMetadataTableName = "distributed_metadata" + AttributesMetadataLocalTableName = "metadata" )