diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index e8929552b8..fb3af10b3e 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -25,6 +25,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/metrics" metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/app/parser" + "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/constants" @@ -67,7 +68,7 @@ type APIHandler struct { ruleManager *rules.Manager featureFlags interfaces.FeatureLookup ready func(http.HandlerFunc) http.HandlerFunc - queryBuilder *queryBuilder + queryBuilder *queryBuilder.QueryBuilder // SetupCompleted indicates if SigNoz is ready for general use. // at the moment, we mark the app ready when the first user @@ -106,12 +107,12 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { featureFlags: opts.FeatureFlags, } - builderOpts := queryBuilderOptions{ + builderOpts := queryBuilder.QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, BuildTraceQuery: tracesV3.PrepareTracesQuery, BuildLogQuery: logsv3.PrepareLogsQuery, } - aH.queryBuilder = NewQueryBuilder(builderOpts) + aH.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts) aH.ready = aH.testReady @@ -2707,7 +2708,7 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que return } - queries, err = aH.queryBuilder.prepareQueries(queryRangeParams, fields, spanKeys) + queries, err = aH.queryBuilder.PrepareQueries(queryRangeParams, fields, spanKeys) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) return diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index a1818cf446..5167dbce1b 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -18,6 +18,7 @@ import ( "go.uber.org/multierr" "go.signoz.io/signoz/pkg/query-service/app/metrics" + "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/model" @@ -911,7 +912,7 @@ func validateQueryRangeParamsV3(qp *v3.QueryRangeParamsV3) error { for _, q := range qp.CompositeQuery.BuilderQueries { expressions = append(expressions, q.Expression) } - errs := validateExpressions(expressions, evalFuncs, qp.CompositeQuery) + errs := validateExpressions(expressions, queryBuilder.EvalFuncs, qp.CompositeQuery) if len(errs) > 0 { return multierr.Combine(errs...) } diff --git a/pkg/query-service/app/query_builder.go b/pkg/query-service/app/queryBuilder/query_builder.go similarity index 92% rename from pkg/query-service/app/query_builder.go rename to pkg/query-service/app/queryBuilder/query_builder.go index 1645496643..b82a85ca4a 100644 --- a/pkg/query-service/app/query_builder.go +++ b/pkg/query-service/app/queryBuilder/query_builder.go @@ -1,4 +1,4 @@ -package app +package queryBuilder import ( "fmt" @@ -33,31 +33,31 @@ var SupportedFunctions = []string{ "radians", } -var evalFuncs = map[string]govaluate.ExpressionFunction{} +var EvalFuncs = map[string]govaluate.ExpressionFunction{} type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, fields map[string]v3.AttributeKey) (string, error) type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error) -type queryBuilder struct { - options queryBuilderOptions +type QueryBuilder struct { + options QueryBuilderOptions } -type queryBuilderOptions struct { +type QueryBuilderOptions struct { BuildTraceQuery prepareTracesQueryFunc BuildLogQuery prepareLogsQueryFunc BuildMetricQuery prepareMetricQueryFunc } -func NewQueryBuilder(options queryBuilderOptions) *queryBuilder { - return &queryBuilder{ +func NewQueryBuilder(options QueryBuilderOptions) *QueryBuilder { + return &QueryBuilder{ options: options, } } func init() { for _, fn := range SupportedFunctions { - evalFuncs[fn] = func(args ...interface{}) (interface{}, error) { + EvalFuncs[fn] = func(args ...interface{}) (interface{}, error) { return nil, nil } } @@ -127,7 +127,7 @@ func expressionToQuery(qp *v3.QueryRangeParamsV3, varToQuery map[string]string, return formulaQuery, nil } -func (qb *queryBuilder) prepareQueries(params *v3.QueryRangeParamsV3, args ...interface{}) (map[string]string, error) { +func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3, args ...interface{}) (map[string]string, error) { queries := make(map[string]string) compositeQuery := params.CompositeQuery @@ -173,7 +173,7 @@ func (qb *queryBuilder) prepareQueries(params *v3.QueryRangeParamsV3, args ...in // Build queries for each expression for _, query := range compositeQuery.BuilderQueries { if query.Expression != query.QueryName { - expression, _ := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, evalFuncs) + expression, _ := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, EvalFuncs) queryString, err := expressionToQuery(params, queries, expression) if err != nil { diff --git a/pkg/query-service/app/query_builder_test.go b/pkg/query-service/app/queryBuilder/query_builder_test.go similarity index 96% rename from pkg/query-service/app/query_builder_test.go rename to pkg/query-service/app/queryBuilder/query_builder_test.go index e72448a738..3716e2bb9f 100644 --- a/pkg/query-service/app/query_builder_test.go +++ b/pkg/query-service/app/queryBuilder/query_builder_test.go @@ -1,4 +1,4 @@ -package app +package queryBuilder import ( "strings" @@ -40,12 +40,12 @@ func TestBuildQueryWithMultipleQueriesAndFormula(t *testing.T) { }, }, } - qbOptions := queryBuilderOptions{ + qbOptions := QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, } qb := NewQueryBuilder(qbOptions) - queries, err := qb.prepareQueries(q) + queries, err := qb.PrepareQueries(q) require.NoError(t, err) @@ -81,12 +81,12 @@ func TestBuildQueryWithIncorrectQueryRef(t *testing.T) { }, } - qbOptions := queryBuilderOptions{ + qbOptions := QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, } qb := NewQueryBuilder(qbOptions) - _, err := qb.prepareQueries(q) + _, err := qb.PrepareQueries(q) require.NoError(t, err) }) @@ -151,12 +151,12 @@ func TestBuildQueryWithThreeOrMoreQueriesRefAndFormula(t *testing.T) { }, } - qbOptions := queryBuilderOptions{ + qbOptions := QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, } qb := NewQueryBuilder(qbOptions) - queries, err := qb.prepareQueries(q) + queries, err := qb.PrepareQueries(q) require.NoError(t, err) diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 15144e3422..bfef25b843 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -134,6 +134,7 @@ func (r ReduceToOperator) Validate() error { type QueryType string const ( + QueryTypeUnknown QueryType = "unknown" QueryTypeBuilder QueryType = "builder" QueryTypeClickHouseSQL QueryType = "clickhouse_sql" QueryTypePromQL QueryType = "promql" diff --git a/pkg/query-service/rules/alerting.go b/pkg/query-service/rules/alerting.go index ea82af76b8..55c2fdfafc 100644 --- a/pkg/query-service/rules/alerting.go +++ b/pkg/query-service/rules/alerting.go @@ -8,7 +8,7 @@ import ( "time" "github.com/pkg/errors" - "go.signoz.io/signoz/pkg/query-service/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/utils/labels" ) @@ -139,19 +139,19 @@ const ( ) type RuleCondition struct { - CompositeMetricQuery *model.CompositeMetricQuery `json:"compositeMetricQuery,omitempty" yaml:"compositeMetricQuery,omitempty"` - CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"` - Target *float64 `yaml:"target,omitempty" json:"target,omitempty"` - MatchType `json:"matchType,omitempty"` + CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"` + CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"` + Target *float64 `yaml:"target,omitempty" json:"target,omitempty"` + MatchType `json:"matchType,omitempty"` } func (rc *RuleCondition) IsValid() bool { - if rc.CompositeMetricQuery == nil { + if rc.CompositeQuery == nil { return false } - if rc.QueryType() == model.QUERY_BUILDER { + if rc.QueryType() == v3.QueryTypeBuilder { if rc.Target == nil { return false } @@ -159,9 +159,9 @@ func (rc *RuleCondition) IsValid() bool { return false } } - if rc.QueryType() == model.PROM { + if rc.QueryType() == v3.QueryTypePromQL { - if len(rc.CompositeMetricQuery.PromQueries) == 0 { + if len(rc.CompositeQuery.PromQueries) == 0 { return false } } @@ -169,11 +169,11 @@ func (rc *RuleCondition) IsValid() bool { } // QueryType is a short hand method to get query type -func (rc *RuleCondition) QueryType() model.QueryType { - if rc.CompositeMetricQuery != nil { - return rc.CompositeMetricQuery.QueryType +func (rc *RuleCondition) QueryType() v3.QueryType { + if rc.CompositeQuery != nil { + return rc.CompositeQuery.QueryType } - return 0 + return v3.QueryTypeUnknown } // String is useful in printing rule condition in logs diff --git a/pkg/query-service/rules/apiParams.go b/pkg/query-service/rules/apiParams.go index 53503c6c52..649ba6373d 100644 --- a/pkg/query-service/rules/apiParams.go +++ b/pkg/query-service/rules/apiParams.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "go.signoz.io/signoz/pkg/query-service/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.uber.org/zap" "go.signoz.io/signoz/pkg/query-service/utils/times" @@ -90,9 +91,9 @@ func parseIntoRule(initRule PostableRule, content []byte, kind string) (*Postabl rule.EvalWindow = Duration(5 * time.Minute) rule.Frequency = Duration(1 * time.Minute) rule.RuleCondition = &RuleCondition{ - CompositeMetricQuery: &model.CompositeMetricQuery{ - QueryType: model.PROM, - PromQueries: map[string]*model.PromQuery{ + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypePromQL, + PromQueries: map[string]*v3.PromQuery{ "A": { Query: rule.Expr, }, @@ -110,14 +111,14 @@ func parseIntoRule(initRule PostableRule, content []byte, kind string) (*Postabl } if rule.RuleCondition != nil { - if rule.RuleCondition.CompositeMetricQuery.QueryType == model.QUERY_BUILDER { + if rule.RuleCondition.CompositeQuery.QueryType == v3.QueryTypeBuilder { rule.RuleType = RuleTypeThreshold - } else if rule.RuleCondition.CompositeMetricQuery.QueryType == model.PROM { + } else if rule.RuleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL { rule.RuleType = RuleTypeProm } - for qLabel, q := range rule.RuleCondition.CompositeMetricQuery.BuilderQueries { - if q.MetricName != "" && q.Expression == "" { + for qLabel, q := range rule.RuleCondition.CompositeQuery.BuilderQueries { + if q.AggregateAttribute.Key != "" && q.Expression == "" { q.Expression = qLabel } } @@ -153,7 +154,7 @@ func (r *PostableRule) Validate() (errs []error) { if r.RuleCondition == nil { errs = append(errs, errors.Errorf("rule condition is required")) } else { - if r.RuleCondition.CompositeMetricQuery == nil { + if r.RuleCondition.CompositeQuery == nil { errs = append(errs, errors.Errorf("composite metric query is required")) } } diff --git a/pkg/query-service/rules/promRule.go b/pkg/query-service/rules/promRule.go index f29a8311fc..76afef01b4 100644 --- a/pkg/query-service/rules/promRule.go +++ b/pkg/query-service/rules/promRule.go @@ -12,7 +12,7 @@ import ( plabels "github.com/prometheus/prometheus/model/labels" pql "github.com/prometheus/prometheus/promql" - "go.signoz.io/signoz/pkg/query-service/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" qslabels "go.signoz.io/signoz/pkg/query-service/utils/labels" "go.signoz.io/signoz/pkg/query-service/utils/times" "go.signoz.io/signoz/pkg/query-service/utils/timestamp" @@ -288,9 +288,9 @@ func (r *PromRule) SendAlerts(ctx context.Context, ts time.Time, resendDelay tim func (r *PromRule) getPqlQuery() (string, error) { - if r.ruleCondition.CompositeMetricQuery.QueryType == model.PROM { - if len(r.ruleCondition.CompositeMetricQuery.PromQueries) > 0 { - if promQuery, ok := r.ruleCondition.CompositeMetricQuery.PromQueries["A"]; ok { + if r.ruleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL { + if len(r.ruleCondition.CompositeQuery.PromQueries) > 0 { + if promQuery, ok := r.ruleCondition.CompositeQuery.PromQueries["A"]; ok { query := promQuery.Query if query == "" { return query, fmt.Errorf("a promquery needs to be set for this rule to function") diff --git a/pkg/query-service/rules/thresholdRule.go b/pkg/query-service/rules/thresholdRule.go index 9da34d2ef3..39d0aa0cad 100644 --- a/pkg/query-service/rules/thresholdRule.go +++ b/pkg/query-service/rules/thresholdRule.go @@ -14,15 +14,20 @@ import ( "go.uber.org/zap" "github.com/ClickHouse/clickhouse-go/v2" - "go.signoz.io/signoz/pkg/query-service/app/metrics" + + "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" "go.signoz.io/signoz/pkg/query-service/constants" - qsmodel "go.signoz.io/signoz/pkg/query-service/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/utils/labels" querytemplate "go.signoz.io/signoz/pkg/query-service/utils/queryTemplate" "go.signoz.io/signoz/pkg/query-service/utils/times" "go.signoz.io/signoz/pkg/query-service/utils/timestamp" "go.signoz.io/signoz/pkg/query-service/utils/value" + logsv3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" + tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + yaml "gopkg.in/yaml.v2" ) @@ -48,6 +53,8 @@ type ThresholdRule struct { // map of active alerts active map[uint64]*Alert + queryBuilder *queryBuilder.QueryBuilder + opts ThresholdRuleOpts } @@ -92,6 +99,13 @@ func NewThresholdRule( t.evalWindow = 5 * time.Minute } + builderOpts := queryBuilder.QueryBuilderOptions{ + BuildMetricQuery: metricsv3.PrepareMetricQuery, + BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildLogQuery: logsv3.PrepareLogsQuery, + } + t.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts) + zap.S().Info("msg:", "creating new alerting rule", "\t name:", t.name, "\t condition:", t.ruleCondition.String(), "\t generatorURL:", t.GeneratorURL()) return &t, nil @@ -338,25 +352,25 @@ func (r *ThresholdRule) CheckCondition(v float64) bool { } } -func (r *ThresholdRule) prepareQueryRange(ts time.Time) *qsmodel.QueryRangeParamsV2 { +func (r *ThresholdRule) prepareQueryRange(ts time.Time) *v3.QueryRangeParamsV3 { // todo(amol): add 30 seconds to evalWindow for rate calc - if r.ruleCondition.QueryType() == qsmodel.CLICKHOUSE { - return &qsmodel.QueryRangeParamsV2{ - Start: ts.Add(-time.Duration(r.evalWindow)).UnixMilli(), - End: ts.UnixMilli(), - Step: 30, - CompositeMetricQuery: r.ruleCondition.CompositeMetricQuery, - Variables: make(map[string]interface{}, 0), + if r.ruleCondition.QueryType() == v3.QueryTypeClickHouseSQL { + return &v3.QueryRangeParamsV3{ + Start: ts.Add(-time.Duration(r.evalWindow)).UnixMilli(), + End: ts.UnixMilli(), + Step: 30, + CompositeQuery: r.ruleCondition.CompositeQuery, + Variables: make(map[string]interface{}, 0), } } // default mode - return &qsmodel.QueryRangeParamsV2{ - Start: ts.Add(-time.Duration(r.evalWindow)).UnixMilli(), - End: ts.UnixMilli(), - Step: 30, - CompositeMetricQuery: r.ruleCondition.CompositeMetricQuery, + return &v3.QueryRangeParamsV3{ + Start: ts.Add(-time.Duration(r.evalWindow)).UnixMilli(), + End: ts.UnixMilli(), + Step: 30, + CompositeQuery: r.ruleCondition.CompositeQuery, } } @@ -502,7 +516,7 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer } } else { - if r.Condition().QueryType() == qsmodel.QUERY_BUILDER { + if r.Condition().QueryType() == v3.QueryTypeBuilder { // for query builder, time series data // we skip the first record to support rate cases correctly // improvement(amol): explore approaches to limit this only for @@ -537,9 +551,9 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer func (r *ThresholdRule) prepareBuilderQueries(ts time.Time) (map[string]string, error) { params := r.prepareQueryRange(ts) - runQueries := metrics.PrepareBuilderMetricQueries(params, constants.SIGNOZ_TIMESERIES_TABLENAME) + runQueries, err := r.queryBuilder.PrepareQueries(params) - return runQueries.Queries, runQueries.Err + return runQueries, err } func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]string, error) { @@ -549,7 +563,7 @@ func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]strin return nil, fmt.Errorf("rule condition is empty") } - if r.ruleCondition.QueryType() != qsmodel.CLICKHOUSE { + if r.ruleCondition.QueryType() != v3.QueryTypeClickHouseSQL { zap.S().Debugf("ruleid:", r.ID(), "\t msg: unsupported query type in prepareClickhouseQueries()") return nil, fmt.Errorf("failed to prepare clickhouse queries") } @@ -557,9 +571,9 @@ func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]strin params := r.prepareQueryRange(ts) // replace reserved go template variables - querytemplate.AssignReservedVars(params) + querytemplate.AssignReservedVarsV3(params) - for name, chQuery := range r.ruleCondition.CompositeMetricQuery.ClickHouseQueries { + for name, chQuery := range r.ruleCondition.CompositeQuery.ClickHouseQueries { if chQuery.Disabled { continue } @@ -586,7 +600,7 @@ func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]strin // query looks if alert condition is being // satisfied and returns the signals func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch clickhouse.Conn) (Vector, error) { - if r.ruleCondition == nil || r.ruleCondition.CompositeMetricQuery == nil { + if r.ruleCondition == nil || r.ruleCondition.CompositeQuery == nil { r.SetHealth(HealthBad) return nil, fmt.Errorf("invalid rule condition") } @@ -596,7 +610,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch c var err error // fetch the target query based on query type - if r.ruleCondition.QueryType() == qsmodel.QUERY_BUILDER { + if r.ruleCondition.QueryType() == v3.QueryTypeBuilder { queries, err = r.prepareBuilderQueries(ts) @@ -605,7 +619,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch c return nil, fmt.Errorf("failed to prepare metric queries") } - } else if r.ruleCondition.QueryType() == qsmodel.CLICKHOUSE { + } else if r.ruleCondition.QueryType() == v3.QueryTypeClickHouseSQL { queries, err = r.prepareClickhouseQueries(ts)