feat(query-service): alerts integration with query builder v3 (#2663)

pkg/query-service/app/http_handler.go

@@ -25,6 +25,7 @@ import (
 	"go.signoz.io/signoz/pkg/query-service/app/metrics"
 	metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3"
 	"go.signoz.io/signoz/pkg/query-service/app/parser"
+	"go.signoz.io/signoz/pkg/query-service/app/queryBuilder"
 	tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3"
 	"go.signoz.io/signoz/pkg/query-service/auth"
 	"go.signoz.io/signoz/pkg/query-service/constants"
@@ -67,7 +68,7 @@ type APIHandler struct {
 	ruleManager  *rules.Manager
 	featureFlags interfaces.FeatureLookup
 	ready        func(http.HandlerFunc) http.HandlerFunc
-	queryBuilder *queryBuilder
+	queryBuilder *queryBuilder.QueryBuilder
 
 	// SetupCompleted indicates if SigNoz is ready for general use.
 	// at the moment, we mark the app ready when the first user
@@ -106,12 +107,12 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) {
 		featureFlags: opts.FeatureFlags,
 	}
 
-	builderOpts := queryBuilderOptions{
+	builderOpts := queryBuilder.QueryBuilderOptions{
 		BuildMetricQuery: metricsv3.PrepareMetricQuery,
 		BuildTraceQuery:  tracesV3.PrepareTracesQuery,
 		BuildLogQuery:    logsv3.PrepareLogsQuery,
 	}
-	aH.queryBuilder = NewQueryBuilder(builderOpts)
+	aH.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts)
 
 	aH.ready = aH.testReady
 
@@ -2707,7 +2708,7 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que
 			return
 		}
 
-		queries, err = aH.queryBuilder.prepareQueries(queryRangeParams, fields, spanKeys)
+		queries, err = aH.queryBuilder.PrepareQueries(queryRangeParams, fields, spanKeys)
 		if err != nil {
 			RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
 			return

pkg/query-service/app/parser.go

@@ -18,6 +18,7 @@ import (
 	"go.uber.org/multierr"
 
 	"go.signoz.io/signoz/pkg/query-service/app/metrics"
+	"go.signoz.io/signoz/pkg/query-service/app/queryBuilder"
 	"go.signoz.io/signoz/pkg/query-service/auth"
 	"go.signoz.io/signoz/pkg/query-service/constants"
 	"go.signoz.io/signoz/pkg/query-service/model"
@@ -911,7 +912,7 @@ func validateQueryRangeParamsV3(qp *v3.QueryRangeParamsV3) error {
 	for _, q := range qp.CompositeQuery.BuilderQueries {
 		expressions = append(expressions, q.Expression)
 	}
-	errs := validateExpressions(expressions, evalFuncs, qp.CompositeQuery)
+	errs := validateExpressions(expressions, queryBuilder.EvalFuncs, qp.CompositeQuery)
 	if len(errs) > 0 {
 		return multierr.Combine(errs...)
 	}

pkg/query-service/app/queryBuilder/query_builder.go

@@ -1,4 +1,4 @@
-package app
+package queryBuilder
 
 import (
 	"fmt"
@@ -33,31 +33,31 @@ var SupportedFunctions = []string{
 	"radians",
 }
 
-var evalFuncs = map[string]govaluate.ExpressionFunction{}
+var EvalFuncs = map[string]govaluate.ExpressionFunction{}
 
 type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error)
 type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, fields map[string]v3.AttributeKey) (string, error)
 type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error)
 
-type queryBuilder struct {
+type QueryBuilder struct {
-	options queryBuilderOptions
+	options QueryBuilderOptions
 }
 
-type queryBuilderOptions struct {
+type QueryBuilderOptions struct {
 	BuildTraceQuery  prepareTracesQueryFunc
 	BuildLogQuery    prepareLogsQueryFunc
 	BuildMetricQuery prepareMetricQueryFunc
 }
 
-func NewQueryBuilder(options queryBuilderOptions) *queryBuilder {
+func NewQueryBuilder(options QueryBuilderOptions) *QueryBuilder {
-	return &queryBuilder{
+	return &QueryBuilder{
 		options: options,
 	}
 }
 
 func init() {
 	for _, fn := range SupportedFunctions {
-		evalFuncs[fn] = func(args ...interface{}) (interface{}, error) {
+		EvalFuncs[fn] = func(args ...interface{}) (interface{}, error) {
 			return nil, nil
 		}
 	}
@@ -127,7 +127,7 @@ func expressionToQuery(qp *v3.QueryRangeParamsV3, varToQuery map[string]string,
 	return formulaQuery, nil
 }
 
-func (qb *queryBuilder) prepareQueries(params *v3.QueryRangeParamsV3, args ...interface{}) (map[string]string, error) {
+func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3, args ...interface{}) (map[string]string, error) {
 	queries := make(map[string]string)
 
 	compositeQuery := params.CompositeQuery
@@ -173,7 +173,7 @@ func (qb *queryBuilder) prepareQueries(params *v3.QueryRangeParamsV3, args ...in
 		// Build queries for each expression
 		for _, query := range compositeQuery.BuilderQueries {
 			if query.Expression != query.QueryName {
-				expression, _ := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, evalFuncs)
+				expression, _ := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, EvalFuncs)
 
 				queryString, err := expressionToQuery(params, queries, expression)
 				if err != nil {

pkg/query-service/app/queryBuilder/query_builder_test.go

@@ -1,4 +1,4 @@
-package app
+package queryBuilder
 
 import (
 	"strings"
@@ -40,12 +40,12 @@ func TestBuildQueryWithMultipleQueriesAndFormula(t *testing.T) {
 				},
 			},
 		}
-		qbOptions := queryBuilderOptions{
+		qbOptions := QueryBuilderOptions{
 			BuildMetricQuery: metricsv3.PrepareMetricQuery,
 		}
 		qb := NewQueryBuilder(qbOptions)
 
-		queries, err := qb.prepareQueries(q)
+		queries, err := qb.PrepareQueries(q)
 
 		require.NoError(t, err)
 
@@ -81,12 +81,12 @@ func TestBuildQueryWithIncorrectQueryRef(t *testing.T) {
 			},
 		}
 
-		qbOptions := queryBuilderOptions{
+		qbOptions := QueryBuilderOptions{
 			BuildMetricQuery: metricsv3.PrepareMetricQuery,
 		}
 		qb := NewQueryBuilder(qbOptions)
 
-		_, err := qb.prepareQueries(q)
+		_, err := qb.PrepareQueries(q)
 
 		require.NoError(t, err)
 	})
@@ -151,12 +151,12 @@ func TestBuildQueryWithThreeOrMoreQueriesRefAndFormula(t *testing.T) {
 			},
 		}
 
-		qbOptions := queryBuilderOptions{
+		qbOptions := QueryBuilderOptions{
 			BuildMetricQuery: metricsv3.PrepareMetricQuery,
 		}
 		qb := NewQueryBuilder(qbOptions)
 
-		queries, err := qb.prepareQueries(q)
+		queries, err := qb.PrepareQueries(q)
 
 		require.NoError(t, err)
 

pkg/query-service/model/v3/v3.go

@@ -134,6 +134,7 @@ func (r ReduceToOperator) Validate() error {
 type QueryType string
 
 const (
+	QueryTypeUnknown       QueryType = "unknown"
 	QueryTypeBuilder       QueryType = "builder"
 	QueryTypeClickHouseSQL QueryType = "clickhouse_sql"
 	QueryTypePromQL        QueryType = "promql"

pkg/query-service/rules/alerting.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"go.signoz.io/signoz/pkg/query-service/model"
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.signoz.io/signoz/pkg/query-service/utils/labels"
 )
 
@@ -139,19 +139,19 @@ const (
 )
 
 type RuleCondition struct {
-	CompositeMetricQuery *model.CompositeMetricQuery `json:"compositeMetricQuery,omitempty" yaml:"compositeMetricQuery,omitempty"`
+	CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"`
-	CompareOp            CompareOp                   `yaml:"op,omitempty" json:"op,omitempty"`
+	CompareOp      CompareOp          `yaml:"op,omitempty" json:"op,omitempty"`
-	Target               *float64                    `yaml:"target,omitempty" json:"target,omitempty"`
+	Target         *float64           `yaml:"target,omitempty" json:"target,omitempty"`
-	MatchType            `json:"matchType,omitempty"`
+	MatchType      `json:"matchType,omitempty"`
 }
 
 func (rc *RuleCondition) IsValid() bool {
 
-	if rc.CompositeMetricQuery == nil {
+	if rc.CompositeQuery == nil {
 		return false
 	}
 
-	if rc.QueryType() == model.QUERY_BUILDER {
+	if rc.QueryType() == v3.QueryTypeBuilder {
 		if rc.Target == nil {
 			return false
 		}
@@ -159,9 +159,9 @@ func (rc *RuleCondition) IsValid() bool {
 			return false
 		}
 	}
-	if rc.QueryType() == model.PROM {
+	if rc.QueryType() == v3.QueryTypePromQL {
 
-		if len(rc.CompositeMetricQuery.PromQueries) == 0 {
+		if len(rc.CompositeQuery.PromQueries) == 0 {
 			return false
 		}
 	}
@@ -169,11 +169,11 @@ func (rc *RuleCondition) IsValid() bool {
 }
 
 // QueryType is a short hand method to get query type
-func (rc *RuleCondition) QueryType() model.QueryType {
+func (rc *RuleCondition) QueryType() v3.QueryType {
-	if rc.CompositeMetricQuery != nil {
+	if rc.CompositeQuery != nil {
-		return rc.CompositeMetricQuery.QueryType
+		return rc.CompositeQuery.QueryType
 	}
-	return 0
+	return v3.QueryTypeUnknown
 }
 
 // String is useful in printing rule condition in logs

pkg/query-service/rules/apiParams.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/pkg/errors"
 	"go.signoz.io/signoz/pkg/query-service/model"
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.uber.org/zap"
 
 	"go.signoz.io/signoz/pkg/query-service/utils/times"
@@ -90,9 +91,9 @@ func parseIntoRule(initRule PostableRule, content []byte, kind string) (*Postabl
 		rule.EvalWindow = Duration(5 * time.Minute)
 		rule.Frequency = Duration(1 * time.Minute)
 		rule.RuleCondition = &RuleCondition{
-			CompositeMetricQuery: &model.CompositeMetricQuery{
+			CompositeQuery: &v3.CompositeQuery{
-				QueryType: model.PROM,
+				QueryType: v3.QueryTypePromQL,
-				PromQueries: map[string]*model.PromQuery{
+				PromQueries: map[string]*v3.PromQuery{
 					"A": {
 						Query: rule.Expr,
 					},
@@ -110,14 +111,14 @@ func parseIntoRule(initRule PostableRule, content []byte, kind string) (*Postabl
 	}
 
 	if rule.RuleCondition != nil {
-		if rule.RuleCondition.CompositeMetricQuery.QueryType == model.QUERY_BUILDER {
+		if rule.RuleCondition.CompositeQuery.QueryType == v3.QueryTypeBuilder {
 			rule.RuleType = RuleTypeThreshold
-		} else if rule.RuleCondition.CompositeMetricQuery.QueryType == model.PROM {
+		} else if rule.RuleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL {
 			rule.RuleType = RuleTypeProm
 		}
 
-		for qLabel, q := range rule.RuleCondition.CompositeMetricQuery.BuilderQueries {
+		for qLabel, q := range rule.RuleCondition.CompositeQuery.BuilderQueries {
-			if q.MetricName != "" && q.Expression == "" {
+			if q.AggregateAttribute.Key != "" && q.Expression == "" {
 				q.Expression = qLabel
 			}
 		}
@@ -153,7 +154,7 @@ func (r *PostableRule) Validate() (errs []error) {
 	if r.RuleCondition == nil {
 		errs = append(errs, errors.Errorf("rule condition is required"))
 	} else {
-		if r.RuleCondition.CompositeMetricQuery == nil {
+		if r.RuleCondition.CompositeQuery == nil {
 			errs = append(errs, errors.Errorf("composite metric query is required"))
 		}
 	}

pkg/query-service/rules/promRule.go

@@ -12,7 +12,7 @@ import (
 
 	plabels "github.com/prometheus/prometheus/model/labels"
 	pql "github.com/prometheus/prometheus/promql"
-	"go.signoz.io/signoz/pkg/query-service/model"
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	qslabels "go.signoz.io/signoz/pkg/query-service/utils/labels"
 	"go.signoz.io/signoz/pkg/query-service/utils/times"
 	"go.signoz.io/signoz/pkg/query-service/utils/timestamp"
@@ -288,9 +288,9 @@ func (r *PromRule) SendAlerts(ctx context.Context, ts time.Time, resendDelay tim
 
 func (r *PromRule) getPqlQuery() (string, error) {
 
-	if r.ruleCondition.CompositeMetricQuery.QueryType == model.PROM {
+	if r.ruleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL {
-		if len(r.ruleCondition.CompositeMetricQuery.PromQueries) > 0 {
+		if len(r.ruleCondition.CompositeQuery.PromQueries) > 0 {
-			if promQuery, ok := r.ruleCondition.CompositeMetricQuery.PromQueries["A"]; ok {
+			if promQuery, ok := r.ruleCondition.CompositeQuery.PromQueries["A"]; ok {
 				query := promQuery.Query
 				if query == "" {
 					return query, fmt.Errorf("a promquery needs to be set for this rule to function")

pkg/query-service/rules/thresholdRule.go

@@ -14,15 +14,20 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
-	"go.signoz.io/signoz/pkg/query-service/app/metrics"
+
+	"go.signoz.io/signoz/pkg/query-service/app/queryBuilder"
 	"go.signoz.io/signoz/pkg/query-service/constants"
-	qsmodel "go.signoz.io/signoz/pkg/query-service/model"
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.signoz.io/signoz/pkg/query-service/utils/labels"
 	querytemplate "go.signoz.io/signoz/pkg/query-service/utils/queryTemplate"
 	"go.signoz.io/signoz/pkg/query-service/utils/times"
 	"go.signoz.io/signoz/pkg/query-service/utils/timestamp"
 	"go.signoz.io/signoz/pkg/query-service/utils/value"
 
+	logsv3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3"
+	metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3"
+	tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3"
+
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -48,6 +53,8 @@ type ThresholdRule struct {
 	// map of active alerts
 	active map[uint64]*Alert
 
+	queryBuilder *queryBuilder.QueryBuilder
+
 	opts ThresholdRuleOpts
 }
 
@@ -92,6 +99,13 @@ func NewThresholdRule(
 		t.evalWindow = 5 * time.Minute
 	}
 
+	builderOpts := queryBuilder.QueryBuilderOptions{
+		BuildMetricQuery: metricsv3.PrepareMetricQuery,
+		BuildTraceQuery:  tracesV3.PrepareTracesQuery,
+		BuildLogQuery:    logsv3.PrepareLogsQuery,
+	}
+	t.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts)
+
 	zap.S().Info("msg:", "creating new alerting rule", "\t name:", t.name, "\t condition:", t.ruleCondition.String(), "\t generatorURL:", t.GeneratorURL())
 
 	return &t, nil
@@ -338,25 +352,25 @@ func (r *ThresholdRule) CheckCondition(v float64) bool {
 	}
 }
 
-func (r *ThresholdRule) prepareQueryRange(ts time.Time) *qsmodel.QueryRangeParamsV2 {
+func (r *ThresholdRule) prepareQueryRange(ts time.Time) *v3.QueryRangeParamsV3 {
 	// todo(amol): add 30 seconds to evalWindow for rate calc
 
-	if r.ruleCondition.QueryType() == qsmodel.CLICKHOUSE {
+	if r.ruleCondition.QueryType() == v3.QueryTypeClickHouseSQL {
-		return &qsmodel.QueryRangeParamsV2{
+		return &v3.QueryRangeParamsV3{
-			Start:                ts.Add(-time.Duration(r.evalWindow)).UnixMilli(),
+			Start:          ts.Add(-time.Duration(r.evalWindow)).UnixMilli(),
-			End:                  ts.UnixMilli(),
+			End:            ts.UnixMilli(),
-			Step:                 30,
+			Step:           30,
-			CompositeMetricQuery: r.ruleCondition.CompositeMetricQuery,
+			CompositeQuery: r.ruleCondition.CompositeQuery,
-			Variables:            make(map[string]interface{}, 0),
+			Variables:      make(map[string]interface{}, 0),
 		}
 	}
 
 	// default mode
-	return &qsmodel.QueryRangeParamsV2{
+	return &v3.QueryRangeParamsV3{
-		Start:                ts.Add(-time.Duration(r.evalWindow)).UnixMilli(),
+		Start:          ts.Add(-time.Duration(r.evalWindow)).UnixMilli(),
-		End:                  ts.UnixMilli(),
+		End:            ts.UnixMilli(),
-		Step:                 30,
+		Step:           30,
-		CompositeMetricQuery: r.ruleCondition.CompositeMetricQuery,
+		CompositeQuery: r.ruleCondition.CompositeQuery,
 	}
 }
 
@@ -502,7 +516,7 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
 			}
 
 		} else {
-			if r.Condition().QueryType() == qsmodel.QUERY_BUILDER {
+			if r.Condition().QueryType() == v3.QueryTypeBuilder {
 				// for query builder, time series data
 				// we skip the first record to support rate cases correctly
 				// improvement(amol): explore approaches to limit this only for
@@ -537,9 +551,9 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
 
 func (r *ThresholdRule) prepareBuilderQueries(ts time.Time) (map[string]string, error) {
 	params := r.prepareQueryRange(ts)
-	runQueries := metrics.PrepareBuilderMetricQueries(params, constants.SIGNOZ_TIMESERIES_TABLENAME)
+	runQueries, err := r.queryBuilder.PrepareQueries(params)
 
-	return runQueries.Queries, runQueries.Err
+	return runQueries, err
 }
 
 func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]string, error) {
@@ -549,7 +563,7 @@ func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]strin
 		return nil, fmt.Errorf("rule condition is empty")
 	}
 
-	if r.ruleCondition.QueryType() != qsmodel.CLICKHOUSE {
+	if r.ruleCondition.QueryType() != v3.QueryTypeClickHouseSQL {
 		zap.S().Debugf("ruleid:", r.ID(), "\t msg: unsupported query type in prepareClickhouseQueries()")
 		return nil, fmt.Errorf("failed to prepare clickhouse queries")
 	}
@@ -557,9 +571,9 @@ func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]strin
 	params := r.prepareQueryRange(ts)
 
 	// replace reserved go template variables
-	querytemplate.AssignReservedVars(params)
+	querytemplate.AssignReservedVarsV3(params)
 
-	for name, chQuery := range r.ruleCondition.CompositeMetricQuery.ClickHouseQueries {
+	for name, chQuery := range r.ruleCondition.CompositeQuery.ClickHouseQueries {
 		if chQuery.Disabled {
 			continue
 		}
@@ -586,7 +600,7 @@ func (r *ThresholdRule) prepareClickhouseQueries(ts time.Time) (map[string]strin
 // query looks if alert condition is being
 // satisfied and returns the signals
 func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch clickhouse.Conn) (Vector, error) {
-	if r.ruleCondition == nil || r.ruleCondition.CompositeMetricQuery == nil {
+	if r.ruleCondition == nil || r.ruleCondition.CompositeQuery == nil {
 		r.SetHealth(HealthBad)
 		return nil, fmt.Errorf("invalid rule condition")
 	}
@@ -596,7 +610,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch c
 	var err error
 
 	// fetch the target query based on query type
-	if r.ruleCondition.QueryType() == qsmodel.QUERY_BUILDER {
+	if r.ruleCondition.QueryType() == v3.QueryTypeBuilder {
 
 		queries, err = r.prepareBuilderQueries(ts)
 
@@ -605,7 +619,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch c
 			return nil, fmt.Errorf("failed to prepare metric queries")
 		}
 
-	} else if r.ruleCondition.QueryType() == qsmodel.CLICKHOUSE {
+	} else if r.ruleCondition.QueryType() == v3.QueryTypeClickHouseSQL {
 
 		queries, err = r.prepareClickhouseQueries(ts)