fix: update prom rule to use range query (#4461)

frontend/src/container/FormAlertRules/RuleOptions.tsx

@@ -83,7 +83,7 @@ function RuleOptions({
 		</InlineSelect>
 	);
 
-	const renderThresholdMatchOpts = (): JSX.Element => (
+	const renderMatchOpts = (): JSX.Element => (
 		<InlineSelect
 			getPopupContainer={popupContainer}
 			defaultValue={defaultMatchType}
@@ -98,17 +98,13 @@ function RuleOptions({
 		</InlineSelect>
 	);
 
-	const renderPromMatchOpts = (): JSX.Element => (
-		<InlineSelect
-			getPopupContainer={popupContainer}
-			defaultValue={defaultMatchType}
-			style={{ minWidth: '130px' }}
-			value={alertDef.condition?.matchType}
-			onChange={(value: string | unknown): void => handleMatchOptChange(value)}
-		>
-			<Select.Option value="1">{t('option_atleastonce')}</Select.Option>
-		</InlineSelect>
-	);
+	const onChangeEvalWindow = (value: string | unknown): void => {
+		const ew = (value as string) || alertDef.evalWindow;
+		setAlertDef({
+			...alertDef,
+			evalWindow: ew,
+		});
+	};
 
 	const renderEvalWindows = (): JSX.Element => (
 		<InlineSelect
@@ -116,13 +112,7 @@ function RuleOptions({
 			defaultValue={defaultEvalWindow}
 			style={{ minWidth: '120px' }}
 			value={alertDef.evalWindow}
-			onChange={(value: string | unknown): void => {
-				const ew = (value as string) || alertDef.evalWindow;
-				setAlertDef({
-					...alertDef,
-					evalWindow: ew,
-				});
-			}}
+			onChange={onChangeEvalWindow}
 		>
 			<Select.Option value="5m0s">{t('option_5min')}</Select.Option>
 			<Select.Option value="10m0s">{t('option_10min')}</Select.Option>
@@ -133,6 +123,20 @@ function RuleOptions({
 		</InlineSelect>
 	);
 
+	const renderPromEvalWindows = (): JSX.Element => (
+		<InlineSelect
+			getPopupContainer={popupContainer}
+			defaultValue={defaultEvalWindow}
+			style={{ minWidth: '120px' }}
+			value={alertDef.evalWindow}
+			onChange={onChangeEvalWindow}
+		>
+			<Select.Option value="5m0s">{t('option_5min')}</Select.Option>
+			<Select.Option value="10m0s">{t('option_10min')}</Select.Option>
+			<Select.Option value="15m0s">{t('option_15min')}</Select.Option>
+		</InlineSelect>
+	);
+
 	const renderThresholdRuleOpts = (): JSX.Element => (
 		<Form.Item>
 			<Typography.Text>
@@ -147,7 +151,7 @@ function RuleOptions({
 					onChange={onChangeSelectedQueryName}
 				/>
 				<Typography.Text>is</Typography.Text>
-				{renderCompareOps()} {t('text_condition2')} {renderThresholdMatchOpts()}{' '}
+				{renderCompareOps()} {t('text_condition2')} {renderMatchOpts()}{' '}
 				{t('text_condition3')} {renderEvalWindows()}
 			</Typography.Text>
 		</Form.Item>
@@ -167,7 +171,8 @@ function RuleOptions({
 					onChange={onChangeSelectedQueryName}
 				/>
 				<Typography.Text>is</Typography.Text>
-				{renderCompareOps()} {t('text_condition2')} {renderPromMatchOpts()}
+				{renderCompareOps()} {t('text_condition2')} {renderMatchOpts()}
+				{t('text_condition3')} {renderPromEvalWindows()}
 			</Typography.Text>
 		</Form.Item>
 	);

pkg/query-service/pqlEngine/engine.go

@@ -10,7 +10,6 @@ import (
 	"github.com/prometheus/common/promlog"
 	plog "github.com/prometheus/common/promlog"
 	pconfig "github.com/prometheus/prometheus/config"
-	plabels "github.com/prometheus/prometheus/model/labels"
 	pql "github.com/prometheus/prometheus/promql"
 	pstorage "github.com/prometheus/prometheus/storage"
 	premote "github.com/prometheus/prometheus/storage/remote"
@@ -89,8 +88,8 @@ func NewPqlEngine(config *pconfig.Config) (*PqlEngine, error) {
 	}, nil
 }
 
-func (p *PqlEngine) RunAlertQuery(ctx context.Context, qs string, t time.Time) (pql.Vector, error) {
-	q, err := p.engine.NewInstantQuery(ctx, p.fanoutStorage, nil, qs, t)
+func (p *PqlEngine) RunAlertQuery(ctx context.Context, qs string, start, end time.Time, interval time.Duration) (pql.Matrix, error) {
+	q, err := p.engine.NewRangeQuery(ctx, p.fanoutStorage, nil, qs, start, end, interval)
 	if err != nil {
 		return nil, err
 	}
@@ -101,16 +100,26 @@ func (p *PqlEngine) RunAlertQuery(ctx context.Context, qs string, t time.Time) (
 		return nil, res.Err
 	}
 
-	switch v := res.Value.(type) {
+	switch typ := res.Value.(type) {
 	case pql.Vector:
-		return v, nil
+		series := make([]pql.Series, 0, len(typ))
+		value := res.Value.(pql.Vector)
+		for _, smpl := range value {
+			series = append(series, pql.Series{
+				Metric: smpl.Metric,
+				Floats: []pql.FPoint{{T: smpl.T, F: smpl.F}},
+			})
+		}
+		return series, nil
 	case pql.Scalar:
-		return pql.Vector{pql.Sample{
-			T:      v.T,
-			F:      v.V,
-			H:      nil,
-			Metric: plabels.Labels{},
-		}}, nil
+		value := res.Value.(pql.Scalar)
+		series := make([]pql.Series, 0, 1)
+		series = append(series, pql.Series{
+			Floats: []pql.FPoint{{T: value.T, F: value.V}},
+		})
+		return series, nil
+	case pql.Matrix:
+		return res.Value.(pql.Matrix), nil
 	default:
 		return nil, fmt.Errorf("rule result is not a vector or scalar")
 	}

pkg/query-service/rules/promRule.go

@@ -3,6 +3,7 @@ package rules
 import (
 	"context"
 	"fmt"
+	"math"
 	"sync"
 	"time"
 
@@ -115,7 +116,9 @@ func (r *PromRule) targetVal() float64 {
 		return 0
 	}
 
-	return *r.ruleCondition.Target
+	unitConverter := converter.FromUnit(converter.Unit(r.ruleCondition.TargetUnit))
+	value := unitConverter.Convert(converter.Value{F: *r.ruleCondition.Target, U: converter.Unit(r.ruleCondition.TargetUnit)}, converter.Unit(r.Unit()))
+	return value.F
 }
 
 func (r *PromRule) Type() RuleType {
@@ -322,14 +325,7 @@ func (r *PromRule) getPqlQuery() (string, error) {
 				if query == "" {
 					return query, fmt.Errorf("a promquery needs to be set for this rule to function")
 				}
-				if r.ruleCondition.Target != nil && r.ruleCondition.CompareOp != CompareOpNone {
-					unitConverter := converter.FromUnit(converter.Unit(r.ruleCondition.TargetUnit))
-					value := unitConverter.Convert(converter.Value{F: *r.ruleCondition.Target, U: converter.Unit(r.ruleCondition.TargetUnit)}, converter.Unit(r.Unit()))
-					query = fmt.Sprintf("(%s) %s %f", query, ResolveCompareOp(r.ruleCondition.CompareOp), value.F)
-					return query, nil
-				} else {
-					return query, nil
-				}
+				return query, nil
 			}
 		}
 	}
@@ -337,8 +333,26 @@ func (r *PromRule) getPqlQuery() (string, error) {
 	return "", fmt.Errorf("invalid promql rule query")
 }
 
+func (r *PromRule) matchType() MatchType {
+	if r.ruleCondition == nil {
+		return AtleastOnce
+	}
+	return r.ruleCondition.MatchType
+}
+
+func (r *PromRule) compareOp() CompareOp {
+	if r.ruleCondition == nil {
+		return ValueIsEq
+	}
+	return r.ruleCondition.CompareOp
+}
+
 func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (interface{}, error) {
 
+	start := ts.Add(-r.evalWindow)
+	end := ts
+	interval := 60 * time.Second // TODO(srikanthccv): this should be configurable
+
 	valueFormatter := formatter.FromUnit(r.Unit())
 
 	q, err := r.getPqlQuery()
@@ -346,7 +360,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (
 		return nil, err
 	}
 	zap.S().Info("rule:", r.Name(), "\t evaluating promql query: ", q)
-	res, err := queriers.PqlEngine.RunAlertQuery(ctx, q, ts)
+	res, err := queriers.PqlEngine.RunAlertQuery(ctx, q, start, end, interval)
 	if err != nil {
 		r.SetHealth(HealthBad)
 		r.SetLastError(err)
@@ -360,16 +374,25 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (
 
 	var alerts = make(map[uint64]*Alert, len(res))
 
-	for _, smpl := range res {
-		l := make(map[string]string, len(smpl.Metric))
-		for _, lbl := range smpl.Metric {
+	for _, series := range res {
+		l := make(map[string]string, len(series.Metric))
+		for _, lbl := range series.Metric {
 			l[lbl.Name] = lbl.Value
 		}
 
+		if len(series.Floats) == 0 {
+			continue
+		}
+
+		alertSmpl, shouldAlert := r.shouldAlert(series)
+		if !shouldAlert {
+			continue
+		}
+
 		thresholdFormatter := formatter.FromUnit(r.ruleCondition.TargetUnit)
 		threshold := thresholdFormatter.Format(r.targetVal(), r.ruleCondition.TargetUnit)
 
-		tmplData := AlertTemplateData(l, valueFormatter.Format(smpl.F, r.Unit()), threshold)
+		tmplData := AlertTemplateData(l, valueFormatter.Format(alertSmpl.F, r.Unit()), threshold)
 		// Inject some convenience variables that are easier to remember for users
 		// who are not used to Go's templating system.
 		defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}"
@@ -392,7 +415,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (
 			return result
 		}
 
-		lb := plabels.NewBuilder(smpl.Metric).Del(plabels.MetricName)
+		lb := plabels.NewBuilder(alertSmpl.Metric).Del(plabels.MetricName)
 
 		for _, l := range r.labels {
 			lb.Set(l.Name, expand(l.Value))
@@ -425,7 +448,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (
 			Annotations:  annotations,
 			ActiveAt:     ts,
 			State:        StatePending,
-			Value:        smpl.F,
+			Value:        alertSmpl.F,
 			GeneratorURL: r.GeneratorURL(),
 			Receivers:    r.preferredChannels,
 		}
@@ -473,6 +496,137 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (
 	return len(r.active), nil
 }
 
+func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
+	var alertSmpl pql.Sample
+	var shouldAlert bool
+	switch r.matchType() {
+	case AtleastOnce:
+		// If any sample matches the condition, the rule is firing.
+		if r.compareOp() == ValueIsAbove {
+			for _, smpl := range series.Floats {
+				if smpl.F > r.targetVal() {
+					alertSmpl = pql.Sample{F: smpl.F, T: smpl.T, Metric: series.Metric}
+					shouldAlert = true
+					break
+				}
+			}
+		} else if r.compareOp() == ValueIsBelow {
+			for _, smpl := range series.Floats {
+				if smpl.F < r.targetVal() {
+					alertSmpl = pql.Sample{F: smpl.F, T: smpl.T, Metric: series.Metric}
+					shouldAlert = true
+					break
+				}
+			}
+		} else if r.compareOp() == ValueIsEq {
+			for _, smpl := range series.Floats {
+				if smpl.F == r.targetVal() {
+					alertSmpl = pql.Sample{F: smpl.F, T: smpl.T, Metric: series.Metric}
+					shouldAlert = true
+					break
+				}
+			}
+		} else if r.compareOp() == ValueIsNotEq {
+			for _, smpl := range series.Floats {
+				if smpl.F != r.targetVal() {
+					alertSmpl = pql.Sample{F: smpl.F, T: smpl.T, Metric: series.Metric}
+					shouldAlert = true
+					break
+				}
+			}
+		}
+	case AllTheTimes:
+		// If all samples match the condition, the rule is firing.
+		shouldAlert = true
+		alertSmpl = pql.Sample{F: r.targetVal(), Metric: series.Metric}
+		if r.compareOp() == ValueIsAbove {
+			for _, smpl := range series.Floats {
+				if smpl.F <= r.targetVal() {
+					shouldAlert = false
+					break
+				}
+			}
+		} else if r.compareOp() == ValueIsBelow {
+			for _, smpl := range series.Floats {
+				if smpl.F >= r.targetVal() {
+					shouldAlert = false
+					break
+				}
+			}
+		} else if r.compareOp() == ValueIsEq {
+			for _, smpl := range series.Floats {
+				if smpl.F != r.targetVal() {
+					shouldAlert = false
+					break
+				}
+			}
+		} else if r.compareOp() == ValueIsNotEq {
+			for _, smpl := range series.Floats {
+				if smpl.F == r.targetVal() {
+					shouldAlert = false
+					break
+				}
+			}
+		}
+	case OnAverage:
+		// If the average of all samples matches the condition, the rule is firing.
+		var sum float64
+		for _, smpl := range series.Floats {
+			if math.IsNaN(smpl.F) {
+				continue
+			}
+			sum += smpl.F
+		}
+		avg := sum / float64(len(series.Floats))
+		alertSmpl = pql.Sample{F: avg, Metric: series.Metric}
+		if r.compareOp() == ValueIsAbove {
+			if avg > r.targetVal() {
+				shouldAlert = true
+			}
+		} else if r.compareOp() == ValueIsBelow {
+			if avg < r.targetVal() {
+				shouldAlert = true
+			}
+		} else if r.compareOp() == ValueIsEq {
+			if avg == r.targetVal() {
+				shouldAlert = true
+			}
+		} else if r.compareOp() == ValueIsNotEq {
+			if avg != r.targetVal() {
+				shouldAlert = true
+			}
+		}
+	case InTotal:
+		// If the sum of all samples matches the condition, the rule is firing.
+		var sum float64
+		for _, smpl := range series.Floats {
+			if math.IsNaN(smpl.F) {
+				continue
+			}
+			sum += smpl.F
+		}
+		alertSmpl = pql.Sample{F: sum, Metric: series.Metric}
+		if r.compareOp() == ValueIsAbove {
+			if sum > r.targetVal() {
+				shouldAlert = true
+			}
+		} else if r.compareOp() == ValueIsBelow {
+			if sum < r.targetVal() {
+				shouldAlert = true
+			}
+		} else if r.compareOp() == ValueIsEq {
+			if sum == r.targetVal() {
+				shouldAlert = true
+			}
+		} else if r.compareOp() == ValueIsNotEq {
+			if sum != r.targetVal() {
+				shouldAlert = true
+			}
+		}
+	}
+	return alertSmpl, shouldAlert
+}
+
 func (r *PromRule) String() string {
 
 	ar := PostableRule{

pkg/query-service/rules/promrule_test.go

@@ -0,0 +1,622 @@
+package rules
+
+import (
+	"testing"
+	"time"
+
+	pql "github.com/prometheus/prometheus/promql"
+	"github.com/stretchr/testify/assert"
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+)
+
+type testLogger struct {
+	t *testing.T
+}
+
+func (l testLogger) Log(args ...interface{}) error {
+	l.t.Log(args...)
+	return nil
+}
+
+func TestPromRuleShouldAlert(t *testing.T) {
+	postableRule := PostableRule{
+		Alert:      "Test Rule",
+		AlertType:  "METRIC_BASED_ALERT",
+		RuleType:   RuleTypeProm,
+		EvalWindow: Duration(5 * time.Minute),
+		Frequency:  Duration(1 * time.Minute),
+		RuleCondition: &RuleCondition{
+			CompositeQuery: &v3.CompositeQuery{
+				QueryType: v3.QueryTypePromQL,
+				PromQueries: map[string]*v3.PromQuery{
+					"A": {
+						Query: "dummy_query", // This is not used in the test
+					},
+				},
+			},
+		},
+	}
+
+	cases := []struct {
+		values      pql.Series
+		expectAlert bool
+		compareOp   string
+		matchType   string
+		target      float64
+	}{
+		// Test cases for Equals Always
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "3", // Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "3", // Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "3", // Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "3", // Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		// Test cases for Equals Once
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "3", // Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "3", // Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "3", // Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "3", // Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		// Test cases for Greater Than Always
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "1", // Greater Than
+			matchType:   "2", // Always
+			target:      1.5,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "1", // Greater Than
+			matchType:   "2", // Always
+			target:      4.5,
+		},
+		// Test cases for Greater Than Once
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "1", // Greater Than
+			matchType:   "1", // Once
+			target:      4.5,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 4.0},
+					{F: 4.0},
+					{F: 4.0},
+					{F: 4.0},
+					{F: 4.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "1", // Greater Than
+			matchType:   "1", // Once
+			target:      4.5,
+		},
+		// Test cases for Not Equals Always
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "4", // Not Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 0.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "4", // Not Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "4", // Not Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.0},
+					{F: 0.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "4", // Not Equals
+			matchType:   "2", // Always
+			target:      0.0,
+		},
+		// Test cases for Not Equals Once
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "4", // Not Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+					{F: 0.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "4", // Not Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 0.0},
+					{F: 0.0},
+					{F: 1.0},
+					{F: 0.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "4", // Not Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+					{F: 1.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "4", // Not Equals
+			matchType:   "1", // Once
+			target:      0.0,
+		},
+		// Test cases for Less Than Always
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 1.5},
+					{F: 1.5},
+					{F: 1.5},
+					{F: 1.5},
+					{F: 1.5},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "2", // Less Than
+			matchType:   "2", // Always
+			target:      4,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "2", // Less Than
+			matchType:   "2", // Always
+			target:      4,
+		},
+		// Test cases for Less Than Once
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+					{F: 2.5},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "2", // Less Than
+			matchType:   "1", // Once
+			target:      4,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+					{F: 4.5},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "2", // Less Than
+			matchType:   "1", // Once
+			target:      4,
+		},
+		// Test cases for OnAverage
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "3", // Equals
+			matchType:   "3", // OnAverage
+			target:      6.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "3", // Equals
+			matchType:   "3", // OnAverage
+			target:      4.5,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "4", // Not Equals
+			matchType:   "3", // OnAverage
+			target:      4.5,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "4", // Not Equals
+			matchType:   "3", // OnAverage
+			target:      6.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "1", // Greater Than
+			matchType:   "3", // OnAverage
+			target:      4.5,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "2", // Less Than
+			matchType:   "3", // OnAverage
+			target:      12.0,
+		},
+		// Test cases for InTotal
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "3", // Equals
+			matchType:   "4", // InTotal
+			target:      30.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 4.0},
+					{F: 6.0},
+					{F: 8.0},
+					{F: 2.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "3", // Equals
+			matchType:   "4", // InTotal
+			target:      20.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "4", // Not Equals
+			matchType:   "4", // InTotal
+			target:      9.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "4", // Not Equals
+			matchType:   "4", // InTotal
+			target:      10.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 10.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "1", // Greater Than
+			matchType:   "4", // InTotal
+			target:      10.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 10.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "1", // Greater Than
+			matchType:   "4", // InTotal
+			target:      20.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 10.0},
+				},
+			},
+			expectAlert: true,
+			compareOp:   "2", // Less Than
+			matchType:   "4", // InTotal
+			target:      30.0,
+		},
+		{
+			values: pql.Series{
+				Floats: []pql.FPoint{
+					{F: 10.0},
+					{F: 10.0},
+				},
+			},
+			expectAlert: false,
+			compareOp:   "2", // Less Than
+			matchType:   "4", // InTotal
+			target:      20.0,
+		},
+	}
+
+	for idx, c := range cases {
+		postableRule.RuleCondition.CompareOp = CompareOp(c.compareOp)
+		postableRule.RuleCondition.MatchType = MatchType(c.matchType)
+		postableRule.RuleCondition.Target = &c.target
+
+		rule, err := NewPromRule("69", &postableRule, testLogger{t}, PromRuleOpts{})
+		if err != nil {
+			assert.NoError(t, err)
+		}
+
+		_, shoulAlert := rule.shouldAlert(c.values)
+		assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx)
+	}
+}

pkg/query-service/rules/thresholdRule.go

@@ -1025,7 +1025,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time, queriers *Querie
 		// who are not used to Go's templating system.
 		defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}"
 
-		// utility function to apply go template on labels and annots
+		// utility function to apply go template on labels and annotations
 		expand := func(text string) string {
 
 			tmpl := NewTemplateExpander(