GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-13 21:06:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: use better value for threshold value in alert description (#5844)

Browse Source
This commit is contained in:
Srikanth Chekuri 2024-09-04 18:30:04 +05:30 committed by GitHub
parent 3544ffdcc6
commit 6019b38da5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 351 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pkg/query-service/rules/prom_rule.go
View File

@ -591,6 +591,16 @@ func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
break
}
}
// use min value from the series
if shouldAlert {
var minValue float64 = math.Inf(1)
for _, smpl := range series.Floats {
if smpl.F < minValue {
minValue = smpl.F
}
}
alertSmpl = pql.Sample{F: minValue, Metric: series.Metric}
}
} else if r.compareOp() == ValueIsBelow {
for _, smpl := range series.Floats {
if smpl.F >= r.targetVal() {
@ -598,6 +608,15 @@ func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
break
}
}
if shouldAlert {
var maxValue float64 = math.Inf(-1)
for _, smpl := range series.Floats {
if smpl.F > maxValue {
maxValue = smpl.F
}
}
alertSmpl = pql.Sample{F: maxValue, Metric: series.Metric}
}
} else if r.compareOp() == ValueIsEq {
for _, smpl := range series.Floats {
if smpl.F != r.targetVal() {
@ -612,6 +631,14 @@ func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
break
}
}
if shouldAlert {
for _, smpl := range series.Floats {
if !math.IsInf(smpl.F, 0) && !math.IsNaN(smpl.F) {
alertSmpl = pql.Sample{F: smpl.F, Metric: series.Metric}
break
}
}
}
}
case OnAverage:
// If the average of all samples matches the condition, the rule is firing.

223
pkg/query-service/rules/promrule_test.go
View File

@ -30,11 +30,12 @@ func TestPromRuleShouldAlert(t *testing.T) {
}
cases := []struct {
values pql.Series
expectAlert bool
compareOp string
matchType string
target float64
values pql.Series
expectAlert bool
compareOp string
matchType string
target float64
expectedAlertSample v3.Point
}{
// Test cases for Equals Always
{
@ -47,10 +48,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 0.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "2", // Always
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "2", // Always
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: pql.Series{
@ -108,10 +110,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 0.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: pql.Series{
@ -123,10 +126,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 1.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: pql.Series{
@ -138,10 +142,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 1.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: pql.Series{
@ -169,10 +174,43 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "2", // Always
target: 1.5,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "2", // Always
target: 1.5,
expectedAlertSample: v3.Point{Value: 2.0},
},
{
values: pql.Series{
Floats: []pql.FPoint{
{F: 11.0},
{F: 4.0},
{F: 3.0},
{F: 7.0},
{F: 12.0},
},
},
expectAlert: true,
compareOp: "1", // Above
matchType: "2", // Always
target: 2.0,
expectedAlertSample: v3.Point{Value: 3.0},
},
{
values: pql.Series{
Floats: []pql.FPoint{
{F: 11.0},
{F: 4.0},
{F: 3.0},
{F: 7.0},
{F: 12.0},
},
},
expectAlert: true,
compareOp: "2", // Below
matchType: "2", // Always
target: 13.0,
expectedAlertSample: v3.Point{Value: 12.0},
},
{
values: pql.Series{
@ -200,10 +238,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "1", // Once
target: 4.5,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "1", // Once
target: 4.5,
expectedAlertSample: v3.Point{Value: 10.0},
},
{
values: pql.Series{
@ -261,10 +300,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 1.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "2", // Always
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "2", // Always
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
{
values: pql.Series{
@ -292,10 +332,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 0.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
{
values: pql.Series{
@ -322,10 +363,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 1.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
{
values: pql.Series{
@ -337,10 +379,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 1.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
// Test cases for Less Than Always
{
@ -353,10 +396,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 1.5},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "2", // Always
target: 4,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "2", // Always
target: 4,
expectedAlertSample: v3.Point{Value: 1.5},
},
{
values: pql.Series{
@ -384,10 +428,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.5},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "1", // Once
target: 4,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "1", // Once
target: 4,
expectedAlertSample: v3.Point{Value: 2.5},
},
{
values: pql.Series{
@ -415,10 +460,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "3", // OnAverage
target: 6.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "3", // OnAverage
target: 6.0,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: pql.Series{
@ -445,10 +491,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "3", // OnAverage
target: 4.5,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "3", // OnAverage
target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: pql.Series{
@ -475,10 +522,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "3", // OnAverage
target: 4.5,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "3", // OnAverage
target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: pql.Series{
@ -490,10 +538,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "3", // OnAverage
target: 12.0,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "3", // OnAverage
target: 12.0,
expectedAlertSample: v3.Point{Value: 6.0},
},
// Test cases for InTotal
{
@ -506,10 +555,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 2.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "4", // InTotal
target: 30.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "4", // InTotal
target: 30.0,
expectedAlertSample: v3.Point{Value: 30.0},
},
{
values: pql.Series{
@ -532,10 +582,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 10.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "4", // InTotal
target: 9.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "4", // InTotal
target: 9.0,
expectedAlertSample: v3.Point{Value: 10.0},
},
{
values: pql.Series{
@ -555,10 +606,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 10.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "4", // InTotal
target: 10.0,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "4", // InTotal
target: 10.0,
expectedAlertSample: v3.Point{Value: 20.0},
},
{
values: pql.Series{
@ -579,10 +631,11 @@ func TestPromRuleShouldAlert(t *testing.T) {
{F: 10.0},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "4", // InTotal
target: 30.0,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "4", // InTotal
target: 30.0,
expectedAlertSample: v3.Point{Value: 20.0},
},
{
values: pql.Series{

28
pkg/query-service/rules/threshold_rule.go
View File

@ -1205,6 +1205,16 @@ func (r *ThresholdRule) shouldAlert(series v3.Series) (Sample, bool) {
break
}
}
// use min value from the series
if shouldAlert {
var minValue float64 = math.Inf(1)
for _, smpl := range series.Points {
if smpl.Value < minValue {
minValue = smpl.Value
}
}
alertSmpl = Sample{Point: Point{V: minValue}, Metric: lblsNormalized, MetricOrig: lbls}
}
} else if r.compareOp() == ValueIsBelow {
for _, smpl := range series.Points {
if smpl.Value >= r.targetVal() {
@ -1212,6 +1222,15 @@ func (r *ThresholdRule) shouldAlert(series v3.Series) (Sample, bool) {
break
}
}
if shouldAlert {
var maxValue float64 = math.Inf(-1)
for _, smpl := range series.Points {
if smpl.Value > maxValue {
maxValue = smpl.Value
}
}
alertSmpl = Sample{Point: Point{V: maxValue}, Metric: lblsNormalized, MetricOrig: lbls}
}
} else if r.compareOp() == ValueIsEq {
for _, smpl := range series.Points {
if smpl.Value != r.targetVal() {
@ -1226,6 +1245,15 @@ func (r *ThresholdRule) shouldAlert(series v3.Series) (Sample, bool) {
break
}
}
// use any non-inf or nan value from the series
if shouldAlert {
for _, smpl := range series.Points {
if !math.IsInf(smpl.Value, 0) && !math.IsNaN(smpl.Value) {
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lblsNormalized, MetricOrig: lbls}
break
}
}
}
}
case OnAverage:
// If the average of all samples matches the condition, the rule is firing.

244
pkg/query-service/rules/threshold_rule_test.go
View File

@ -42,11 +42,12 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
}
cases := []struct {
values v3.Series
expectAlert bool
compareOp string
matchType string
target float64
values v3.Series
expectAlert bool
compareOp string
matchType string
target float64
expectedAlertSample v3.Point
}{
// Test cases for Equals Always
{
@ -59,10 +60,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 0.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "2", // Always
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "2", // Always
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: v3.Series{
@ -120,10 +122,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 0.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: v3.Series{
@ -135,10 +138,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 1.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: v3.Series{
@ -150,10 +154,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 1.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
},
{
values: v3.Series{
@ -181,10 +186,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "2", // Always
target: 1.5,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "2", // Always
target: 1.5,
expectedAlertSample: v3.Point{Value: 2.0},
},
{
values: v3.Series{
@ -212,10 +218,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "1", // Once
target: 4.5,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "1", // Once
target: 4.5,
expectedAlertSample: v3.Point{Value: 10.0},
},
{
values: v3.Series{
@ -273,10 +280,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 1.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "2", // Always
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "2", // Always
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
{
values: v3.Series{
@ -304,10 +312,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 0.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
{
values: v3.Series{
@ -334,10 +343,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 1.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
{
values: v3.Series{
@ -349,10 +359,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 1.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "1", // Once
target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
},
// Test cases for Less Than Always
{
@ -365,10 +376,27 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 1.5},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "2", // Always
target: 4,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "2", // Always
target: 4,
expectedAlertSample: v3.Point{Value: 1.5},
},
{
values: v3.Series{
Points: []v3.Point{
{Value: 1.5},
{Value: 2.5},
{Value: 1.5},
{Value: 3.5},
{Value: 1.5},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "2", // Always
target: 4,
expectedAlertSample: v3.Point{Value: 3.5},
},
{
values: v3.Series{
@ -396,10 +424,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.5},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "1", // Once
target: 4,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "1", // Once
target: 4,
expectedAlertSample: v3.Point{Value: 2.5},
},
{
values: v3.Series{
@ -427,10 +456,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "3", // OnAverage
target: 6.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "3", // OnAverage
target: 6.0,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: v3.Series{
@ -457,10 +487,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "3", // OnAverage
target: 4.5,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "3", // OnAverage
target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: v3.Series{
@ -487,10 +518,43 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "3", // OnAverage
target: 4.5,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "3", // OnAverage
target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: v3.Series{
Points: []v3.Point{
{Value: 11.0},
{Value: 4.0},
{Value: 3.0},
{Value: 7.0},
{Value: 12.0},
},
},
expectAlert: true,
compareOp: "1", // Above
matchType: "2", // Always
target: 2.0,
expectedAlertSample: v3.Point{Value: 3.0},
},
{
values: v3.Series{
Points: []v3.Point{
{Value: 11.0},
{Value: 4.0},
{Value: 3.0},
{Value: 7.0},
{Value: 12.0},
},
},
expectAlert: true,
compareOp: "2", // Below
matchType: "2", // Always
target: 13.0,
expectedAlertSample: v3.Point{Value: 12.0},
},
{
values: v3.Series{
@ -502,10 +566,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "3", // OnAverage
target: 12.0,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "3", // OnAverage
target: 12.0,
expectedAlertSample: v3.Point{Value: 6.0},
},
// Test cases for InTotal
{
@ -518,10 +583,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 2.0},
},
},
expectAlert: true,
compareOp: "3", // Equals
matchType: "4", // InTotal
target: 30.0,
expectAlert: true,
compareOp: "3", // Equals
matchType: "4", // InTotal
target: 30.0,
expectedAlertSample: v3.Point{Value: 30.0},
},
{
values: v3.Series{
@ -544,10 +610,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 10.0},
},
},
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "4", // InTotal
target: 9.0,
expectAlert: true,
compareOp: "4", // Not Equals
matchType: "4", // InTotal
target: 9.0,
expectedAlertSample: v3.Point{Value: 10.0},
},
{
values: v3.Series{
@ -567,10 +634,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 10.0},
},
},
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "4", // InTotal
target: 10.0,
expectAlert: true,
compareOp: "1", // Greater Than
matchType: "4", // InTotal
target: 10.0,
expectedAlertSample: v3.Point{Value: 20.0},
},
{
values: v3.Series{
@ -591,10 +659,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
{Value: 10.0},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "4", // InTotal
target: 30.0,
expectAlert: true,
compareOp: "2", // Less Than
matchType: "4", // InTotal
target: 30.0,
expectedAlertSample: v3.Point{Value: 20.0},
},
{
values: v3.Series{
@ -626,8 +695,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
values.Points[i].Timestamp = time.Now().UnixMilli()
}
_, shoulAlert := rule.shouldAlert(c.values)
smpl, shoulAlert := rule.shouldAlert(c.values)
assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx)
if shoulAlert {
assert.Equal(t, c.expectedAlertSample.Value, smpl.V, "Test case %d", idx)
}
}
}