GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-15 04:45:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: use better value for threshold value in alert description (#5844)

Browse Source
This commit is contained in:
Srikanth Chekuri 2024-09-04 18:30:04 +05:30 committed by GitHub
parent 3544ffdcc6
commit 6019b38da5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 351 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pkg/query-service/rules/prom_rule.go
View File

@ -591,6 +591,16 @@ func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
break break
} }
} }
// use min value from the series
if shouldAlert {
var minValue float64 = math.Inf(1)
for _, smpl := range series.Floats {
if smpl.F < minValue {
minValue = smpl.F
}
}
alertSmpl = pql.Sample{F: minValue, Metric: series.Metric}
}
} else if r.compareOp() == ValueIsBelow { } else if r.compareOp() == ValueIsBelow {
for _, smpl := range series.Floats { for _, smpl := range series.Floats {
if smpl.F >= r.targetVal() { if smpl.F >= r.targetVal() {
@ -598,6 +608,15 @@ func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
break break
} }
} }
if shouldAlert {
var maxValue float64 = math.Inf(-1)
for _, smpl := range series.Floats {
if smpl.F > maxValue {
maxValue = smpl.F
}
}
alertSmpl = pql.Sample{F: maxValue, Metric: series.Metric}
}
} else if r.compareOp() == ValueIsEq { } else if r.compareOp() == ValueIsEq {
for _, smpl := range series.Floats { for _, smpl := range series.Floats {
if smpl.F != r.targetVal() { if smpl.F != r.targetVal() {
@ -612,6 +631,14 @@ func (r *PromRule) shouldAlert(series pql.Series) (pql.Sample, bool) {
break break
} }
} }
if shouldAlert {
for _, smpl := range series.Floats {
if !math.IsInf(smpl.F, 0) && !math.IsNaN(smpl.F) {
alertSmpl = pql.Sample{F: smpl.F, Metric: series.Metric}
break
}
}
}
} }
case OnAverage: case OnAverage:
// If the average of all samples matches the condition, the rule is firing. // If the average of all samples matches the condition, the rule is firing.

53
pkg/query-service/rules/promrule_test.go
View File

@ -35,6 +35,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp string compareOp string
matchType string matchType string
target float64 target float64
expectedAlertSample v3.Point
}{ }{
// Test cases for Equals Always // Test cases for Equals Always
{ {
@ -51,6 +52,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "2", // Always matchType: "2", // Always
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -112,6 +114,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -127,6 +130,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -142,6 +146,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -173,6 +178,39 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "2", // Always matchType: "2", // Always
target: 1.5, target: 1.5,
expectedAlertSample: v3.Point{Value: 2.0},
},
{
values: pql.Series{
Floats: []pql.FPoint{
{F: 11.0},
{F: 4.0},
{F: 3.0},
{F: 7.0},
{F: 12.0},
},
},
expectAlert: true,
compareOp: "1", // Above
matchType: "2", // Always
target: 2.0,
expectedAlertSample: v3.Point{Value: 3.0},
},
{
values: pql.Series{
Floats: []pql.FPoint{
{F: 11.0},
{F: 4.0},
{F: 3.0},
{F: 7.0},
{F: 12.0},
},
},
expectAlert: true,
compareOp: "2", // Below
matchType: "2", // Always
target: 13.0,
expectedAlertSample: v3.Point{Value: 12.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -204,6 +242,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "1", // Once matchType: "1", // Once
target: 4.5, target: 4.5,
expectedAlertSample: v3.Point{Value: 10.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -265,6 +304,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "2", // Always matchType: "2", // Always
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -296,6 +336,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -326,6 +367,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -341,6 +383,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
// Test cases for Less Than Always // Test cases for Less Than Always
{ {
@ -357,6 +400,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "2", // Always matchType: "2", // Always
target: 4, target: 4,
expectedAlertSample: v3.Point{Value: 1.5},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -388,6 +432,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "1", // Once matchType: "1", // Once
target: 4, target: 4,
expectedAlertSample: v3.Point{Value: 2.5},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -419,6 +464,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 6.0, target: 6.0,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -449,6 +495,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 4.5, target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -479,6 +526,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 4.5, target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -494,6 +542,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 12.0, target: 12.0,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
// Test cases for InTotal // Test cases for InTotal
{ {
@ -510,6 +559,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "4", // InTotal matchType: "4", // InTotal
target: 30.0, target: 30.0,
expectedAlertSample: v3.Point{Value: 30.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -536,6 +586,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "4", // InTotal matchType: "4", // InTotal
target: 9.0, target: 9.0,
expectedAlertSample: v3.Point{Value: 10.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -559,6 +610,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "4", // InTotal matchType: "4", // InTotal
target: 10.0, target: 10.0,
expectedAlertSample: v3.Point{Value: 20.0},
}, },
{ {
values: pql.Series{ values: pql.Series{
@ -583,6 +635,7 @@ func TestPromRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "4", // InTotal matchType: "4", // InTotal
target: 30.0, target: 30.0,
expectedAlertSample: v3.Point{Value: 20.0},
}, },
{ {
values: pql.Series{ values: pql.Series{

28
pkg/query-service/rules/threshold_rule.go
View File

@ -1205,6 +1205,16 @@ func (r *ThresholdRule) shouldAlert(series v3.Series) (Sample, bool) {
break break
} }
} }
// use min value from the series
if shouldAlert {
var minValue float64 = math.Inf(1)
for _, smpl := range series.Points {
if smpl.Value < minValue {
minValue = smpl.Value
}
}
alertSmpl = Sample{Point: Point{V: minValue}, Metric: lblsNormalized, MetricOrig: lbls}
}
} else if r.compareOp() == ValueIsBelow { } else if r.compareOp() == ValueIsBelow {
for _, smpl := range series.Points { for _, smpl := range series.Points {
if smpl.Value >= r.targetVal() { if smpl.Value >= r.targetVal() {
@ -1212,6 +1222,15 @@ func (r *ThresholdRule) shouldAlert(series v3.Series) (Sample, bool) {
break break
} }
} }
if shouldAlert {
var maxValue float64 = math.Inf(-1)
for _, smpl := range series.Points {
if smpl.Value > maxValue {
maxValue = smpl.Value
}
}
alertSmpl = Sample{Point: Point{V: maxValue}, Metric: lblsNormalized, MetricOrig: lbls}
}
} else if r.compareOp() == ValueIsEq { } else if r.compareOp() == ValueIsEq {
for _, smpl := range series.Points { for _, smpl := range series.Points {
if smpl.Value != r.targetVal() { if smpl.Value != r.targetVal() {
@ -1226,6 +1245,15 @@ func (r *ThresholdRule) shouldAlert(series v3.Series) (Sample, bool) {
break break
} }
} }
// use any non-inf or nan value from the series
if shouldAlert {
for _, smpl := range series.Points {
if !math.IsInf(smpl.Value, 0) && !math.IsNaN(smpl.Value) {
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lblsNormalized, MetricOrig: lbls}
break
}
}
}
} }
case OnAverage: case OnAverage:
// If the average of all samples matches the condition, the rule is firing. // If the average of all samples matches the condition, the rule is firing.

74
pkg/query-service/rules/threshold_rule_test.go
View File

@ -47,6 +47,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp string compareOp string
matchType string matchType string
target float64 target float64
expectedAlertSample v3.Point
}{ }{
// Test cases for Equals Always // Test cases for Equals Always
{ {
@ -63,6 +64,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "2", // Always matchType: "2", // Always
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -124,6 +126,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -139,6 +142,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -154,6 +158,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 0.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -185,6 +190,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "2", // Always matchType: "2", // Always
target: 1.5, target: 1.5,
expectedAlertSample: v3.Point{Value: 2.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -216,6 +222,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "1", // Once matchType: "1", // Once
target: 4.5, target: 4.5,
expectedAlertSample: v3.Point{Value: 10.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -277,6 +284,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "2", // Always matchType: "2", // Always
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -308,6 +316,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -338,6 +347,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -353,6 +363,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "1", // Once matchType: "1", // Once
target: 0.0, target: 0.0,
expectedAlertSample: v3.Point{Value: 1.0},
}, },
// Test cases for Less Than Always // Test cases for Less Than Always
{ {
@ -369,6 +380,23 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "2", // Always matchType: "2", // Always
target: 4, target: 4,
expectedAlertSample: v3.Point{Value: 1.5},
},
{
values: v3.Series{
Points: []v3.Point{
{Value: 1.5},
{Value: 2.5},
{Value: 1.5},
{Value: 3.5},
{Value: 1.5},
},
},
expectAlert: true,
compareOp: "2", // Less Than
matchType: "2", // Always
target: 4,
expectedAlertSample: v3.Point{Value: 3.5},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -400,6 +428,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "1", // Once matchType: "1", // Once
target: 4, target: 4,
expectedAlertSample: v3.Point{Value: 2.5},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -431,6 +460,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 6.0, target: 6.0,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -461,6 +491,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 4.5, target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -491,6 +522,39 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 4.5, target: 4.5,
expectedAlertSample: v3.Point{Value: 6.0},
},
{
values: v3.Series{
Points: []v3.Point{
{Value: 11.0},
{Value: 4.0},
{Value: 3.0},
{Value: 7.0},
{Value: 12.0},
},
},
expectAlert: true,
compareOp: "1", // Above
matchType: "2", // Always
target: 2.0,
expectedAlertSample: v3.Point{Value: 3.0},
},
{
values: v3.Series{
Points: []v3.Point{
{Value: 11.0},
{Value: 4.0},
{Value: 3.0},
{Value: 7.0},
{Value: 12.0},
},
},
expectAlert: true,
compareOp: "2", // Below
matchType: "2", // Always
target: 13.0,
expectedAlertSample: v3.Point{Value: 12.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -506,6 +570,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "3", // OnAverage matchType: "3", // OnAverage
target: 12.0, target: 12.0,
expectedAlertSample: v3.Point{Value: 6.0},
}, },
// Test cases for InTotal // Test cases for InTotal
{ {
@ -522,6 +587,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "3", // Equals compareOp: "3", // Equals
matchType: "4", // InTotal matchType: "4", // InTotal
target: 30.0, target: 30.0,
expectedAlertSample: v3.Point{Value: 30.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -548,6 +614,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "4", // Not Equals compareOp: "4", // Not Equals
matchType: "4", // InTotal matchType: "4", // InTotal
target: 9.0, target: 9.0,
expectedAlertSample: v3.Point{Value: 10.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -571,6 +638,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "1", // Greater Than compareOp: "1", // Greater Than
matchType: "4", // InTotal matchType: "4", // InTotal
target: 10.0, target: 10.0,
expectedAlertSample: v3.Point{Value: 20.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -595,6 +663,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
compareOp: "2", // Less Than compareOp: "2", // Less Than
matchType: "4", // InTotal matchType: "4", // InTotal
target: 30.0, target: 30.0,
expectedAlertSample: v3.Point{Value: 20.0},
}, },
{ {
values: v3.Series{ values: v3.Series{
@ -626,8 +695,11 @@ func TestThresholdRuleShouldAlert(t *testing.T) {
values.Points[i].Timestamp = time.Now().UnixMilli() values.Points[i].Timestamp = time.Now().UnixMilli()
} }
_, shoulAlert := rule.shouldAlert(c.values) smpl, shoulAlert := rule.shouldAlert(c.values)
assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx) assert.Equal(t, c.expectAlert, shoulAlert, "Test case %d", idx)
if shoulAlert {
assert.Equal(t, c.expectedAlertSample.Value, smpl.V, "Test case %d", idx)
}
} }
} }