From 7a4156a3b76afb630057156ba881120796a7db8d Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 16 Aug 2023 14:22:40 +0530 Subject: [PATCH] feat(alerts/query-service): measurement unit support for alerts (#2673) --- go.mod | 2 +- go.sum | 3 +- pkg/query-service/converter/bool.go | 4 + pkg/query-service/converter/converter.go | 137 ++++++++++++++++++ pkg/query-service/converter/data.go | 4 + pkg/query-service/converter/data_rate.go | 4 + .../converter/percent_converter.go | 4 + pkg/query-service/converter/throughput.go | 4 + pkg/query-service/converter/time.go | 4 + pkg/query-service/formatter/bool.go | 4 + pkg/query-service/formatter/data.go | 4 + pkg/query-service/formatter/data_rate.go | 4 + pkg/query-service/formatter/formatter.go | 2 + pkg/query-service/formatter/none.go | 4 + pkg/query-service/formatter/percent.go | 4 + pkg/query-service/formatter/throughput.go | 4 + pkg/query-service/formatter/time.go | 4 + pkg/query-service/model/v3/v3.go | 1 + pkg/query-service/rules/alerting.go | 1 + pkg/query-service/rules/apiParams.go | 2 +- pkg/query-service/rules/promRule.go | 18 ++- pkg/query-service/rules/templates.go | 6 +- pkg/query-service/rules/thresholdRule.go | 32 +++- 23 files changed, 242 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 9a91513386..b5bc300e0b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230523034029-2b7ff773052c github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230517094211-cd3f3f0aea85 github.com/coreos/go-oidc/v3 v3.4.0 - github.com/dustin/go-humanize v1.0.0 + github.com/dustin/go-humanize v1.0.1 github.com/go-co-op/gocron v1.30.1 github.com/go-kit/log v0.2.1 github.com/go-redis/redis/v8 v8.11.5 diff --git a/go.sum b/go.sum index 923ab20a20..23dbda27c2 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,9 @@ github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+ github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 h1:IPrmumsT9t5BS7XcPhgsCTlkWbYg80SEXUzDpReaU6Y= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= diff --git a/pkg/query-service/converter/bool.go b/pkg/query-service/converter/bool.go index 2183c3b2a4..74d6bb78a6 100644 --- a/pkg/query-service/converter/bool.go +++ b/pkg/query-service/converter/bool.go @@ -7,6 +7,10 @@ func NewBoolConverter() Converter { return &boolConverter{} } +func (*boolConverter) Name() string { + return "bool" +} + func (c *boolConverter) Convert(v Value, to Unit) Value { // There is no conversion to be done for bool return Value{ diff --git a/pkg/query-service/converter/converter.go b/pkg/query-service/converter/converter.go index 82f87b8805..44a92b8f50 100644 --- a/pkg/query-service/converter/converter.go +++ b/pkg/query-service/converter/converter.go @@ -11,12 +11,20 @@ type Value struct { // Converter converts values from one unit to another type Converter interface { + // Convert converts the given value to the given unit Convert(v Value, to Unit) Value + + // Name returns the name of the converter + Name() string } // noneConverter is a converter that does not convert type noneConverter struct{} +func (*noneConverter) Name() string { + return "none" +} + func (c *noneConverter) Convert(v Value, to Unit) Value { return v } @@ -51,3 +59,132 @@ func FromUnit(u Unit) Converter { return NoneConverter } } + +func UnitToName(u string) string { + switch u { + case "ns": + return " ns" + case "us": + return " us" + case "ms": + return " ms" + case "s": + return " s" + case "m": + return " minutes" + case "h": + return " hours" + case "d": + return " days" + case "bytes": + return " bytes" + case "decbytes": + return " bytes" + case "bits": + return " bits" + case "decbits": + return " bits" + case "kbytes": + return " KiB" + case "decKbytes": + return " kB" + case "mbytes": + return " MiB" + case "decMbytes": + return " MB" + case "gbytes": + return " GiB" + case "decGbytes": + return " GB" + case "tbytes": + return " TiB" + case "decTbytes": + return " TB" + case "pbytes": + return " PiB" + case "decPbytes": + return " PB" + case "binBps": + return " bytes/sec(IEC)" + case "Bps": + return " bytes/sec(SI)" + case "binbps": + return " bits/sec(IEC)" + case "bps": + return " bits/sec(SI)" + case "KiBs": + return " KiB/sec" + case "Kibits": + return " Kibit/sec" + case "KBs": + return " kB/sec" + case "Kbits": + return " kbit/sec" + case "MiBs": + return " MiB/sec" + case "Mibits": + return " Mibit/sec" + case "MBs": + return " MB/sec" + case "Mbits": + return " Mbit/sec" + case "GiBs": + return " GiB/sec" + case "Gibits": + return " Gibit/sec" + case "GBs": + return " GB/sec" + case "Gbits": + return " Gbit/sec" + case "TiBs": + return " TiB/sec" + case "Tibits": + return " Tibit/sec" + case "TBs": + return " TB/sec" + case "Tbits": + return " Tbit/sec" + case "PiBs": + return " PiB/sec" + case "Pibits": + return " Pibit/sec" + case "PBs": + return " PB/sec" + case "Pbits": + return " Pbit/sec" + case "percent": + return " %" + case "percentunit": + return " %" + case "bool": + return "" + case "bool_yes_no": + return "" + case "bool_true_false": + return "" + case "bool_1_0": + return "" + case "cps": + return " counts/sec (cps)" + case "ops": + return " ops/sec (ops)" + case "reqps": + return " requests/sec (rps)" + case "rps": + return " reads/sec (rps)" + case "wps": + return " writes/sec (wps)" + case "iops": + return " I/O ops/sec (iops)" + case "cpm": + return " counts/min (cpm)" + case "opm": + return " ops/min (opm)" + case "rpm": + return " reads/min (rpm)" + case "wpm": + return " writes/min (wpm)" + default: + return u + } +} diff --git a/pkg/query-service/converter/data.go b/pkg/query-service/converter/data.go index f7252a2bc3..1dfb183fb1 100644 --- a/pkg/query-service/converter/data.go +++ b/pkg/query-service/converter/data.go @@ -54,6 +54,10 @@ func NewDataConverter() Converter { return &dataConverter{} } +func (*dataConverter) Name() string { + return "data" +} + func FromDataUnit(u Unit) float64 { switch u { case "bytes": // base 2 diff --git a/pkg/query-service/converter/data_rate.go b/pkg/query-service/converter/data_rate.go index 3951341733..7b5fcebdca 100644 --- a/pkg/query-service/converter/data_rate.go +++ b/pkg/query-service/converter/data_rate.go @@ -50,6 +50,10 @@ func NewDataRateConverter() Converter { return &dataRateConverter{} } +func (*dataRateConverter) Name() string { + return "data_rate" +} + func FromDataRateUnit(u Unit) float64 { // See https://github.com/SigNoz/signoz/blob/5a81f5f90b34845f5b4b3bdd46acf29d04bf3987/frontend/src/container/NewWidget/RightContainer/dataFormatCategories.ts#L62-L85 switch u { diff --git a/pkg/query-service/converter/percent_converter.go b/pkg/query-service/converter/percent_converter.go index 6597da93aa..166b89b540 100644 --- a/pkg/query-service/converter/percent_converter.go +++ b/pkg/query-service/converter/percent_converter.go @@ -7,6 +7,10 @@ func NewPercentConverter() Converter { return &percentConverter{} } +func (*percentConverter) Name() string { + return "percent" +} + func FromPercentUnit(u Unit) float64 { switch u { case "percent": diff --git a/pkg/query-service/converter/throughput.go b/pkg/query-service/converter/throughput.go index 7c38257b81..9fdb4703db 100644 --- a/pkg/query-service/converter/throughput.go +++ b/pkg/query-service/converter/throughput.go @@ -8,6 +8,10 @@ func NewThroughputConverter() Converter { return &throughputConverter{} } +func (*throughputConverter) Name() string { + return "throughput" +} + func (c *throughputConverter) Convert(v Value, to Unit) Value { // There is no conversion to be done for throughput return Value{ diff --git a/pkg/query-service/converter/time.go b/pkg/query-service/converter/time.go index c4f1f03c8a..8dd458cb6d 100644 --- a/pkg/query-service/converter/time.go +++ b/pkg/query-service/converter/time.go @@ -23,6 +23,10 @@ func NewDurationConverter() Converter { return &durationConverter{} } +func (*durationConverter) Name() string { + return "duration" +} + func FromTimeUnit(u Unit) Duration { switch u { case "ns": diff --git a/pkg/query-service/formatter/bool.go b/pkg/query-service/formatter/bool.go index 7080a2bb04..74952cde0e 100644 --- a/pkg/query-service/formatter/bool.go +++ b/pkg/query-service/formatter/bool.go @@ -8,6 +8,10 @@ func NewBoolFormatter() Formatter { return &boolFormatter{} } +func (*boolFormatter) Name() string { + return "bool" +} + func toBool(value float64) string { if value == 0 { return "false" diff --git a/pkg/query-service/formatter/data.go b/pkg/query-service/formatter/data.go index ecf7e6e64d..7ec9e30124 100644 --- a/pkg/query-service/formatter/data.go +++ b/pkg/query-service/formatter/data.go @@ -14,6 +14,10 @@ func NewDataFormatter() Formatter { return &dataFormatter{} } +func (*dataFormatter) Name() string { + return "data" +} + func (f *dataFormatter) Format(value float64, unit string) string { switch unit { case "bytes": diff --git a/pkg/query-service/formatter/data_rate.go b/pkg/query-service/formatter/data_rate.go index ef1053fc93..4027bf3b84 100644 --- a/pkg/query-service/formatter/data_rate.go +++ b/pkg/query-service/formatter/data_rate.go @@ -14,6 +14,10 @@ func NewDataRateFormatter() Formatter { return &dataRateFormatter{} } +func (*dataRateFormatter) Name() string { + return "data_rate" +} + func (f *dataRateFormatter) Format(value float64, unit string) string { switch unit { case "binBps": diff --git a/pkg/query-service/formatter/formatter.go b/pkg/query-service/formatter/formatter.go index 5c700b53c4..bfb0627b93 100644 --- a/pkg/query-service/formatter/formatter.go +++ b/pkg/query-service/formatter/formatter.go @@ -2,6 +2,8 @@ package formatter type Formatter interface { Format(value float64, unit string) string + + Name() string } var ( diff --git a/pkg/query-service/formatter/none.go b/pkg/query-service/formatter/none.go index 8f2fbab6df..208df98821 100644 --- a/pkg/query-service/formatter/none.go +++ b/pkg/query-service/formatter/none.go @@ -8,6 +8,10 @@ func NewNoneFormatter() Formatter { return &noneFormatter{} } +func (*noneFormatter) Name() string { + return "none" +} + func (f *noneFormatter) Format(value float64, unit string) string { return fmt.Sprintf("%v", value) } diff --git a/pkg/query-service/formatter/percent.go b/pkg/query-service/formatter/percent.go index 699ff4894b..9740737a2e 100644 --- a/pkg/query-service/formatter/percent.go +++ b/pkg/query-service/formatter/percent.go @@ -8,6 +8,10 @@ func NewPercentFormatter() Formatter { return &percentFormatter{} } +func (*percentFormatter) Name() string { + return "percent" +} + func toPercent(value float64, decimals DecimalCount) string { return toFixed(value, decimals) + "%" } diff --git a/pkg/query-service/formatter/throughput.go b/pkg/query-service/formatter/throughput.go index 72f8c1a74a..6bad95e019 100644 --- a/pkg/query-service/formatter/throughput.go +++ b/pkg/query-service/formatter/throughput.go @@ -9,6 +9,10 @@ func NewThroughputFormatter() Formatter { return &throughputFormatter{} } +func (*throughputFormatter) Name() string { + return "throughput" +} + func simpleCountUnit(value float64, decimals *int, symbol string) string { units := []string{"", "K", "M", "B", "T"} scaler := scaledUnits(1000, units, 0) diff --git a/pkg/query-service/formatter/time.go b/pkg/query-service/formatter/time.go index 3d9b04d1b7..8c3d860f06 100644 --- a/pkg/query-service/formatter/time.go +++ b/pkg/query-service/formatter/time.go @@ -12,6 +12,10 @@ func NewDurationFormatter() Formatter { return &durationFormatter{} } +func (*durationFormatter) Name() string { + return "duration" +} + func (f *durationFormatter) Format(value float64, unit string) string { switch unit { case "ns": diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 1fb4cf8e60..0cf4e2ed68 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -373,6 +373,7 @@ type CompositeQuery struct { PromQueries map[string]*PromQuery `json:"promQueries,omitempty"` PanelType PanelType `json:"panelType"` QueryType QueryType `json:"queryType"` + Unit string `json:"unit,omitempty"` } func (c *CompositeQuery) Validate() error { diff --git a/pkg/query-service/rules/alerting.go b/pkg/query-service/rules/alerting.go index 55c2fdfafc..60cb4c9384 100644 --- a/pkg/query-service/rules/alerting.go +++ b/pkg/query-service/rules/alerting.go @@ -143,6 +143,7 @@ type RuleCondition struct { CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"` Target *float64 `yaml:"target,omitempty" json:"target,omitempty"` MatchType `json:"matchType,omitempty"` + TargetUnit string `json:"targetUnit,omitempty"` } func (rc *RuleCondition) IsValid() bool { diff --git a/pkg/query-service/rules/apiParams.go b/pkg/query-service/rules/apiParams.go index 649ba6373d..f33af06e81 100644 --- a/pkg/query-service/rules/apiParams.go +++ b/pkg/query-service/rules/apiParams.go @@ -198,7 +198,7 @@ func testTemplateParsing(rl *PostableRule) (errs []error) { } // Trying to parse templates. - tmplData := AlertTemplateData(make(map[string]string), 0, 0) + tmplData := AlertTemplateData(make(map[string]string), "0", "0") defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}" parseTest := func(text string) error { tmpl := NewTemplateExpander( diff --git a/pkg/query-service/rules/promRule.go b/pkg/query-service/rules/promRule.go index 76afef01b4..ef7177472e 100644 --- a/pkg/query-service/rules/promRule.go +++ b/pkg/query-service/rules/promRule.go @@ -3,6 +3,7 @@ package rules import ( "context" "fmt" + "strconv" "sync" "time" @@ -12,6 +13,8 @@ import ( plabels "github.com/prometheus/prometheus/model/labels" pql "github.com/prometheus/prometheus/promql" + "go.signoz.io/signoz/pkg/query-service/converter" + "go.signoz.io/signoz/pkg/query-service/formatter" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" qslabels "go.signoz.io/signoz/pkg/query-service/utils/labels" "go.signoz.io/signoz/pkg/query-service/utils/times" @@ -255,6 +258,13 @@ func (r *PromRule) ActiveAlerts() []*Alert { return res } +func (r *PromRule) Unit() string { + if r.ruleCondition != nil && r.ruleCondition.CompositeQuery != nil { + return r.ruleCondition.CompositeQuery.Unit + } + return "" +} + // ForEachActiveAlert runs the given function on each alert. // This should be used when you want to use the actual alerts from the ThresholdRule // and not on its copy. @@ -296,7 +306,9 @@ func (r *PromRule) getPqlQuery() (string, error) { return query, fmt.Errorf("a promquery needs to be set for this rule to function") } if r.ruleCondition.Target != nil && r.ruleCondition.CompareOp != CompareOpNone { - query = fmt.Sprintf("%s %s %f", query, ResolveCompareOp(r.ruleCondition.CompareOp), *r.ruleCondition.Target) + unitConverter := converter.FromUnit(converter.Unit(r.ruleCondition.TargetUnit)) + value := unitConverter.Convert(converter.Value{F: *r.ruleCondition.Target, U: converter.Unit(r.ruleCondition.TargetUnit)}, converter.Unit(r.Unit())) + query = fmt.Sprintf("%s %s %f", query, ResolveCompareOp(r.ruleCondition.CompareOp), value.F) return query, nil } else { return query, nil @@ -310,6 +322,8 @@ func (r *PromRule) getPqlQuery() (string, error) { func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (interface{}, error) { + valueFormatter := formatter.FromUnit(r.Unit()) + q, err := r.getPqlQuery() if err != nil { return nil, err @@ -335,7 +349,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) ( l[lbl.Name] = lbl.Value } - tmplData := AlertTemplateData(l, smpl.V, r.targetVal()) + tmplData := AlertTemplateData(l, valueFormatter.Format(smpl.V, r.Unit()), strconv.FormatFloat(r.targetVal(), 'f', 2, 64)+converter.UnitToName(r.ruleCondition.TargetUnit)) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}" diff --git a/pkg/query-service/rules/templates.go b/pkg/query-service/rules/templates.go index 473a5cfa87..9cc49f787d 100644 --- a/pkg/query-service/rules/templates.go +++ b/pkg/query-service/rules/templates.go @@ -201,11 +201,11 @@ func NewTemplateExpander( } // AlertTemplateData returns the interface to be used in expanding the template. -func AlertTemplateData(labels map[string]string, value float64, threshold float64) interface{} { +func AlertTemplateData(labels map[string]string, value string, threshold string) interface{} { return struct { Labels map[string]string - Value float64 - Threshold float64 + Value string + Threshold string }{ Labels: labels, Value: value, diff --git a/pkg/query-service/rules/thresholdRule.go b/pkg/query-service/rules/thresholdRule.go index fbd65d9948..507e6fa289 100644 --- a/pkg/query-service/rules/thresholdRule.go +++ b/pkg/query-service/rules/thresholdRule.go @@ -7,6 +7,7 @@ import ( "math" "reflect" "sort" + "strconv" "sync" "text/template" "time" @@ -14,6 +15,7 @@ import ( "go.uber.org/zap" "github.com/ClickHouse/clickhouse-go/v2" + "go.signoz.io/signoz/pkg/query-service/converter" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" "go.signoz.io/signoz/pkg/query-service/constants" @@ -26,6 +28,7 @@ import ( logsv3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" metricsv3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + "go.signoz.io/signoz/pkg/query-service/formatter" yaml "gopkg.in/yaml.v2" ) @@ -324,6 +327,14 @@ func (r *ThresholdRule) SendAlerts(ctx context.Context, ts time.Time, resendDela }) notifyFunc(ctx, "", alerts...) } + +func (r *ThresholdRule) Unit() string { + if r.ruleCondition != nil && r.ruleCondition.CompositeQuery != nil { + return r.ruleCondition.CompositeQuery.Unit + } + return "" +} + func (r *ThresholdRule) CheckCondition(v float64) bool { if math.IsNaN(v) { @@ -336,16 +347,20 @@ func (r *ThresholdRule) CheckCondition(v float64) bool { return false } - zap.S().Debugf("target:", v, *r.ruleCondition.Target) + unitConverter := converter.FromUnit(converter.Unit(r.ruleCondition.TargetUnit)) + + value := unitConverter.Convert(converter.Value{F: *r.ruleCondition.Target, U: converter.Unit(r.ruleCondition.TargetUnit)}, converter.Unit(r.Unit())) + + zap.S().Debugf("Checking condition for rule: %s, Converter=%s, Value=%f, Target=%f, CompareOp=%s", r.Name(), unitConverter.Name(), v, value.F, r.ruleCondition.CompareOp) switch r.ruleCondition.CompareOp { case ValueIsEq: - return v == *r.ruleCondition.Target + return v == value.F case ValueIsNotEq: - return v != *r.ruleCondition.Target + return v != value.F case ValueIsBelow: - return v < *r.ruleCondition.Target + return v < value.F case ValueIsAbove: - return v > *r.ruleCondition.Target + return v > value.F default: return false } @@ -716,6 +731,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time, ch c func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) (interface{}, error) { + valueFormatter := formatter.FromUnit(r.Unit()) res, err := r.buildAndRunQuery(ctx, ts, queriers.Ch) if err != nil { @@ -737,7 +753,11 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time, queriers *Querie l[lbl.Name] = lbl.Value } - tmplData := AlertTemplateData(l, smpl.V, r.targetVal()) + value := valueFormatter.Format(smpl.V, r.Unit()) + threshold := strconv.FormatFloat(r.targetVal(), 'f', 2, 64) + converter.UnitToName(r.ruleCondition.TargetUnit) + zap.S().Debugf("Alert template data for rule %s: Formatter=%s, Value=%s, Threshold=%s", r.Name(), valueFormatter.Name(), value, threshold) + + tmplData := AlertTemplateData(l, value, threshold) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}"