diff --git a/frontend/public/locales/en-GB/alerts.json b/frontend/public/locales/en-GB/alerts.json index 5b102e147d..fb360e579b 100644 --- a/frontend/public/locales/en-GB/alerts.json +++ b/frontend/public/locales/en-GB/alerts.json @@ -111,5 +111,7 @@ "exceptions_based_alert": "Exceptions-based Alert", "exceptions_based_alert_desc": "Send a notification when a condition occurs in the exceptions data.", "field_unit": "Threshold unit", + "text_alert_on_absent": "Send a notification if data is missing for", + "text_for": "minutes", "selected_query_placeholder": "Select query" } diff --git a/frontend/public/locales/en/alerts.json b/frontend/public/locales/en/alerts.json index 455ade61e3..0349568c70 100644 --- a/frontend/public/locales/en/alerts.json +++ b/frontend/public/locales/en/alerts.json @@ -111,5 +111,7 @@ "exceptions_based_alert": "Exceptions-based Alert", "exceptions_based_alert_desc": "Send a notification when a condition occurs in the exceptions data.", "field_unit": "Threshold unit", + "text_alert_on_absent": "Send a notification if data is missing for", + "text_for": "minutes", "selected_query_placeholder": "Select query" } diff --git a/frontend/src/container/FormAlertRules/RuleOptions.tsx b/frontend/src/container/FormAlertRules/RuleOptions.tsx index e4ae376c73..88e7c83979 100644 --- a/frontend/src/container/FormAlertRules/RuleOptions.tsx +++ b/frontend/src/container/FormAlertRules/RuleOptions.tsx @@ -1,4 +1,5 @@ import { + Checkbox, Form, InputNumber, InputNumberProps, @@ -213,28 +214,66 @@ function RuleOptions({ ? renderPromRuleOptions() : renderThresholdRuleOpts()} - - - e.currentTarget.blur()} - /> - + + + + e.currentTarget.blur()} + /> + - - + + + + + { + setAlertDef({ + ...alertDef, + condition: { + ...alertDef.condition, + alertOnAbsent: e.target.checked, + }, + }); + }} + /> + + {t('text_alert_on_absent')} + + + { + setAlertDef({ + ...alertDef, + condition: { + ...alertDef.condition, + absentFor: Number(value) || 0, + }, + }); + }} + type="number" + onWheel={(e): void => e.currentTarget.blur()} + /> + + {t('text_for')} + diff --git a/frontend/src/types/api/alerts/def.ts b/frontend/src/types/api/alerts/def.ts index 1d40801229..af3a4bc912 100644 --- a/frontend/src/types/api/alerts/def.ts +++ b/frontend/src/types/api/alerts/def.ts @@ -32,6 +32,8 @@ export interface RuleCondition { matchType?: string; targetUnit?: string; selectedQueryName?: string; + alertOnAbsent?: boolean | undefined; + absentFor?: number | undefined; } export interface Labels { diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index b820a62c58..6181a66ea8 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -224,7 +224,8 @@ var TimeoutExcludedRoutes = map[string]bool{ // alert related constants const ( // AlertHelpPage is used in case default alert repo url is not set - AlertHelpPage = "https://signoz.io/docs/userguide/alerts-management/#generator-url" + AlertHelpPage = "https://signoz.io/docs/userguide/alerts-management/#generator-url" + AlertTimeFormat = "2006-01-02 15:04:05" ) func GetOrDefaultEnv(key string, fallback string) string { diff --git a/pkg/query-service/rules/alerting.go b/pkg/query-service/rules/alerting.go index 623d5dea21..b2ee0b53d0 100644 --- a/pkg/query-service/rules/alerting.go +++ b/pkg/query-service/rules/alerting.go @@ -142,9 +142,11 @@ type RuleCondition struct { CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"` CompareOp CompareOp `yaml:"op,omitempty" json:"op,omitempty"` Target *float64 `yaml:"target,omitempty" json:"target,omitempty"` - MatchType `json:"matchType,omitempty"` - TargetUnit string `json:"targetUnit,omitempty"` - SelectedQuery string `json:"selectedQueryName,omitempty"` + AlertOnAbsent bool `yaml:"alertOnAbsent,omitempty" json:"alertOnAbsent,omitempty"` + AbsentFor time.Duration `yaml:"absentFor,omitempty" json:"absentFor,omitempty"` + MatchType MatchType `json:"matchType,omitempty"` + TargetUnit string `json:"targetUnit,omitempty"` + SelectedQuery string `json:"selectedQueryName,omitempty"` } func (rc *RuleCondition) IsValid() bool { diff --git a/pkg/query-service/rules/resultTypes.go b/pkg/query-service/rules/resultTypes.go index 78474526bd..5f39208b01 100644 --- a/pkg/query-service/rules/resultTypes.go +++ b/pkg/query-service/rules/resultTypes.go @@ -20,6 +20,8 @@ type Sample struct { // Label keys as-is from the result query. // The original labels are used to prepare the related{logs, traces} link in alert notification MetricOrig labels.Labels + + IsMissing bool } func (s Sample) String() string { diff --git a/pkg/query-service/rules/thresholdRule.go b/pkg/query-service/rules/thresholdRule.go index e62897c996..0fdb3745ca 100644 --- a/pkg/query-service/rules/thresholdRule.go +++ b/pkg/query-service/rules/thresholdRule.go @@ -71,7 +71,9 @@ type ThresholdRule struct { temporalityMap map[string]map[v3.Temporality]bool opts ThresholdRuleOpts - typ string + + lastTimestampWithDatapoints time.Time + typ string } type ThresholdRuleOpts struct { @@ -531,6 +533,7 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer if err := rows.Scan(vars...); err != nil { return nil, err } + r.lastTimestampWithDatapoints = time.Now() sample := Sample{} // Why do we maintain two labels sets? Alertmanager requires @@ -555,8 +558,8 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer if colName == "ts" || colName == "interval" { sample.Point.T = timval.Unix() } else { - lbls.Set(colName, timval.Format("2006-01-02 15:04:05")) - lblsOrig.Set(columnNames[i], timval.Format("2006-01-02 15:04:05")) + lbls.Set(colName, timval.Format(constants.AlertTimeFormat)) + lblsOrig.Set(columnNames[i], timval.Format(constants.AlertTimeFormat)) } case *float64: @@ -709,6 +712,20 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer zap.S().Debugf("ruleid:", r.ID(), "\t resultmap(potential alerts):", len(resultMap)) + // if the data is missing for `For` duration then we should send alert + if r.ruleCondition.AlertOnAbsent && r.lastTimestampWithDatapoints.Add(r.Condition().AbsentFor).Before(time.Now()) { + zap.S().Debugf("ruleid:", r.ID(), "\t msg: no data found for rule condition") + lbls := labels.NewBuilder(labels.Labels{}) + if !r.lastTimestampWithDatapoints.IsZero() { + lbls.Set("lastSeen", r.lastTimestampWithDatapoints.Format(constants.AlertTimeFormat)) + } + result = append(result, Sample{ + Metric: lbls.Labels(), + IsMissing: true, + }) + return result, nil + } + for _, sample := range resultMap { // check alert rule condition before dumping results, if sendUnmatchedResults // is set then add results irrespective of condition @@ -1177,6 +1194,11 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time, queriers *Querie annotations := make(labels.Labels, 0, len(r.annotations)) for _, a := range r.annotations { + if smpl.IsMissing { + if a.Name == labels.AlertDescriptionLabel || a.Name == labels.AlertSummaryLabel { + a.Value = labels.AlertMissingData + } + } annotations = append(annotations, labels.Label{Name: normalizeLabelName(a.Name), Value: expand(a.Value)}) } diff --git a/pkg/query-service/utils/labels/labels.go b/pkg/query-service/utils/labels/labels.go index 2b289bbe5a..2e0041aafc 100644 --- a/pkg/query-service/utils/labels/labels.go +++ b/pkg/query-service/utils/labels/labels.go @@ -23,8 +23,11 @@ const ( AlertRuleIdLabel = "ruleId" RuleSourceLabel = "ruleSource" - RuleThresholdLabel = "threshold" - AlertSummaryLabel = "summary" + RuleThresholdLabel = "threshold" + AlertSummaryLabel = "summary" + AlertDescriptionLabel = "description" + + AlertMissingData = "Missing data" ) // Label is a key/value pair of strings.