feat: add support for alerting on absent metric (#3245)

frontend/public/locales/en-GB/alerts.json

@@ -111,5 +111,7 @@
 	"exceptions_based_alert": "Exceptions-based Alert",
 	"exceptions_based_alert_desc": "Send a notification when a condition occurs in the exceptions data.",
 	"field_unit": "Threshold unit",
+	"text_alert_on_absent": "Send a notification if data is missing for",
+	"text_for": "minutes",
 	"selected_query_placeholder": "Select query"
 }

frontend/public/locales/en/alerts.json

@@ -111,5 +111,7 @@
 	"exceptions_based_alert": "Exceptions-based Alert",
 	"exceptions_based_alert_desc": "Send a notification when a condition occurs in the exceptions data.",
 	"field_unit": "Threshold unit",
+	"text_alert_on_absent": "Send a notification if data is missing for",
+	"text_for": "minutes",
 	"selected_query_placeholder": "Select query"
 }

frontend/src/container/FormAlertRules/RuleOptions.tsx

@@ -1,4 +1,5 @@
 import {
+	Checkbox,
 	Form,
 	InputNumber,
 	InputNumberProps,
@@ -213,6 +214,7 @@ function RuleOptions({
 					? renderPromRuleOptions()
 					: renderThresholdRuleOpts()}
 
+				<Space direction="vertical" size="large">
 					<Space direction="horizontal" align="center">
 						<Form.Item noStyle name={['condition', 'target']}>
 							<InputNumber
@@ -236,6 +238,43 @@ function RuleOptions({
 							/>
 						</Form.Item>
 					</Space>
+					<Space direction="horizontal" align="center">
+						<Form.Item noStyle name={['condition', 'alertOnAbsent']}>
+							<Checkbox
+								checked={alertDef?.condition?.alertOnAbsent}
+								onChange={(e): void => {
+									setAlertDef({
+										...alertDef,
+										condition: {
+											...alertDef.condition,
+											alertOnAbsent: e.target.checked,
+										},
+									});
+								}}
+							/>
+						</Form.Item>
+						<Typography.Text>{t('text_alert_on_absent')}</Typography.Text>
+
+						<Form.Item noStyle name={['condition', 'absentFor']}>
+							<InputNumber
+								min={1}
+								value={alertDef?.condition?.absentFor}
+								onChange={(value): void => {
+									setAlertDef({
+										...alertDef,
+										condition: {
+											...alertDef.condition,
+											absentFor: Number(value) || 0,
+										},
+									});
+								}}
+								type="number"
+								onWheel={(e): void => e.currentTarget.blur()}
+							/>
+						</Form.Item>
+						<Typography.Text>{t('text_for')}</Typography.Text>
+					</Space>
+				</Space>
 			</FormContainer>
 		</>
 	);

frontend/src/types/api/alerts/def.ts

@@ -32,6 +32,8 @@ export interface RuleCondition {
 	matchType?: string;
 	targetUnit?: string;
 	selectedQueryName?: string;
+	alertOnAbsent?: boolean | undefined;
+	absentFor?: number | undefined;
 }
 
 export interface Labels {

pkg/query-service/constants/constants.go

@@ -225,6 +225,7 @@ var TimeoutExcludedRoutes = map[string]bool{
 const (
 	// AlertHelpPage is used in case default alert repo url is not set
 	AlertHelpPage   = "https://signoz.io/docs/userguide/alerts-management/#generator-url"
+	AlertTimeFormat = "2006-01-02 15:04:05"
 )
 
 func GetOrDefaultEnv(key string, fallback string) string {

pkg/query-service/rules/alerting.go

@@ -142,7 +142,9 @@ type RuleCondition struct {
 	CompositeQuery *v3.CompositeQuery `json:"compositeQuery,omitempty" yaml:"compositeQuery,omitempty"`
 	CompareOp      CompareOp          `yaml:"op,omitempty" json:"op,omitempty"`
 	Target         *float64           `yaml:"target,omitempty" json:"target,omitempty"`
-	MatchType      `json:"matchType,omitempty"`
+	AlertOnAbsent  bool               `yaml:"alertOnAbsent,omitempty" json:"alertOnAbsent,omitempty"`
+	AbsentFor      time.Duration      `yaml:"absentFor,omitempty" json:"absentFor,omitempty"`
+	MatchType      MatchType          `json:"matchType,omitempty"`
 	TargetUnit     string             `json:"targetUnit,omitempty"`
 	SelectedQuery  string             `json:"selectedQueryName,omitempty"`
 }

pkg/query-service/rules/resultTypes.go

@@ -20,6 +20,8 @@ type Sample struct {
 	// Label keys as-is from the result query.
 	// The original labels are used to prepare the related{logs, traces} link in alert notification
 	MetricOrig labels.Labels
+
+	IsMissing bool
 }
 
 func (s Sample) String() string {

pkg/query-service/rules/thresholdRule.go

@@ -71,6 +71,8 @@ type ThresholdRule struct {
 	temporalityMap map[string]map[v3.Temporality]bool
 
 	opts ThresholdRuleOpts
+
+	lastTimestampWithDatapoints time.Time
 	typ                         string
 }
 
@@ -531,6 +533,7 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
 		if err := rows.Scan(vars...); err != nil {
 			return nil, err
 		}
+		r.lastTimestampWithDatapoints = time.Now()
 
 		sample := Sample{}
 		// Why do we maintain two labels sets? Alertmanager requires
@@ -555,8 +558,8 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
 				if colName == "ts" || colName == "interval" {
 					sample.Point.T = timval.Unix()
 				} else {
-					lbls.Set(colName, timval.Format("2006-01-02 15:04:05"))
+					lbls.Set(colName, timval.Format(constants.AlertTimeFormat))
-					lblsOrig.Set(columnNames[i], timval.Format("2006-01-02 15:04:05"))
+					lblsOrig.Set(columnNames[i], timval.Format(constants.AlertTimeFormat))
 				}
 
 			case *float64:
@@ -709,6 +712,20 @@ func (r *ThresholdRule) runChQuery(ctx context.Context, db clickhouse.Conn, quer
 
 	zap.S().Debugf("ruleid:", r.ID(), "\t resultmap(potential alerts):", len(resultMap))
 
+	// if the data is missing for `For` duration then we should send alert
+	if r.ruleCondition.AlertOnAbsent && r.lastTimestampWithDatapoints.Add(r.Condition().AbsentFor).Before(time.Now()) {
+		zap.S().Debugf("ruleid:", r.ID(), "\t msg: no data found for rule condition")
+		lbls := labels.NewBuilder(labels.Labels{})
+		if !r.lastTimestampWithDatapoints.IsZero() {
+			lbls.Set("lastSeen", r.lastTimestampWithDatapoints.Format(constants.AlertTimeFormat))
+		}
+		result = append(result, Sample{
+			Metric:    lbls.Labels(),
+			IsMissing: true,
+		})
+		return result, nil
+	}
+
 	for _, sample := range resultMap {
 		// check alert rule condition before dumping results, if sendUnmatchedResults
 		// is set then add results irrespective of condition
@@ -1177,6 +1194,11 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time, queriers *Querie
 
 		annotations := make(labels.Labels, 0, len(r.annotations))
 		for _, a := range r.annotations {
+			if smpl.IsMissing {
+				if a.Name == labels.AlertDescriptionLabel || a.Name == labels.AlertSummaryLabel {
+					a.Value = labels.AlertMissingData
+				}
+			}
 			annotations = append(annotations, labels.Label{Name: normalizeLabelName(a.Name), Value: expand(a.Value)})
 		}
 

pkg/query-service/utils/labels/labels.go

@@ -25,6 +25,9 @@ const (
 
 	RuleThresholdLabel    = "threshold"
 	AlertSummaryLabel     = "summary"
+	AlertDescriptionLabel = "description"
+
+	AlertMissingData = "Missing data"
 )
 
 // Label is a key/value pair of strings.