From 88084af4d4a549df68283759a6869b5e5c000bbf Mon Sep 17 00:00:00 2001
From: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
Date: Wed, 29 Jan 2025 19:45:48 +0530
Subject: [PATCH] chore: add cluster, nodes that are sending incorrect host
 metrics (#6969)

---
 pkg/query-service/app/inframetrics/hosts.go | 49 +++++++++++++++++----
 pkg/query-service/model/infra.go            |  2 +
 2 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/pkg/query-service/app/inframetrics/hosts.go b/pkg/query-service/app/inframetrics/hosts.go
index 81e279140f..5372831d58 100644
--- a/pkg/query-service/app/inframetrics/hosts.go
+++ b/pkg/query-service/app/inframetrics/hosts.go
@@ -15,6 +15,7 @@ import (
 	"go.signoz.io/signoz/pkg/query-service/model"
 	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.signoz.io/signoz/pkg/query-service/postprocess"
+	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 )
 
@@ -332,23 +333,53 @@ func (h *HostsRepo) DidSendHostMetricsData(ctx context.Context, req model.HostLi
 	return count > 0, nil
 }
 
-func (h *HostsRepo) IsSendingK8SAgentMetrics(ctx context.Context, req model.HostListRequest) (bool, error) {
+func (h *HostsRepo) IsSendingK8SAgentMetrics(ctx context.Context, req model.HostListRequest) ([]string, []string, error) {
 	names := []string{}
 	for _, metricName := range metricNamesForHosts {
 		names = append(names, metricName)
 	}
 	namesStr := "'" + strings.Join(names, "','") + "'"
 
+	queryForRecentFingerprints := fmt.Sprintf(`
+	SELECT DISTINCT fingerprint
+	FROM %s.%s
+	WHERE metric_name IN (%s)
+		AND unix_milli >= toUnixTimestamp(now() - INTERVAL 5 MINUTE) * 1000`,
+		constants.SIGNOZ_METRIC_DBNAME, constants.SIGNOZ_SAMPLES_V4_TABLENAME, namesStr)
+
 	query := fmt.Sprintf(`
-	SELECT count()
+	SELECT DISTINCT JSONExtractString(labels, 'k8s_cluster_name') as k8s_cluster_name, JSONExtractString(labels, 'k8s_node_name') as k8s_node_name
 	FROM %s.%s
 	WHERE metric_name IN (%s)
 		AND unix_milli >= toUnixTimestamp(now() - INTERVAL 60 MINUTE) * 1000
-		AND JSONExtractString(labels, 'host_name') LIKE '%%-otel-agent%%'`,
-		constants.SIGNOZ_METRIC_DBNAME, constants.SIGNOZ_TIMESERIES_V4_TABLENAME, namesStr)
+		AND JSONExtractString(labels, 'host_name') LIKE '%%-otel-agent%%'
+		AND fingerprint GLOBAL IN (%s)`,
+		constants.SIGNOZ_METRIC_DBNAME, constants.SIGNOZ_TIMESERIES_V4_TABLENAME, namesStr, queryForRecentFingerprints)
 
-	count, err := h.reader.GetCountOfThings(ctx, query)
-	return count > 0, err
+	result, err := h.reader.GetListResultV3(ctx, query)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	clusterNames := make(map[string]struct{})
+	nodeNames := make(map[string]struct{})
+
+	for _, row := range result {
+		switch v := row.Data["k8s_cluster_name"].(type) {
+		case string:
+			clusterNames[v] = struct{}{}
+		case *string:
+			clusterNames[*v] = struct{}{}
+		}
+		switch v := row.Data["k8s_node_name"].(type) {
+		case string:
+			nodeNames[v] = struct{}{}
+		case *string:
+			nodeNames[*v] = struct{}{}
+		}
+	}
+
+	return maps.Keys(clusterNames), maps.Keys(nodeNames), nil
 }
 
 func (h *HostsRepo) GetHostList(ctx context.Context, req model.HostListRequest) (model.HostListResponse, error) {
@@ -372,8 +403,10 @@ func (h *HostsRepo) GetHostList(ctx context.Context, req model.HostListRequest)
 	}
 
 	// don't fail the request if we can't get these values
-	if sendingK8SAgentMetrics, err := h.IsSendingK8SAgentMetrics(ctx, req); err == nil {
-		resp.IsSendingK8SAgentMetrics = sendingK8SAgentMetrics
+	if clusterNames, nodeNames, err := h.IsSendingK8SAgentMetrics(ctx, req); err == nil {
+		resp.IsSendingK8SAgentMetrics = len(clusterNames) > 0 || len(nodeNames) > 0
+		resp.ClusterNames = clusterNames
+		resp.NodeNames = nodeNames
 	}
 	if sentAnyHostMetricsData, err := h.DidSendHostMetricsData(ctx, req); err == nil {
 		resp.SentAnyHostMetricsData = sentAnyHostMetricsData
diff --git a/pkg/query-service/model/infra.go b/pkg/query-service/model/infra.go
index e9c9318efd..19670c42df 100644
--- a/pkg/query-service/model/infra.go
+++ b/pkg/query-service/model/infra.go
@@ -42,6 +42,8 @@ type HostListResponse struct {
 	Total                    int              `json:"total"`
 	SentAnyHostMetricsData   bool             `json:"sentAnyHostMetricsData"`
 	IsSendingK8SAgentMetrics bool             `json:"isSendingK8SAgentMetrics"`
+	ClusterNames             []string         `json:"clusterNames"`
+	NodeNames                []string         `json:"nodeNames"`
 }
 
 func (r *HostListResponse) SortBy(orderBy *v3.OrderBy) {