Feat: logs filter suggestions higher rank for special resource attribs like service.name and env etc (#6060)

* chore: upgrade signoz-otel-collector dependency to v0.102.10

* feat: first stab at ranking resource attribs higher

* chore: add test todo for validating resource attribs get ranked higher in logs filter suggestions

* chore: add test validating higher ranking for special resource attribs

* chore: some cleanup

* chore: some more cleanup

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.23.2
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.102.2
+	github.com/SigNoz/signoz-otel-collector v0.102.10
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3

go.sum

@@ -66,6 +66,8 @@ github.com/SigNoz/prometheus v1.11.1 h1:roM8ugYf4UxaeKKujEeBvoX7ybq3IrS+TB26KiRt
 github.com/SigNoz/prometheus v1.11.1/go.mod h1:uv4mQwZQtx7y4GQ6EdHOi8Wsk07uHNn2XHd1zM85m6I=
 github.com/SigNoz/signoz-otel-collector v0.102.2 h1:SmjsBZjMjTVVpuOlfJXlsDJQbdefQP/9Wz3CyzSuZuU=
 github.com/SigNoz/signoz-otel-collector v0.102.2/go.mod h1:ISAXYhZenojCWg6CdDJtPMpfS6Zwc08+uoxH25tc6Y0=
+github.com/SigNoz/signoz-otel-collector v0.102.10 h1:1zjU31OcRZL6fS0IIag8LA8bdhP4S28dzovDwuOg7Lg=
+github.com/SigNoz/signoz-otel-collector v0.102.10/go.mod h1:APoBVD4aRu9vIny1vdzZSi2wPY3elyjHA/I/rh1hKfs=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=

pkg/query-service/app/clickhouseReader/filter_suggestions.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/SigNoz/signoz-otel-collector/exporter/clickhouselogsexporter/logsv2"
 	"go.signoz.io/signoz/pkg/query-service/model"
 	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.uber.org/zap"
@@ -36,26 +37,7 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs(
 	suggestions.AttributeKeys = attribKeysResp.AttributeKeys
 
 	// Rank suggested attributes
-	slices.SortFunc(suggestions.AttributeKeys, func(a v3.AttributeKey, b v3.AttributeKey) int {
+	attribRanker.sort(suggestions.AttributeKeys)
-
-		// Higher score => higher rank
-		attribKeyScore := func(a v3.AttributeKey) int {
-
-			// Scoring criteria is expected to get more sophisticated in follow up changes
-			if a.Type == v3.AttributeKeyTypeResource {
-				return 2
-			}
-
-			if a.Type == v3.AttributeKeyTypeTag {
-				return 1
-			}
-
-			return 0
-		}
-
-		// To sort in descending order of score the return value must be negative when a > b
-		return attribKeyScore(b) - attribKeyScore(a)
-	})
 
 	// Put together suggested example queries.
 
@@ -268,3 +250,59 @@ func (r *ClickHouseReader) getValuesForLogAttributes(
 
 	return result, nil
 }
+
+var attribRanker = newRankingStrategy()
+
+func newRankingStrategy() attribRankingStrategy {
+	// Some special resource attributes should get ranked above all others.
+	interestingResourceAttrsInDescRank := []string{
+		"service", "service.name", "env", "k8s.namespace.name",
+	}
+
+	// Synonyms of interesting attributes should come next
+	resourceHierarchy := logsv2.ResourceHierarchy()
+	for _, attr := range []string{
+		"service.name",
+		"deployment.environment",
+		"k8s.namespace.name",
+		"k8s.pod.name",
+		"k8s.container.name",
+		"k8s.node.name",
+	} {
+		interestingResourceAttrsInDescRank = append(
+			interestingResourceAttrsInDescRank, resourceHierarchy.Synonyms(attr)...,
+		)
+	}
+
+	interestingResourceAttrsInAscRank := interestingResourceAttrsInDescRank[:]
+	slices.Reverse(interestingResourceAttrsInAscRank)
+
+	return attribRankingStrategy{
+		interestingResourceAttrsInAscRank: interestingResourceAttrsInAscRank,
+	}
+}
+
+type attribRankingStrategy struct {
+	interestingResourceAttrsInAscRank []string
+}
+
+// The higher the score, the higher the rank
+func (s *attribRankingStrategy) score(attrib v3.AttributeKey) int {
+	if attrib.Type == v3.AttributeKeyTypeResource {
+		// 3 + (-1) if attrib.Key is not an interesting resource attribute
+		return 3 + slices.Index(s.interestingResourceAttrsInAscRank, attrib.Key)
+	}
+
+	if attrib.Type == v3.AttributeKeyTypeTag {
+		return 1
+	}
+
+	return 0
+}
+
+func (s *attribRankingStrategy) sort(attribKeys []v3.AttributeKey) {
+	slices.SortFunc(attribKeys, func(a v3.AttributeKey, b v3.AttributeKey) int {
+		// To sort in descending order of score the return value must be negative when a > b
+		return s.score(b) - s.score(a)
+	})
+}

pkg/query-service/tests/integration/filter_suggestions_test.go

@@ -138,6 +138,62 @@ func TestLogsFilterSuggestionsWithExistingFilter(t *testing.T) {
 	}
 }
 
+func TestResourceAttribsRankedHigherInLogsFilterSuggestions(t *testing.T) {
+	require := require.New(t)
+
+	tagKeys := []v3.AttributeKey{}
+	for _, k := range []string{"user_id", "user_email"} {
+		tagKeys = append(tagKeys, v3.AttributeKey{
+			Key:      k,
+			Type:     v3.AttributeKeyTypeTag,
+			DataType: v3.AttributeKeyDataTypeString,
+			IsColumn: false,
+		})
+	}
+
+	specialResourceAttrKeys := []v3.AttributeKey{}
+	for _, k := range []string{"service", "env"} {
+		specialResourceAttrKeys = append(specialResourceAttrKeys, v3.AttributeKey{
+			Key:      k,
+			Type:     v3.AttributeKeyTypeResource,
+			DataType: v3.AttributeKeyDataTypeString,
+			IsColumn: false,
+		})
+	}
+
+	otherResourceAttrKeys := []v3.AttributeKey{}
+	for _, k := range []string{"container_name", "container_id"} {
+		otherResourceAttrKeys = append(otherResourceAttrKeys, v3.AttributeKey{
+			Key:      k,
+			Type:     v3.AttributeKeyTypeResource,
+			DataType: v3.AttributeKeyDataTypeString,
+			IsColumn: false,
+		})
+	}
+
+	tb := NewFilterSuggestionsTestBed(t)
+
+	mockAttrKeysInDB := append(tagKeys, otherResourceAttrKeys...)
+	mockAttrKeysInDB = append(mockAttrKeysInDB, specialResourceAttrKeys...)
+
+	tb.mockAttribKeysQueryResponse(mockAttrKeysInDB)
+
+	expectedTopSuggestions := append(specialResourceAttrKeys, otherResourceAttrKeys...)
+	expectedTopSuggestions = append(expectedTopSuggestions, tagKeys...)
+
+	tb.mockAttribValuesQueryResponse(
+		expectedTopSuggestions[:2], [][]string{{"test"}, {"test"}},
+	)
+
+	suggestionsQueryParams := map[string]string{"examplesLimit": "2"}
+	suggestionsResp := tb.GetQBFilterSuggestionsForLogs(suggestionsQueryParams)
+
+	require.Equal(
+		expectedTopSuggestions,
+		suggestionsResp.AttributeKeys[:len(expectedTopSuggestions)],
+	)
+}
+
 // Mocks response for CH queries made by reader.GetLogAttributeKeys
 func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse(
 	attribsToReturn []v3.AttributeKey,