Chore: qs filter suggestions: example queries for multiple top attributes (#5703)

* chore: put together helper for fetching values for multiple attributes

* chore: poc: use helper for filter suggestions

* chore: add a working impl for getting attrib values for multiple attributes

* chore: start updating integration test to account for new approach for getting log attrib values

* chore: use a global zap logger in filter suggestion tests

* chore: fix attrib values clickhouse query expectation

* chore: only query values for actual attributes when generating example queries

* chore: update clickhouse-go-mock

* chore: cleanup: separate params for attributesLimit and examplesLimit for filter suggestions

* chore: some test cleanup

* chore: some more cleanup

* chore: some more cleanup

---------

Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/sethvargo/go-password v0.2.0
 	github.com/smartystreets/goconvey v1.8.1
 	github.com/soheilhy/cmux v0.1.5
-	github.com/srikanthccv/ClickHouse-go-mock v0.8.0
+	github.com/srikanthccv/ClickHouse-go-mock v0.9.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.103.0
 	go.opentelemetry.io/collector/confmap v0.103.0

go.sum

@@ -716,6 +716,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/srikanthccv/ClickHouse-go-mock v0.8.0 h1:DeeM8XLbTFl6sjYPPwazPEXx7kmRV8TgPFVkt1SqT0Y=
 github.com/srikanthccv/ClickHouse-go-mock v0.8.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24=
+github.com/srikanthccv/ClickHouse-go-mock v0.9.0 h1:XKr1Tb7GL1HlifKH874QGR3R6l0e6takXasROUiZawU=
+github.com/srikanthccv/ClickHouse-go-mock v0.9.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

pkg/query-service/app/clickhouseReader/reader.go

@@ -4414,7 +4414,7 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs(
 		ctx, &v3.FilterAttributeKeyRequest{
 			SearchText: req.SearchText,
 			DataSource: v3.DataSourceLogs,
-			Limit:      req.Limit,
+			Limit:      int(req.AttributesLimit),
 		})
 	if err != nil {
 		return nil, model.InternalError(fmt.Errorf("couldn't get attribute keys: %w", err))
@@ -4458,53 +4458,61 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs(
 		}
 	}
 
-	// Suggest example query for top suggested attribute using existing
+	// Suggest example queries for top suggested log attributes and resource attributes
-	// autocomplete logic for recommending attrib values
+	exampleAttribs := []v3.AttributeKey{}
-	//
+	for _, attrib := range suggestions.AttributeKeys {
-	// Example queries for multiple top attributes using a batch version of
+		isAttributeOrResource := slices.Contains([]v3.AttributeKeyType{
-	// GetLogAttributeValues is expected to come in a follow up change
+			v3.AttributeKeyTypeResource, v3.AttributeKeyTypeTag,
-	if len(suggestions.AttributeKeys) > 0 {
+		}, attrib.Type)
-		topAttrib := suggestions.AttributeKeys[0]
 
-		resp, err := r.GetLogAttributeValues(ctx, &v3.FilterAttributeValueRequest{
+		isNumOrStringType := slices.Contains([]v3.AttributeKeyDataType{
-			DataSource:                 v3.DataSourceLogs,
+			v3.AttributeKeyDataTypeInt64, v3.AttributeKeyDataTypeFloat64, v3.AttributeKeyDataTypeString,
-			FilterAttributeKey:         topAttrib.Key,
+		}, attrib.DataType)
-			FilterAttributeKeyDataType: topAttrib.DataType,
-			TagType:                    v3.TagType(topAttrib.Type),
-			Limit:                      1,
-		})
 
+		if isAttributeOrResource && isNumOrStringType {
+			exampleAttribs = append(exampleAttribs, attrib)
+		}
+
+		if len(exampleAttribs) >= int(req.ExamplesLimit) {
+			break
+		}
+	}
+
+	if len(exampleAttribs) > 0 {
+		exampleAttribValues, err := r.getValuesForLogAttributes(
+			ctx, exampleAttribs, req.ExamplesLimit,
+		)
 		if err != nil {
 			// Do not fail the entire request if only example query generation fails
 			zap.L().Error("could not find attribute values for creating example query", zap.Error(err))
-
 		} else {
-			addExampleQuerySuggestion := func(value any) {
-				exampleQuery := newExampleQuery()
 
-				exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{
+			// add example queries for as many attributes as possible.
-					Key:      topAttrib,
+			// suggest 1st value for 1st attrib, followed by 1st value for second attrib and so on
-					Operator: "=",
+			// and if there is still room, suggest 2nd value for 1st attrib, 2nd value for 2nd attrib and so on
-					Value:    value,
+			for valueIdx := 0; valueIdx < int(req.ExamplesLimit); valueIdx++ {
-				})
+				for attrIdx, attr := range exampleAttribs {
+					needMoreExamples := len(suggestions.ExampleQueries) < int(req.ExamplesLimit)
 
-				suggestions.ExampleQueries = append(
+					if needMoreExamples && valueIdx < len(exampleAttribValues[attrIdx]) {
-					suggestions.ExampleQueries, exampleQuery,
+						exampleQuery := newExampleQuery()
-				)
+						exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{
-			}
+							Key:      attr,
+							Operator: "=",
+							Value:    exampleAttribValues[attrIdx][valueIdx],
+						})
 
-			if len(resp.StringAttributeValues) > 0 {
+						suggestions.ExampleQueries = append(
-				addExampleQuerySuggestion(resp.StringAttributeValues[0])
+							suggestions.ExampleQueries, exampleQuery,
-			} else if len(resp.NumberAttributeValues) > 0 {
+						)
-				addExampleQuerySuggestion(resp.NumberAttributeValues[0])
+					}
-			} else if len(resp.BoolAttributeValues) > 0 {
+				}
-				addExampleQuerySuggestion(resp.BoolAttributeValues[0])
 			}
 		}
 	}
 
 	// Suggest static example queries for standard log attributes if needed.
-	if len(suggestions.ExampleQueries) < req.Limit {
+	if len(suggestions.ExampleQueries) < int(req.ExamplesLimit) {
 		exampleQuery := newExampleQuery()
 		exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{
 			Key: v3.AttributeKey{
@@ -4522,6 +4530,108 @@ func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs(
 	return &suggestions, nil
 }
 
+// Get up to `limit` values seen for each attribute in `attributes`
+// Returns a slice of slices where the ith slice has values for ith entry in `attributes`
+func (r *ClickHouseReader) getValuesForLogAttributes(
+	ctx context.Context, attributes []v3.AttributeKey, limit uint64,
+) ([][]any, *model.ApiError) {
+	// query top `limit` distinct values seen for `tagKey`s of interest
+	// ordered by timestamp when the value was seen
+	query := fmt.Sprintf(
+		`
+		select tagKey, stringTagValue, int64TagValue, float64TagValue
+		from (
+			select
+				tagKey,
+				stringTagValue,
+				int64TagValue,
+				float64TagValue,
+				row_number() over (partition by tagKey order by ts desc) as rank
+			from (
+				select
+					tagKey,
+					stringTagValue,
+					int64TagValue,
+					float64TagValue,
+					max(timestamp) as ts
+				from %s.%s
+				where tagKey in $1
+				group by (tagKey, stringTagValue, int64TagValue, float64TagValue)
+			)
+		)
+		where rank <= %d
+		`,
+		r.logsDB, r.logsTagAttributeTable, limit,
+	)
+
+	attribNames := []string{}
+	for _, attrib := range attributes {
+		attribNames = append(attribNames, attrib.Key)
+	}
+
+	rows, err := r.db.Query(ctx, query, attribNames)
+	if err != nil {
+		zap.L().Error("couldn't query attrib values for suggestions", zap.Error(err))
+		return nil, model.InternalError(fmt.Errorf(
+			"couldn't query attrib values for suggestions: %w", err,
+		))
+	}
+	defer rows.Close()
+
+	result := make([][]any, len(attributes))
+
+	// Helper for getting hold of the result slice to append to for each scanned row
+	resultIdxForAttrib := func(key string, dataType v3.AttributeKeyDataType) int {
+		return slices.IndexFunc(attributes, func(attrib v3.AttributeKey) bool {
+			return attrib.Key == key && attrib.DataType == dataType
+		})
+	}
+
+	// Scan rows and append to result
+	for rows.Next() {
+		var tagKey string
+		var stringValue string
+		var float64Value sql.NullFloat64
+		var int64Value sql.NullInt64
+
+		err := rows.Scan(
+			&tagKey, &stringValue, &int64Value, &float64Value,
+		)
+		if err != nil {
+			return nil, model.InternalError(fmt.Errorf(
+				"couldn't scan attrib value rows: %w", err,
+			))
+		}
+
+		if len(stringValue) > 0 {
+			attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeString)
+			if attrResultIdx >= 0 {
+				result[attrResultIdx] = append(result[attrResultIdx], stringValue)
+			}
+
+		} else if int64Value.Valid {
+			attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeInt64)
+			if attrResultIdx >= 0 {
+				result[attrResultIdx] = append(result[attrResultIdx], int64Value.Int64)
+			}
+
+		} else if float64Value.Valid {
+			attrResultIdx := resultIdxForAttrib(tagKey, v3.AttributeKeyDataTypeFloat64)
+			if attrResultIdx >= 0 {
+				result[attrResultIdx] = append(result[attrResultIdx], float64Value.Float64)
+			}
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, model.InternalError(fmt.Errorf(
+			"couldn't scan attrib value rows: %w", err,
+		))
+	}
+
+	return result, nil
+}
+
 func readRow(vars []interface{}, columnNames []string, countOfNumberCols int) ([]string, map[string]string, []map[string]string, *v3.Point) {
 	// Each row will have a value and a timestamp, and an optional list of label values
 	// example: {Timestamp: ..., Value: ...}

pkg/query-service/app/parser.go

@@ -846,15 +846,41 @@ func parseQBFilterSuggestionsRequest(r *http.Request) (
 		return nil, model.BadRequest(err)
 	}
 
-	limit := baseconstants.DefaultFilterSuggestionsLimit
+	parsePositiveIntQP := func(
-	limitStr := r.URL.Query().Get("limit")
+		queryParam string, defaultValue uint64, maxValue uint64,
-	if len(limitStr) > 0 {
+	) (uint64, *model.ApiError) {
-		limit, err := strconv.Atoi(limitStr)
+		value := defaultValue
-		if err != nil || limit < 1 {
+
-			return nil, model.BadRequest(fmt.Errorf(
+		qpValue := r.URL.Query().Get(queryParam)
-				"invalid limit: %s", limitStr,
+		if len(qpValue) > 0 {
-			))
+			value, err := strconv.Atoi(qpValue)
+
+			if err != nil || value < 1 || value > int(maxValue) {
+				return 0, model.BadRequest(fmt.Errorf(
+					"invalid %s: %s", queryParam, qpValue,
+				))
+			}
 		}
+
+		return value, nil
+	}
+
+	attributesLimit, err := parsePositiveIntQP(
+		"attributesLimit",
+		baseconstants.DefaultFilterSuggestionsAttributesLimit,
+		baseconstants.MaxFilterSuggestionsAttributesLimit,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	examplesLimit, err := parsePositiveIntQP(
+		"examplesLimit",
+		baseconstants.DefaultFilterSuggestionsExamplesLimit,
+		baseconstants.MaxFilterSuggestionsExamplesLimit,
+	)
+	if err != nil {
+		return nil, err
 	}
 
 	var existingFilter *v3.FilterSet
@@ -875,10 +901,11 @@ func parseQBFilterSuggestionsRequest(r *http.Request) (
 	searchText := r.URL.Query().Get("searchText")
 
 	return &v3.QBFilterSuggestionsRequest{
-		DataSource:     dataSource,
+		DataSource:      dataSource,
-		Limit:          limit,
+		SearchText:      searchText,
-		SearchText:     searchText,
+		ExistingFilter:  existingFilter,
-		ExistingFilter: existingFilter,
+		AttributesLimit: attributesLimit,
+		ExamplesLimit:   examplesLimit,
 	}, nil
 }
 

pkg/query-service/constants/constants.go

@@ -417,4 +417,7 @@ var TracesListViewDefaultSelectedColumns = []v3.AttributeKey{
 	},
 }
 
-const DefaultFilterSuggestionsLimit = 100
+const DefaultFilterSuggestionsAttributesLimit = 50
+const MaxFilterSuggestionsAttributesLimit = 100
+const DefaultFilterSuggestionsExamplesLimit = 2
+const MaxFilterSuggestionsExamplesLimit = 10

pkg/query-service/model/v3/v3.go

@@ -253,10 +253,11 @@ type FilterAttributeKeyRequest struct {
 }
 
 type QBFilterSuggestionsRequest struct {
-	DataSource     DataSource `json:"dataSource"`
+	DataSource      DataSource `json:"dataSource"`
-	SearchText     string     `json:"searchText"`
+	SearchText      string     `json:"searchText"`
-	Limit          int        `json:"limit"`
+	ExistingFilter  *FilterSet `json:"existingFilter"`
-	ExistingFilter *FilterSet `json:"existing_filter"`
+	AttributesLimit uint64     `json:"attributesLimit"`
+	ExamplesLimit   uint64     `json:"examplesLimit"`
 }
 
 type QBFilterSuggestionsResponse struct {

pkg/query-service/tests/integration/filter_suggestions_test.go

@@ -19,6 +19,7 @@ import (
 	"go.signoz.io/signoz/pkg/query-service/model"
 	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.signoz.io/signoz/pkg/query-service/utils"
+	"go.uber.org/zap"
 )
 
 // If no data has been received yet, filter suggestions should contain
@@ -59,7 +60,9 @@ func TestLogsFilterSuggestionsWithoutExistingFilter(t *testing.T) {
 	testAttribValue := "test-container"
 
 	tb.mockAttribKeysQueryResponse([]v3.AttributeKey{testAttrib})
-	tb.mockAttribValuesQueryResponse(testAttrib, []string{testAttribValue})
+	tb.mockAttribValuesQueryResponse(
+		[]v3.AttributeKey{testAttrib}, [][]string{{testAttribValue}},
+	)
 	suggestionsQueryParams := map[string]string{}
 	suggestionsResp := tb.GetQBFilterSuggestionsForLogs(suggestionsQueryParams)
 
@@ -71,6 +74,7 @@ func TestLogsFilterSuggestionsWithoutExistingFilter(t *testing.T) {
 	))
 
 	require.Greater(len(suggestionsResp.ExampleQueries), 0)
+
 	require.True(slices.ContainsFunc(
 		suggestionsResp.ExampleQueries, func(q v3.FilterSet) bool {
 			return slices.ContainsFunc(q.Items, func(i v3.FilterItem) bool {
@@ -113,7 +117,10 @@ func TestLogsFilterSuggestionsWithExistingFilter(t *testing.T) {
 	}
 
 	tb.mockAttribKeysQueryResponse([]v3.AttributeKey{testAttrib, testFilterAttrib})
-	tb.mockAttribValuesQueryResponse(testAttrib, []string{testAttribValue})
+	tb.mockAttribValuesQueryResponse(
+		[]v3.AttributeKey{testAttrib, testFilterAttrib},
+		[][]string{{testAttribValue}, {testFilterAttribValue}},
+	)
 
 	testFilterJson, err := json.Marshal(testFilter)
 	require.Nil(err, "couldn't serialize existing filter to JSON")
@@ -152,7 +159,7 @@ func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse(
 	tb.mockClickhouse.ExpectQuery(
 		"select.*from.*signoz_logs.distributed_tag_attributes.*",
 	).WithArgs(
-		constants.DefaultFilterSuggestionsLimit,
+		constants.DefaultFilterSuggestionsAttributesLimit,
 	).WillReturnRows(
 		mockhouse.NewRows(cols, values),
 	)
@@ -169,22 +176,30 @@ func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse(
 
 // Mocks response for CH queries made by reader.GetLogAttributeValues
 func (tb *FilterSuggestionsTestBed) mockAttribValuesQueryResponse(
-	expectedAttrib v3.AttributeKey,
+	expectedAttribs []v3.AttributeKey,
-	stringValuesToReturn []string,
+	stringValuesToReturn [][]string,
 ) {
-	cols := []mockhouse.ColumnType{}
+	resultCols := []mockhouse.ColumnType{
-	cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "stringTagValue"})
+		{Type: "String", Name: "tagKey"},
+		{Type: "String", Name: "stringTagValue"},
+		{Type: "Nullable(Int64)", Name: "int64TagValue"},
+		{Type: "Nullable(Float64)", Name: "float64TagValue"},
+	}
 
-	values := [][]any{}
+	expectedAttribKeysInQuery := []string{}
-	for _, v := range stringValuesToReturn {
+	mockResultRows := [][]any{}
-		rowValues := []any{}
+	for idx, attrib := range expectedAttribs {
-		rowValues = append(rowValues, v)
+		expectedAttribKeysInQuery = append(expectedAttribKeysInQuery, attrib.Key)
-		values = append(values, rowValues)
+		for _, stringTagValue := range stringValuesToReturn[idx] {
+			mockResultRows = append(mockResultRows, []any{
+				attrib.Key, stringTagValue, nil, nil,
+			})
+		}
 	}
 
 	tb.mockClickhouse.ExpectQuery(
-		"select distinct.*stringTagValue.*from.*signoz_logs.distributed_tag_attributes.*",
+		"select.*tagKey.*stringTagValue.*int64TagValue.*float64TagValue.*distributed_tag_attributes.*tagKey.*in.*",
-	).WithArgs(string(expectedAttrib.Key), v3.TagType(expectedAttrib.Type), 1).WillReturnRows(mockhouse.NewRows(cols, values))
+	).WithArgs(expectedAttribKeysInQuery).WillReturnRows(mockhouse.NewRows(resultCols, mockResultRows))
 }
 
 type FilterSuggestionsTestBed struct {
@@ -244,6 +259,13 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed {
 		t.Fatalf("could not create a test user: %v", apiErr)
 	}
 
+	logger := zap.NewExample()
+	originalLogger := zap.L()
+	zap.ReplaceGlobals(logger)
+	t.Cleanup(func() {
+		zap.ReplaceGlobals(originalLogger)
+	})
+
 	return &FilterSuggestionsTestBed{
 		t:              t,
 		testUser:       user,