diff --git a/go.mod b/go.mod index b62ebab95f..f2dadeede1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.signoz.io/signoz go 1.21.3 require ( - github.com/ClickHouse/clickhouse-go/v2 v2.20.0 + github.com/ClickHouse/clickhouse-go/v2 v2.23.2 github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd github.com/SigNoz/signoz-otel-collector v0.102.2 @@ -46,7 +46,7 @@ require ( github.com/sethvargo/go-password v0.2.0 github.com/smartystreets/goconvey v1.8.1 github.com/soheilhy/cmux v0.1.5 - github.com/srikanthccv/ClickHouse-go-mock v0.7.0 + github.com/srikanthccv/ClickHouse-go-mock v0.8.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.102.1 go.opentelemetry.io/collector/confmap v0.102.1 @@ -83,7 +83,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect - github.com/ClickHouse/ch-go v0.61.3 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go v1.53.16 // indirect @@ -156,7 +156,7 @@ require ( github.com/segmentio/backo-go v1.0.1 // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/spf13/cobra v1.8.0 // indirect diff --git a/go.sum b/go.sum index 6638d7b40d..e3df44618e 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.61.3 h1:MmBwUhXrAOBZK7n/sWBzq6FdIQ01cuF2SaaO8KlDRzI= github.com/ClickHouse/ch-go v0.61.3/go.mod h1:1PqXjMz/7S1ZUaKvwPA3i35W2bz2mAMFeCi6DIXgGwQ= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= github.com/ClickHouse/clickhouse-go/v2 v2.20.0 h1:bvlLQ31XJfl7MxIqAq2l1G6JhHYzqEXdvfpMeU6bkKc= github.com/ClickHouse/clickhouse-go/v2 v2.20.0/go.mod h1:VQfyA+tCwCRw2G7ogfY8V0fq/r0yJWzy8UDrjiP/Lbs= +github.com/ClickHouse/clickhouse-go/v2 v2.23.2 h1:+DAKPMnxLS7pduQZsrJc8OhdLS2L9MfDEJ2TS+hpYDM= +github.com/ClickHouse/clickhouse-go/v2 v2.23.2/go.mod h1:aNap51J1OM3yxQJRgM+AlP/MPkGBCL8A74uQThoQhR0= github.com/Code-Hex/go-generics-cache v1.5.1 h1:6vhZGc5M7Y/YD8cIUcY8kcuQLB4cHR7U+0KMqAA0KcU= github.com/Code-Hex/go-generics-cache v1.5.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= @@ -693,6 +697,8 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= @@ -718,6 +724,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/srikanthccv/ClickHouse-go-mock v0.7.0 h1:XhRMX2663xkDGq3DYavw8m75O94s9u76hOIjo9QBl8c= github.com/srikanthccv/ClickHouse-go-mock v0.7.0/go.mod h1:IJZ/eL1h4cOy/Jo3PzNKXSPmqRus15BC2MbduYPpA/g= +github.com/srikanthccv/ClickHouse-go-mock v0.8.0 h1:DeeM8XLbTFl6sjYPPwazPEXx7kmRV8TgPFVkt1SqT0Y= +github.com/srikanthccv/ClickHouse-go-mock v0.8.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index e9123d1ff6..0c8ba834fe 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -13,6 +13,7 @@ import ( "os" "reflect" "regexp" + "slices" "sort" "strconv" "strings" @@ -4357,6 +4358,128 @@ func (r *ClickHouseReader) GetLogAttributeValues(ctx context.Context, req *v3.Fi } +func (r *ClickHouseReader) GetQBFilterSuggestionsForLogs( + ctx context.Context, + req *v3.QBFilterSuggestionsRequest, +) (*v3.QBFilterSuggestionsResponse, *model.ApiError) { + suggestions := v3.QBFilterSuggestionsResponse{ + AttributeKeys: []v3.AttributeKey{}, + ExampleQueries: []v3.FilterSet{}, + } + + // Use existing autocomplete logic for generating attribute suggestions + attribKeysResp, err := r.GetLogAttributeKeys( + ctx, &v3.FilterAttributeKeyRequest{ + SearchText: req.SearchText, + DataSource: v3.DataSourceLogs, + Limit: req.Limit, + }) + if err != nil { + return nil, model.InternalError(fmt.Errorf("couldn't get attribute keys: %w", err)) + } + + suggestions.AttributeKeys = attribKeysResp.AttributeKeys + + // Rank suggested attributes + slices.SortFunc(suggestions.AttributeKeys, func(a v3.AttributeKey, b v3.AttributeKey) int { + + // Higher score => higher rank + attribKeyScore := func(a v3.AttributeKey) int { + + // Scoring criteria is expected to get more sophisticated in follow up changes + if a.Type == v3.AttributeKeyTypeResource { + return 2 + } + + if a.Type == v3.AttributeKeyTypeTag { + return 1 + } + + return 0 + } + + // To sort in descending order of score the return value must be negative when a > b + return attribKeyScore(b) - attribKeyScore(a) + }) + + // Put together suggested example queries. + + newExampleQuery := func() v3.FilterSet { + // Include existing filter in example query if specified. + if req.ExistingFilter != nil { + return *req.ExistingFilter + } + + return v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{}, + } + } + + // Suggest example query for top suggested attribute using existing + // autocomplete logic for recommending attrib values + // + // Example queries for multiple top attributes using a batch version of + // GetLogAttributeValues is expected to come in a follow up change + if len(suggestions.AttributeKeys) > 0 { + topAttrib := suggestions.AttributeKeys[0] + + resp, err := r.GetLogAttributeValues(ctx, &v3.FilterAttributeValueRequest{ + DataSource: v3.DataSourceLogs, + FilterAttributeKey: topAttrib.Key, + FilterAttributeKeyDataType: topAttrib.DataType, + TagType: v3.TagType(topAttrib.Type), + Limit: 1, + }) + + if err != nil { + // Do not fail the entire request if only example query generation fails + zap.L().Error("could not find attribute values for creating example query", zap.Error(err)) + + } else { + addExampleQuerySuggestion := func(value any) { + exampleQuery := newExampleQuery() + + exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{ + Key: topAttrib, + Operator: "=", + Value: value, + }) + + suggestions.ExampleQueries = append( + suggestions.ExampleQueries, exampleQuery, + ) + } + + if len(resp.StringAttributeValues) > 0 { + addExampleQuerySuggestion(resp.StringAttributeValues[0]) + } else if len(resp.NumberAttributeValues) > 0 { + addExampleQuerySuggestion(resp.NumberAttributeValues[0]) + } else if len(resp.BoolAttributeValues) > 0 { + addExampleQuerySuggestion(resp.BoolAttributeValues[0]) + } + } + } + + // Suggest static example queries for standard log attributes if needed. + if len(suggestions.ExampleQueries) < req.Limit { + exampleQuery := newExampleQuery() + exampleQuery.Items = append(exampleQuery.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "body", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + }, + Operator: "contains", + Value: "error", + }) + suggestions.ExampleQueries = append(suggestions.ExampleQueries, exampleQuery) + } + + return &suggestions, nil +} + func readRow(vars []interface{}, columnNames []string, countOfNumberCols int) ([]string, map[string]string, []map[string]string, *v3.Point) { // Each row will have a value and a timestamp, and an optional list of label values // example: {Timestamp: ..., Value: ...} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 29cb807686..33dd443555 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -302,6 +302,8 @@ func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMid subRouter.HandleFunc("/query_range", am.ViewAccess(aH.QueryRangeV3)).Methods(http.MethodPost) subRouter.HandleFunc("/query_range/format", am.ViewAccess(aH.QueryRangeV3Format)).Methods(http.MethodPost) + subRouter.HandleFunc("/filter_suggestions", am.ViewAccess(aH.getQueryBuilderSuggestions)).Methods(http.MethodGet) + // live logs subRouter.HandleFunc("/logs/livetail", am.ViewAccess(aH.liveTailLogs)).Methods(http.MethodGet) } @@ -3150,6 +3152,30 @@ func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r * aH.Respond(w, response) } +func (aH *APIHandler) getQueryBuilderSuggestions(w http.ResponseWriter, r *http.Request) { + req, err := parseQBFilterSuggestionsRequest(r) + if err != nil { + RespondError(w, err, nil) + return + } + + if req.DataSource != v3.DataSourceLogs { + // Support for traces and metrics might come later + RespondError(w, model.BadRequest( + fmt.Errorf("suggestions not supported for %s", req.DataSource), + ), nil) + return + } + + response, err := aH.reader.GetQBFilterSuggestionsForLogs(r.Context(), req) + if err != nil { + RespondError(w, err, nil) + return + } + + aH.Respond(w, response) +} + func (aH *APIHandler) autoCompleteAttributeKeys(w http.ResponseWriter, r *http.Request) { var response *v3.FilterAttributeKeyResponse req, err := parseFilterAttributeKeyRequest(r) diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index 47da531d0b..9bdde09be6 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -2,6 +2,7 @@ package app import ( "bytes" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -837,6 +838,50 @@ func parseAggregateAttributeRequest(r *http.Request) (*v3.AggregateAttributeRequ return &req, nil } +func parseQBFilterSuggestionsRequest(r *http.Request) ( + *v3.QBFilterSuggestionsRequest, *model.ApiError, +) { + dataSource := v3.DataSource(r.URL.Query().Get("dataSource")) + if err := dataSource.Validate(); err != nil { + return nil, model.BadRequest(err) + } + + limit := baseconstants.DefaultFilterSuggestionsLimit + limitStr := r.URL.Query().Get("limit") + if len(limitStr) > 0 { + limit, err := strconv.Atoi(limitStr) + if err != nil || limit < 1 { + return nil, model.BadRequest(fmt.Errorf( + "invalid limit: %s", limitStr, + )) + } + } + + var existingFilter *v3.FilterSet + existingFilterB64 := r.URL.Query().Get("existingFilter") + if len(existingFilterB64) > 0 { + decodedFilterJson, err := base64.RawURLEncoding.DecodeString(existingFilterB64) + if err != nil { + return nil, model.BadRequest(fmt.Errorf("couldn't base64 decode existingFilter: %w", err)) + } + + existingFilter = &v3.FilterSet{} + err = json.Unmarshal(decodedFilterJson, existingFilter) + if err != nil { + return nil, model.BadRequest(fmt.Errorf("couldn't JSON decode existingFilter: %w", err)) + } + } + + searchText := r.URL.Query().Get("searchText") + + return &v3.QBFilterSuggestionsRequest{ + DataSource: dataSource, + Limit: limit, + SearchText: searchText, + ExistingFilter: existingFilter, + }, nil +} + func parseFilterAttributeKeyRequest(r *http.Request) (*v3.FilterAttributeKeyRequest, error) { var req v3.FilterAttributeKeyRequest diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 561480b81a..e7e3bcf2a7 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -407,3 +407,5 @@ var TracesListViewDefaultSelectedColumns = []v3.AttributeKey{ IsColumn: true, }, } + +const DefaultFilterSuggestionsLimit = 100 diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index fea923ac27..53c5ae8d68 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -93,6 +93,10 @@ type Reader interface { GetLogAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) GetLogAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) GetUsers(ctx context.Context) ([]model.UserPayload, error) + GetQBFilterSuggestionsForLogs( + ctx context.Context, + req *v3.QBFilterSuggestionsRequest, + ) (*v3.QBFilterSuggestionsResponse, *model.ApiError) // Connection needed for rules, not ideal but required GetConn() clickhouse.Conn diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index e6ac8441d6..76e781ffc1 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -252,6 +252,18 @@ type FilterAttributeKeyRequest struct { Limit int `json:"limit"` } +type QBFilterSuggestionsRequest struct { + DataSource DataSource `json:"dataSource"` + SearchText string `json:"searchText"` + Limit int `json:"limit"` + ExistingFilter *FilterSet `json:"existing_filter"` +} + +type QBFilterSuggestionsResponse struct { + AttributeKeys []AttributeKey `json:"attributes"` + ExampleQueries []FilterSet `json:"example_queries"` +} + type AttributeKeyDataType string const ( diff --git a/pkg/query-service/tests/integration/filter_suggestions_test.go b/pkg/query-service/tests/integration/filter_suggestions_test.go new file mode 100644 index 0000000000..8c379b1c10 --- /dev/null +++ b/pkg/query-service/tests/integration/filter_suggestions_test.go @@ -0,0 +1,279 @@ +package tests + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + "testing" + + mockhouse "github.com/srikanthccv/ClickHouse-go-mock" + "github.com/stretchr/testify/require" + "go.signoz.io/signoz/pkg/query-service/app" + "go.signoz.io/signoz/pkg/query-service/auth" + "go.signoz.io/signoz/pkg/query-service/constants" + "go.signoz.io/signoz/pkg/query-service/dao" + "go.signoz.io/signoz/pkg/query-service/featureManager" + "go.signoz.io/signoz/pkg/query-service/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +// If no data has been received yet, filter suggestions should contain +// standard log fields and static example queries based on them +func TestDefaultLogsFilterSuggestions(t *testing.T) { + require := require.New(t) + tb := NewFilterSuggestionsTestBed(t) + + tb.mockAttribKeysQueryResponse([]v3.AttributeKey{}) + suggestionsQueryParams := map[string]string{} + suggestionsResp := tb.GetQBFilterSuggestionsForLogs(suggestionsQueryParams) + + require.Greater(len(suggestionsResp.AttributeKeys), 0) + require.True(slices.ContainsFunc( + suggestionsResp.AttributeKeys, func(a v3.AttributeKey) bool { + return a.Key == "body" + }, + )) + + require.Greater(len(suggestionsResp.ExampleQueries), 0) + require.False(slices.ContainsFunc( + suggestionsResp.AttributeKeys, func(a v3.AttributeKey) bool { + return a.Type == v3.AttributeKeyTypeTag || a.Type == v3.AttributeKeyTypeResource + }, + )) +} + +func TestLogsFilterSuggestionsWithoutExistingFilter(t *testing.T) { + require := require.New(t) + tb := NewFilterSuggestionsTestBed(t) + + testAttrib := v3.AttributeKey{ + Key: "container_id", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: false, + } + testAttribValue := "test-container" + + tb.mockAttribKeysQueryResponse([]v3.AttributeKey{testAttrib}) + tb.mockAttribValuesQueryResponse(testAttrib, []string{testAttribValue}) + suggestionsQueryParams := map[string]string{} + suggestionsResp := tb.GetQBFilterSuggestionsForLogs(suggestionsQueryParams) + + require.Greater(len(suggestionsResp.AttributeKeys), 0) + require.True(slices.ContainsFunc( + suggestionsResp.AttributeKeys, func(a v3.AttributeKey) bool { + return a.Key == testAttrib.Key && a.Type == testAttrib.Type + }, + )) + + require.Greater(len(suggestionsResp.ExampleQueries), 0) + require.True(slices.ContainsFunc( + suggestionsResp.ExampleQueries, func(q v3.FilterSet) bool { + return slices.ContainsFunc(q.Items, func(i v3.FilterItem) bool { + return i.Key.Key == testAttrib.Key && i.Value == testAttribValue + }) + }, + )) +} + +// If a filter already exists, suggested example queries should +// contain existing filter +func TestLogsFilterSuggestionsWithExistingFilter(t *testing.T) { + require := require.New(t) + tb := NewFilterSuggestionsTestBed(t) + + testAttrib := v3.AttributeKey{ + Key: "container_id", + Type: v3.AttributeKeyTypeResource, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: false, + } + testAttribValue := "test-container" + + testFilterAttrib := v3.AttributeKey{ + Key: "tenant_id", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: false, + } + testFilterAttribValue := "test-tenant" + testFilter := v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: testFilterAttrib, + Operator: "=", + Value: testFilterAttribValue, + }, + }, + } + + tb.mockAttribKeysQueryResponse([]v3.AttributeKey{testAttrib, testFilterAttrib}) + tb.mockAttribValuesQueryResponse(testAttrib, []string{testAttribValue}) + + testFilterJson, err := json.Marshal(testFilter) + require.Nil(err, "couldn't serialize existing filter to JSON") + suggestionsQueryParams := map[string]string{ + "existingFilter": base64.RawURLEncoding.EncodeToString(testFilterJson), + } + suggestionsResp := tb.GetQBFilterSuggestionsForLogs(suggestionsQueryParams) + + require.Greater(len(suggestionsResp.AttributeKeys), 0) + + // All example queries should contain the existing filter as a prefix + require.Greater(len(suggestionsResp.ExampleQueries), 0) + for _, q := range suggestionsResp.ExampleQueries { + require.Equal(q.Items[0], testFilter.Items[0]) + } +} + +// Mocks response for CH queries made by reader.GetLogAttributeKeys +func (tb *FilterSuggestionsTestBed) mockAttribKeysQueryResponse( + attribsToReturn []v3.AttributeKey, +) { + cols := []mockhouse.ColumnType{} + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tagKey"}) + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tagType"}) + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "tagDataType"}) + + values := [][]any{} + for _, a := range attribsToReturn { + rowValues := []any{} + rowValues = append(rowValues, a.Key) + rowValues = append(rowValues, string(a.Type)) + rowValues = append(rowValues, string(a.DataType)) + values = append(values, rowValues) + } + + tb.mockClickhouse.ExpectQuery( + "select.*from.*signoz_logs.distributed_tag_attributes.*", + ).WithArgs( + constants.DefaultFilterSuggestionsLimit, + ).WillReturnRows( + mockhouse.NewRows(cols, values), + ) + + // Add expectation for the create table query used to determine + // if an attribute is a column + cols = []mockhouse.ColumnType{{Type: "String", Name: "statement"}} + values = [][]any{{"CREATE TABLE signoz_logs.distributed_logs"}} + tb.mockClickhouse.ExpectSelect( + "SHOW CREATE TABLE.*", + ).WillReturnRows(mockhouse.NewRows(cols, values)) + +} + +// Mocks response for CH queries made by reader.GetLogAttributeValues +func (tb *FilterSuggestionsTestBed) mockAttribValuesQueryResponse( + expectedAttrib v3.AttributeKey, + stringValuesToReturn []string, +) { + cols := []mockhouse.ColumnType{} + cols = append(cols, mockhouse.ColumnType{Type: "String", Name: "stringTagValue"}) + + values := [][]any{} + for _, v := range stringValuesToReturn { + rowValues := []any{} + rowValues = append(rowValues, v) + values = append(values, rowValues) + } + + tb.mockClickhouse.ExpectQuery( + "select distinct.*stringTagValue.*from.*signoz_logs.distributed_tag_attributes.*", + ).WithArgs(string(expectedAttrib.Key), v3.TagType(expectedAttrib.Type), 1).WillReturnRows(mockhouse.NewRows(cols, values)) +} + +type FilterSuggestionsTestBed struct { + t *testing.T + testUser *model.User + qsHttpHandler http.Handler + mockClickhouse mockhouse.ClickConnMockCommon +} + +func (tb *FilterSuggestionsTestBed) GetQBFilterSuggestionsForLogs( + queryParams map[string]string, +) *v3.QBFilterSuggestionsResponse { + + _, dsExistsInQP := queryParams["dataSource"] + require.False(tb.t, dsExistsInQP) + queryParams["dataSource"] = "logs" + + result := tb.QSGetRequest("/api/v3/filter_suggestions", queryParams) + + dataJson, err := json.Marshal(result.Data) + if err != nil { + tb.t.Fatalf("could not marshal apiResponse.Data: %v", err) + } + + var resp v3.QBFilterSuggestionsResponse + err = json.Unmarshal(dataJson, &resp) + if err != nil { + tb.t.Fatalf("could not unmarshal apiResponse.Data json into PipelinesResponse") + } + + return &resp +} + +func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed { + testDB := utils.NewQueryServiceDBForTests(t) + + fm := featureManager.StartManager() + reader, mockClickhouse := NewMockClickhouseReader(t, testDB, fm) + mockClickhouse.MatchExpectationsInOrder(false) + + apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ + Reader: reader, + AppDao: dao.DB(), + FeatureFlags: fm, + }) + if err != nil { + t.Fatalf("could not create a new ApiHandler: %v", err) + } + + router := app.NewRouter() + am := app.NewAuthMiddleware(auth.GetUserFromRequest) + apiHandler.RegisterRoutes(router, am) + apiHandler.RegisterQueryRangeV3Routes(router, am) + + user, apiErr := createTestUser() + if apiErr != nil { + t.Fatalf("could not create a test user: %v", apiErr) + } + + return &FilterSuggestionsTestBed{ + t: t, + testUser: user, + qsHttpHandler: router, + mockClickhouse: mockClickhouse, + } +} + +func (tb *FilterSuggestionsTestBed) QSGetRequest( + path string, + queryParams map[string]string, +) *app.ApiResponse { + if len(queryParams) > 0 { + qps := []string{} + for q, v := range queryParams { + qps = append(qps, fmt.Sprintf("%s=%s", q, v)) + } + path = fmt.Sprintf("%s?%s", path, strings.Join(qps, "&")) + } + + req, err := AuthenticatedRequestForTest( + tb.testUser, path, nil, + ) + if err != nil { + tb.t.Fatalf("couldn't create authenticated test request: %v", err) + } + + result, err := HandleTestRequest(tb.qsHttpHandler, req, 200) + if err != nil { + tb.t.Fatalf("test request failed: %v", err) + } + return result +} diff --git a/pkg/query-service/tests/integration/logparsingpipeline_test.go b/pkg/query-service/tests/integration/logparsingpipeline_test.go index 93efe30dc2..e06f7a280b 100644 --- a/pkg/query-service/tests/integration/logparsingpipeline_test.go +++ b/pkg/query-service/tests/integration/logparsingpipeline_test.go @@ -512,7 +512,7 @@ func (tb *LogPipelinesTestBed) PostPipelinesToQSExpectingStatusCode( postablePipelines logparsingpipeline.PostablePipelines, expectedStatusCode int, ) *logparsingpipeline.PipelinesResponse { - req, err := NewAuthenticatedTestRequest( + req, err := AuthenticatedRequestForTest( tb.testUser, "/api/v1/logs/pipelines", postablePipelines, ) if err != nil { @@ -562,7 +562,7 @@ func (tb *LogPipelinesTestBed) PostPipelinesToQS( } func (tb *LogPipelinesTestBed) GetPipelinesFromQS() *logparsingpipeline.PipelinesResponse { - req, err := NewAuthenticatedTestRequest( + req, err := AuthenticatedRequestForTest( tb.testUser, "/api/v1/logs/pipelines/latest", nil, ) if err != nil { diff --git a/pkg/query-service/tests/integration/signoz_integrations_test.go b/pkg/query-service/tests/integration/signoz_integrations_test.go index eae9603888..20b55a5551 100644 --- a/pkg/query-service/tests/integration/signoz_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_integrations_test.go @@ -3,10 +3,7 @@ package tests import ( "encoding/json" "fmt" - "io" "net/http" - "net/http/httptest" - "runtime/debug" "slices" "testing" "time" @@ -501,38 +498,18 @@ func (tb *IntegrationsTestBed) RequestQS( path string, postData interface{}, ) *app.ApiResponse { - req, err := NewAuthenticatedTestRequest( + req, err := AuthenticatedRequestForTest( tb.testUser, path, postData, ) if err != nil { tb.t.Fatalf("couldn't create authenticated test request: %v", err) } - respWriter := httptest.NewRecorder() - tb.qsHttpHandler.ServeHTTP(respWriter, req) - response := respWriter.Result() - responseBody, err := io.ReadAll(response.Body) + result, err := HandleTestRequest(tb.qsHttpHandler, req, 200) if err != nil { - tb.t.Fatalf("couldn't read response body received from QS: %v", err) + tb.t.Fatalf("test request failed: %v", err) } - - if response.StatusCode != 200 { - tb.t.Fatalf( - "unexpected response status from query service for path %s. status: %d, body: %v\n%v", - path, response.StatusCode, string(responseBody), string(debug.Stack()), - ) - } - - var result app.ApiResponse - err = json.Unmarshal(responseBody, &result) - if err != nil { - tb.t.Fatalf( - "Could not unmarshal QS response into an ApiResponse.\nResponse body: %s", - string(responseBody), - ) - } - - return &result + return result } func (tb *IntegrationsTestBed) mockLogQueryResponse(logsInResponse []model.SignozLog) { diff --git a/pkg/query-service/tests/integration/test_utils.go b/pkg/query-service/tests/integration/test_utils.go index 7775171310..65140e5fc8 100644 --- a/pkg/query-service/tests/integration/test_utils.go +++ b/pkg/query-service/tests/integration/test_utils.go @@ -5,8 +5,10 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" + "runtime/debug" "testing" "time" @@ -16,6 +18,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry" mockhouse "github.com/srikanthccv/ClickHouse-go-mock" "github.com/stretchr/testify/require" + "go.signoz.io/signoz/pkg/query-service/app" "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/constants" @@ -172,7 +175,7 @@ func createTestUser() (*model.User, *model.ApiError) { ) } -func NewAuthenticatedTestRequest( +func AuthenticatedRequestForTest( user *model.User, path string, postData interface{}, @@ -198,3 +201,31 @@ func NewAuthenticatedTestRequest( req.Header.Add("Authorization", "Bearer "+userJwt.AccessJwt) return req, nil } + +func HandleTestRequest(handler http.Handler, req *http.Request, expectedStatus int) (*app.ApiResponse, error) { + respWriter := httptest.NewRecorder() + handler.ServeHTTP(respWriter, req) + response := respWriter.Result() + responseBody, err := io.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("couldn't read response body received from QS: %w", err) + } + + if response.StatusCode != expectedStatus { + return nil, fmt.Errorf( + "unexpected response status from query service for path %s. status: %d, body: %v\n%v", + req.URL.Path, response.StatusCode, string(responseBody), string(debug.Stack()), + ) + } + + var result app.ApiResponse + err = json.Unmarshal(responseBody, &result) + if err != nil { + return nil, fmt.Errorf( + "Could not unmarshal QS response into an ApiResponse.\nResponse body: %s", + string(responseBody), + ) + } + + return &result, nil +}