GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-13 04:39:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: metric attribute autocomplete for the aggregation type (#2263)

Browse Source
This commit is contained in:
Srikanth Chekuri 2023-03-04 00:05:16 +05:30 committed by GitHub
parent e3fee332c7
commit 6defa0ac8b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 299 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ee/query-service/app/server.go
View File

@ -231,6 +231,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e
apiHandler.RegisterRoutes(r, am)
apiHandler.RegisterMetricsRoutes(r, am)
apiHandler.RegisterLogsRoutes(r, am)
apiHandler.RegisterQueryRangeV3Routes(r, am)
c := cors.New(cors.Options{
AllowedOrigins: []string{"*"},

36
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -45,6 +45,7 @@ import (
am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
"go.signoz.io/signoz/pkg/query-service/interfaces"
"go.signoz.io/signoz/pkg/query-service/model"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
"go.signoz.io/signoz/pkg/query-service/telemetry"
"go.signoz.io/signoz/pkg/query-service/utils"
"go.uber.org/zap"
@ -3654,6 +3655,41 @@ func (r *ClickHouseReader) QueryDashboardVars(ctx context.Context, query string)
return &result, nil
}
func (r *ClickHouseReader) GetMetricAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) {
var query string
var err error
var rows driver.Rows
var response v3.AggregateAttributeResponse
query = fmt.Sprintf("SELECT DISTINCT(metric_name) from %s.%s WHERE metric_name ILIKE $1", signozMetricDBName, signozTSTableName)
if req.Limit != 0 {
query = query + fmt.Sprintf(" LIMIT %d;", req.Limit)
}
rows, err = r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText))
if err != nil {
zap.S().Error(err)
return nil, fmt.Errorf("error while executing query: %s", err.Error())
}
defer rows.Close()
var metricName string
for rows.Next() {
if err := rows.Scan(&metricName); err != nil {
return nil, fmt.Errorf("error while scanning rows: %s", err.Error())
}
key := v3.AttributeKey{
Key: metricName,
DataType: v3.AttributeKeyDataTypeNumber,
Type: v3.AttributeKeyTypeTag,
}
response.AttributeKeys = append(response.AttributeKeys, key)
}
return &response, nil
}
func (r *ClickHouseReader) CheckClickHouse(ctx context.Context) error {
rows, err := r.db.Query(ctx, "SELECT 1")
if err != nil {

35
pkg/query-service/app/http_handler.go
View File

@ -24,6 +24,7 @@ import (
"go.signoz.io/signoz/pkg/query-service/app/parser"
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
querytemplate "go.signoz.io/signoz/pkg/query-service/utils/queryTemplate"
"go.signoz.io/signoz/pkg/query-service/dao"
@ -237,6 +238,11 @@ func (aH *APIHandler) RegisterMetricsRoutes(router *mux.Router, am *AuthMiddlewa
subRouter.HandleFunc("/autocomplete/tagValue", am.ViewAccess(aH.metricAutocompleteTagValue)).Methods(http.MethodGet)
}
func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMiddleware) {
subRouter := router.PathPrefix("/api/v3").Subrouter()
subRouter.HandleFunc("/autocomplete/aggregate_attributes", am.ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet)
}
func (aH *APIHandler) Respond(w http.ResponseWriter, data interface{}) {
writeHttpResponse(w, data)
}
@ -2246,3 +2252,32 @@ func (aH *APIHandler) logAggregate(w http.ResponseWriter, r *http.Request) {
}
aH.WriteJSON(w, r, res)
}
func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r *http.Request) {
var response *v3.AggregateAttributeResponse
req, err := parseAggregateAttributeRequest(r)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return
}
switch req.DataSource {
case v3.DataSourceMetrics:
response, err = aH.reader.GetMetricAggregateAttributes(r.Context(), req)
case v3.DataSourceLogs:
// TODO: implement
case v3.DataSourceTraces:
// TODO: implement
default:
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("invalid data source")}, nil)
return
}
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return
}
aH.Respond(w, response)
}

36
pkg/query-service/app/parser.go
View File

@ -16,6 +16,7 @@ import (
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
var allowedFunctions = []string{"count", "ratePerSec", "sum", "avg", "min", "max", "p50", "p90", "p95", "p99"}
@ -412,11 +413,11 @@ func extractTagKeys(tags []model.TagQueryParam) ([]model.TagQueryParam, error) {
tag.Key = customStr[0]
}
if tag.Operator == model.ExistsOperator || tag.Operator == model.NotExistsOperator {
if customStr[1] == string(model.TagTypeString) + ")" {
if customStr[1] == string(model.TagTypeString)+")" {
tag.StringValues = []string{" "}
} else if customStr[1] ==string(model.TagTypeBool) + ")" {
} else if customStr[1] == string(model.TagTypeBool)+")" {
tag.BoolValues = []bool{true}
} else if customStr[1] == string(model.TagTypeNumber) + ")" {
} else if customStr[1] == string(model.TagTypeNumber)+")" {
tag.NumberValues = []float64{0}
} else {
return nil, fmt.Errorf("TagKey param is not valid in query")
@ -811,3 +812,32 @@ func parseFilterSet(r *http.Request) (*model.FilterSet, error) {
}
return &filterSet, nil
}
func parseAggregateAttributeRequest(r *http.Request) (*v3.AggregateAttributeRequest, error) {
var req v3.AggregateAttributeRequest
aggregateOperator := v3.AggregateOperator(r.URL.Query().Get("aggregateOperator"))
dataSource := v3.DataSource(r.URL.Query().Get("dataSource"))
aggregateAttribute := r.URL.Query().Get("searchText")
limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
if err != nil {
limit = 50
}
if err := aggregateOperator.Validate(); err != nil {
return nil, err
}
if err := dataSource.Validate(); err != nil {
return nil, err
}
req = v3.AggregateAttributeRequest{
Operator: aggregateOperator,
SearchText: aggregateAttribute,
Limit: limit,
DataSource: dataSource,
}
return &req, nil
}

91
pkg/query-service/app/parser_test.go
View File

@ -3,13 +3,16 @@ package app
import (
"bytes"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/smartystreets/assertions/should"
. "github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/assert"
"go.signoz.io/signoz/pkg/query-service/app/metrics"
"go.signoz.io/signoz/pkg/query-service/model"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
func TestParseFilterSingleFilter(t *testing.T) {
@ -58,3 +61,89 @@ func TestParseFilterNotSupportedOp(t *testing.T) {
So(err, should.BeError, "unsupported operation")
})
}
func TestParseAggregateAttrReques(t *testing.T) {
reqCases := []struct {
desc string
queryString string
expectedOperator v3.AggregateOperator
expectedDataSource v3.DataSource
expectedLimit int
expectedSearchText string
expectErr bool
errMsg string
}{
{
desc: "valid operator and data source",
queryString: "aggregateOperator=sum&dataSource=metrics&searchText=abc",
expectedOperator: v3.AggregateOperatorSum,
expectedDataSource: v3.DataSourceMetrics,
expectedLimit: 50,
expectedSearchText: "abc",
},
{
desc: "different valid operator and data source as logs",
queryString: "aggregateOperator=avg&dataSource=logs&searchText=abc",
expectedOperator: v3.AggregateOperatorAvg,
expectedDataSource: v3.DataSourceLogs,
expectedLimit: 50,
expectedSearchText: "abc",
},
{
desc: "different valid operator and with default search text and limit",
queryString: "aggregateOperator=avg&dataSource=metrics",
expectedOperator: v3.AggregateOperatorAvg,
expectedDataSource: v3.DataSourceMetrics,
expectedLimit: 50,
expectedSearchText: "",
},
{
desc: "valid operator and data source with limit",
queryString: "aggregateOperator=avg&dataSource=traces&limit=10",
expectedOperator: v3.AggregateOperatorAvg,
expectedDataSource: v3.DataSourceTraces,
expectedLimit: 10,
expectedSearchText: "",
},
{
desc: "invalid operator",
queryString: "aggregateOperator=avg1&dataSource=traces&limit=10",
expectErr: true,
errMsg: "invalid operator",
},
{
desc: "invalid data source",
queryString: "aggregateOperator=avg&dataSource=traces1&limit=10",
expectErr: true,
errMsg: "invalid data source",
},
{
desc: "invalid limit",
queryString: "aggregateOperator=avg&dataSource=traces&limit=abc",
expectedOperator: v3.AggregateOperatorAvg,
expectedDataSource: v3.DataSourceTraces,
expectedLimit: 50,
},
}
for _, reqCase := range reqCases {
r := httptest.NewRequest("GET", "/api/v3/autocomplete/aggregate_attributes?"+reqCase.queryString, nil)
aggregateAttrRequest, err := parseAggregateAttributeRequest(r)
if reqCase.expectErr {
if err == nil {
t.Errorf("expected error: %s", reqCase.errMsg)
}
if !strings.Contains(err.Error(), reqCase.errMsg) {
t.Errorf("expected error to contain: %s, got: %s", reqCase.errMsg, err.Error())
}
continue
}
if err != nil {
t.Errorf("unexpected error: %v", err)
}
assert.Equal(t, reqCase.expectedOperator, aggregateAttrRequest.Operator)
assert.Equal(t, reqCase.expectedDataSource, aggregateAttrRequest.DataSource)
assert.Equal(t, reqCase.expectedLimit, aggregateAttrRequest.Limit)
assert.Equal(t, reqCase.expectedSearchText, aggregateAttrRequest.SearchText)
}
}

1
pkg/query-service/app/server.go
View File

@ -182,6 +182,7 @@ func (s *Server) createPublicServer(api *APIHandler) (*http.Server, error) {
api.RegisterRoutes(r, am)
api.RegisterMetricsRoutes(r, am)
api.RegisterLogsRoutes(r, am)
api.RegisterQueryRangeV3Routes(r, am)
c := cors.New(cors.Options{
AllowedOrigins: []string{"*"},

2
pkg/query-service/interfaces/interface.go
View File

@ -9,6 +9,7 @@ import (
"github.com/prometheus/prometheus/util/stats"
am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
"go.signoz.io/signoz/pkg/query-service/model"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
type Reader interface {
@ -56,6 +57,7 @@ type Reader interface {
GetMetricAutocompleteTagValue(ctx context.Context, params *model.MetricAutocompleteTagParams) (*[]string, *model.ApiError)
GetMetricResult(ctx context.Context, query string) ([]*model.Series, error)
GetMetricResultEE(ctx context.Context, query string) ([]*model.Series, string, error)
GetMetricAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error)
GetTotalSpans(ctx context.Context) (uint64, error)
GetSpansInLastHeartBeatInterval(ctx context.Context) (uint64, error)

117
pkg/query-service/model/v3/v3.go
View File

@ -1,6 +1,9 @@
package v3
import "time"
import (
"fmt"
"time"
)
type DataSource string
@ -10,6 +13,15 @@ const (
DataSourceMetrics DataSource = "metrics"
)
func (d DataSource) Validate() error {
switch d {
case DataSourceTraces, DataSourceLogs, DataSourceMetrics:
return nil
default:
return fmt.Errorf("invalid data source: %s", d)
}
}
type AggregateOperator string
const (
@ -45,6 +57,44 @@ const (
AggregateOperatorHistQuant99 AggregateOperator = "hist_quantile_99"
)
func (a AggregateOperator) Validate() error {
switch a {
case AggregateOperatorNoOp,
AggregateOpeatorCount,
AggregateOperatorCountDistinct,
AggregateOperatorSum,
AggregateOperatorAvg,
AggregateOperatorMin,
AggregateOperatorMax,
AggregateOperatorP05,
AggregateOperatorP10,
AggregateOperatorP20,
AggregateOperatorP25,
AggregateOperatorP50,
AggregateOperatorP75,
AggregateOperatorP90,
AggregateOperatorP95,
AggregateOperatorP99,
AggregateOperatorRate,
AggregateOperatorSumRate,
AggregateOperatorAvgRate,
AggregateOperatorMinRate,
AggregateOperatorMaxRate,
AggregateOperatorRateSum,
AggregateOperatorRateAvg,
AggregateOperatorRateMin,
AggregateOperatorRateMax,
AggregateOperatorHistQuant50,
AggregateOperatorHistQuant75,
AggregateOperatorHistQuant90,
AggregateOperatorHistQuant95,
AggregateOperatorHistQuant99:
return nil
default:
return fmt.Errorf("invalid operator: %s", a)
}
}
type ReduceToOperator string
const (
@ -55,6 +105,15 @@ const (
ReduceToOperatorMax ReduceToOperator = "max"
)
func (r ReduceToOperator) Validate() error {
switch r {
case ReduceToOperatorLast, ReduceToOperatorSum, ReduceToOperatorAvg, ReduceToOperatorMin, ReduceToOperatorMax:
return nil
default:
return fmt.Errorf("invalid reduce to operator: %s", r)
}
}
type QueryType string
const (
@ -63,6 +122,15 @@ const (
QueryTypePromQL QueryType = "promql"
)
func (q QueryType) Validate() error {
switch q {
case QueryTypeBuilder, QueryTypeClickHouseSQL, QueryTypePromQL:
return nil
default:
return fmt.Errorf("invalid query type: %s", q)
}
}
type PanelType string
const (
@ -72,6 +140,15 @@ const (
PanelTypeList PanelType = "list"
)
func (p PanelType) Validate() error {
switch p {
case PanelTypeValue, PanelTypeGraph, PanelTypeTable, PanelTypeList:
return nil
default:
return fmt.Errorf("invalid panel type: %s", p)
}
}
// AggregateAttributeRequest is a request to fetch possible attribute keys
// for a selected aggregate operator and search text.
// The context of the selected aggregate operator is used as the
@ -104,26 +181,26 @@ type FilterAttributeKeyRequest struct {
Limit int `json:"limit"`
}
type FilterAttributeKeyDataType string
type AttributeKeyDataType string
const (
FilterAttributeKeyDataTypeString FilterAttributeKeyDataType = "string"
FilterAttributeKeyDataTypeNumber FilterAttributeKeyDataType = "number"
FilterAttributeKeyDataTypeBool FilterAttributeKeyDataType = "bool"
AttributeKeyDataTypeString AttributeKeyDataType = "string"
AttributeKeyDataTypeNumber AttributeKeyDataType = "number"
AttributeKeyDataTypeBool AttributeKeyDataType = "bool"
)
// FilterAttributeValueRequest is a request to fetch possible attribute values
// for a selected aggregate operator, aggregate attribute, filter attribute key
// and search text.
type FilterAttributeValueRequest struct {
DataSource DataSource `json:"dataSource"`
AggregateOperator AggregateOperator `json:"aggregateOperator"`
AggregateAttribute string `json:"aggregateAttribute"`
FilterAttributeKey string `json:"filterAttributeKey"`
FilterAttributeKeyDataType FilterAttributeKeyDataType `json:"filterAttributeKeyDataType"`
TagType TagType `json:"tagType"`
SearchText string `json:"searchText"`
Limit int `json:"limit"`
DataSource DataSource `json:"dataSource"`
AggregateOperator AggregateOperator `json:"aggregateOperator"`
AggregateAttribute string `json:"aggregateAttribute"`
FilterAttributeKey string `json:"filterAttributeKey"`
FilterAttributeKeyDataType AttributeKeyDataType `json:"filterAttributeKeyDataType"`
TagType TagType `json:"tagType"`
SearchText string `json:"searchText"`
Limit int `json:"limit"`
}
type AggregateAttributeResponse struct {
@ -134,10 +211,18 @@ type FilterAttributeKeyResponse struct {
AttributeKeys []AttributeKey `json:"attributeKeys"`
}
type AttributeKeyType string
const (
AttributeKeyTypeColumn AttributeKeyType = "column"
AttributeKeyTypeTag AttributeKeyType = "tag"
AttributeKeyTypeResource AttributeKeyType = "resource"
)
type AttributeKey struct {
Key string `json:"key"`
DataType string `json:"dataType"`
Type string `json:"type"` // "column" or "tag"/"attr"/"attribute" or "resource"?
Key string `json:"key"`
DataType AttributeKeyDataType `json:"dataType"`
Type AttributeKeyType `json:"type"`
}
type FilterAttributeValueResponse struct {