mirror of
https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz
synced 2025-10-17 20:41:26 +08:00
chore: add formula eval in query-service (#4402)
This commit is contained in:
parent
61977ebe86
commit
c6581782d0
4
go.mod
4
go.mod
@ -1,10 +1,10 @@
|
|||||||
module go.signoz.io/signoz
|
module go.signoz.io/signoz
|
||||||
|
|
||||||
go 1.21
|
go 1.21.3
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/ClickHouse/clickhouse-go/v2 v2.15.0
|
github.com/ClickHouse/clickhouse-go/v2 v2.15.0
|
||||||
github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb
|
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
|
||||||
github.com/SigNoz/signoz-otel-collector v0.88.9
|
github.com/SigNoz/signoz-otel-collector v0.88.9
|
||||||
github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
|
github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
|
||||||
github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
|
github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
|
||||||
|
4
go.sum
4
go.sum
@ -94,8 +94,8 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc
|
|||||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||||
github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb h1:bneLSKPf9YUSFmafKx32bynV6QrzViL/s+ZDvQxH1E4=
|
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkbj57eGXx8H3ZJ4zhmQXBnrW523ktj8=
|
||||||
github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb/go.mod h1:JznGDNg9x1cujDKa22RaQOimOvvEfy3nxzDGd8XDgmA=
|
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
|
||||||
github.com/SigNoz/prometheus v1.9.78 h1:bB3yuDrRzi/Mv00kWayR9DZbyjTuGfendSqISyDcXiY=
|
github.com/SigNoz/prometheus v1.9.78 h1:bB3yuDrRzi/Mv00kWayR9DZbyjTuGfendSqISyDcXiY=
|
||||||
github.com/SigNoz/prometheus v1.9.78/go.mod h1:MffmFu2qFILQrOHehx3D0XjYtaZMVfI+Ppeiv98x4Ww=
|
github.com/SigNoz/prometheus v1.9.78/go.mod h1:MffmFu2qFILQrOHehx3D0XjYtaZMVfI+Ppeiv98x4Ww=
|
||||||
github.com/SigNoz/signoz-otel-collector v0.88.9 h1:7bbJSXrSZcQsdEVVLsjsNXm/bWe9MhKu8qfXp8MlXQM=
|
github.com/SigNoz/signoz-otel-collector v0.88.9 h1:7bbJSXrSZcQsdEVVLsjsNXm/bWe9MhKu8qfXp8MlXQM=
|
||||||
|
256
pkg/query-service/app/formula.go
Normal file
256
pkg/query-service/app/formula.go
Normal file
@ -0,0 +1,256 @@
|
|||||||
|
package app
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/SigNoz/govaluate"
|
||||||
|
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Define the ExpressionEvalFunc type
|
||||||
|
type ExpressionEvalFunc func(*govaluate.EvaluableExpression, map[string]float64) float64
|
||||||
|
|
||||||
|
// Helper function to check if one label set is a subset of another
|
||||||
|
func isSubset(super, sub map[string]string) bool {
|
||||||
|
for k, v := range sub {
|
||||||
|
if val, ok := super[k]; !ok || val != v {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to find unique label sets
|
||||||
|
func findUniqueLabelSets(results []*v3.Result) []map[string]string {
|
||||||
|
allLabelSets := make([]map[string]string, 0)
|
||||||
|
// The size of the `results` small, It is the number of queries in the request
|
||||||
|
for _, result := range results {
|
||||||
|
// The size of the `result.Series` slice is usually small, It is the number of series in the query result.
|
||||||
|
// We will limit the number of series in the query result to order of 100-1000.
|
||||||
|
for _, series := range result.Series {
|
||||||
|
allLabelSets = append(allLabelSets, series.Labels)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sort the label sets by the number of labels in descending order
|
||||||
|
sort.Slice(allLabelSets, func(i, j int) bool {
|
||||||
|
return len(allLabelSets[i]) > len(allLabelSets[j])
|
||||||
|
})
|
||||||
|
|
||||||
|
uniqueSets := make([]map[string]string, 0)
|
||||||
|
|
||||||
|
for _, labelSet := range allLabelSets {
|
||||||
|
// If the label set is not a subset of any of the unique label sets, add it to the unique label sets
|
||||||
|
isUnique := true
|
||||||
|
for _, uniqueLabelSet := range uniqueSets {
|
||||||
|
if isSubset(uniqueLabelSet, labelSet) {
|
||||||
|
isUnique = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if isUnique {
|
||||||
|
uniqueSets = append(uniqueSets, labelSet)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return uniqueSets
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to join series on timestamp and calculate new values
|
||||||
|
func joinAndCalculate(results []*v3.Result, uniqueLabelSet map[string]string, expression *govaluate.EvaluableExpression) (*v3.Series, error) {
|
||||||
|
|
||||||
|
uniqueTimestamps := make(map[int64]struct{})
|
||||||
|
// map[queryName]map[timestamp]value
|
||||||
|
seriesMap := make(map[string]map[int64]float64)
|
||||||
|
for _, result := range results {
|
||||||
|
var matchingSeries *v3.Series
|
||||||
|
// We try to find a series that matches the label set from the current query result
|
||||||
|
for _, series := range result.Series {
|
||||||
|
if isSubset(uniqueLabelSet, series.Labels) {
|
||||||
|
matchingSeries = series
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare the seriesMap for quick lookup during evaluation
|
||||||
|
// seriesMap[queryName][timestamp]value contains the value of the series with the given queryName at the given timestamp
|
||||||
|
if matchingSeries != nil {
|
||||||
|
for _, point := range matchingSeries.Points {
|
||||||
|
if _, ok := seriesMap[result.QueryName]; !ok {
|
||||||
|
seriesMap[result.QueryName] = make(map[int64]float64)
|
||||||
|
}
|
||||||
|
seriesMap[result.QueryName][point.Timestamp] = point.Value
|
||||||
|
uniqueTimestamps[point.Timestamp] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
vars := expression.Vars()
|
||||||
|
var doesNotHaveAllVars bool
|
||||||
|
for _, v := range vars {
|
||||||
|
if _, ok := seriesMap[v]; !ok {
|
||||||
|
doesNotHaveAllVars = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// There is no series that matches the label set from all queries
|
||||||
|
// TODO: Does the lack of a series from one query mean that the result should be nil?
|
||||||
|
// Or should we interpret the series as having a value of 0 at all timestamps?
|
||||||
|
// The current behaviour with ClickHouse is to show no data
|
||||||
|
if doesNotHaveAllVars {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
resultSeries := &v3.Series{
|
||||||
|
Labels: uniqueLabelSet,
|
||||||
|
}
|
||||||
|
timestamps := make([]int64, 0)
|
||||||
|
for timestamp := range uniqueTimestamps {
|
||||||
|
timestamps = append(timestamps, timestamp)
|
||||||
|
}
|
||||||
|
sort.Slice(timestamps, func(i, j int) bool {
|
||||||
|
return timestamps[i] < timestamps[j]
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, timestamp := range timestamps {
|
||||||
|
values := make(map[string]interface{})
|
||||||
|
for queryName, series := range seriesMap {
|
||||||
|
values[queryName] = series[timestamp]
|
||||||
|
}
|
||||||
|
newValue, err := expression.Evaluate(values)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
val, ok := newValue.(float64)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("expected float64, got %T", newValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
resultSeries.Points = append(resultSeries.Points, v3.Point{
|
||||||
|
Timestamp: timestamp,
|
||||||
|
Value: val,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return resultSeries, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main function to process the Results
|
||||||
|
// A series can be "join"ed with other series if they have the same label set or one is a subset of the other.
|
||||||
|
// 1. Find all unique label sets
|
||||||
|
// 2. For each unique label set, find a series that matches the label set from each query result
|
||||||
|
// 3. Join the series on timestamp and calculate the new values
|
||||||
|
// 4. Return the new series
|
||||||
|
func processResults(results []*v3.Result, expression *govaluate.EvaluableExpression) (*v3.Result, error) {
|
||||||
|
uniqueLabelSets := findUniqueLabelSets(results)
|
||||||
|
newSeries := make([]*v3.Series, 0)
|
||||||
|
|
||||||
|
for _, labelSet := range uniqueLabelSets {
|
||||||
|
series, err := joinAndCalculate(results, labelSet, expression)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if series != nil {
|
||||||
|
newSeries = append(newSeries, series)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &v3.Result{
|
||||||
|
Series: newSeries,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var SupportedFunctions = []string{"exp", "log", "ln", "exp2", "log2", "exp10", "log10", "sqrt", "cbrt", "erf", "erfc", "lgamma", "tgamma", "sin", "cos", "tan", "asin", "acos", "atan", "degrees", "radians"}
|
||||||
|
|
||||||
|
func evalFuncs() map[string]govaluate.ExpressionFunction {
|
||||||
|
GoValuateFuncs := make(map[string]govaluate.ExpressionFunction)
|
||||||
|
// Returns e to the power of the given argument.
|
||||||
|
GoValuateFuncs["exp"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Exp(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the natural logarithm of the given argument.
|
||||||
|
GoValuateFuncs["log"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Log(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the natural logarithm of the given argument.
|
||||||
|
GoValuateFuncs["ln"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Log(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the base 2 exponential of the given argument.
|
||||||
|
GoValuateFuncs["exp2"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Exp2(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the base 2 logarithm of the given argument.
|
||||||
|
GoValuateFuncs["log2"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Log2(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the base 10 exponential of the given argument.
|
||||||
|
GoValuateFuncs["exp10"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Pow10(int(args[0].(float64))), nil
|
||||||
|
}
|
||||||
|
// Returns the base 10 logarithm of the given argument.
|
||||||
|
GoValuateFuncs["log10"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Log10(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the square root of the given argument.
|
||||||
|
GoValuateFuncs["sqrt"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Sqrt(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the cube root of the given argument.
|
||||||
|
GoValuateFuncs["cbrt"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Cbrt(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the error function of the given argument.
|
||||||
|
GoValuateFuncs["erf"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Erf(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the complementary error function of the given argument.
|
||||||
|
GoValuateFuncs["erfc"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Erfc(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the natural logarithm of the absolute value of the gamma function of the given argument.
|
||||||
|
GoValuateFuncs["lgamma"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
v, _ := math.Lgamma(args[0].(float64))
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
// Returns the gamma function of the given argument.
|
||||||
|
GoValuateFuncs["tgamma"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Gamma(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the sine of the given argument.
|
||||||
|
GoValuateFuncs["sin"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Sin(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the cosine of the given argument.
|
||||||
|
GoValuateFuncs["cos"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Cos(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the tangent of the given argument.
|
||||||
|
GoValuateFuncs["tan"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Tan(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the arcsine of the given argument.
|
||||||
|
GoValuateFuncs["asin"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Asin(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the arccosine of the given argument.
|
||||||
|
GoValuateFuncs["acos"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Acos(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the arctangent of the given argument.
|
||||||
|
GoValuateFuncs["atan"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return math.Atan(args[0].(float64)), nil
|
||||||
|
}
|
||||||
|
// Returns the argument converted from radians to degrees.
|
||||||
|
GoValuateFuncs["degrees"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return args[0].(float64) * 180 / math.Pi, nil
|
||||||
|
}
|
||||||
|
// Returns the argument converted from degrees to radians.
|
||||||
|
GoValuateFuncs["radians"] = func(args ...interface{}) (interface{}, error) {
|
||||||
|
return args[0].(float64) * math.Pi / 180, nil
|
||||||
|
}
|
||||||
|
return GoValuateFuncs
|
||||||
|
}
|
1478
pkg/query-service/app/formula_test.go
Normal file
1478
pkg/query-service/app/formula_test.go
Normal file
File diff suppressed because it is too large
Load Diff
@ -14,6 +14,7 @@ import (
|
|||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/SigNoz/govaluate"
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
_ "github.com/mattn/go-sqlite3"
|
_ "github.com/mattn/go-sqlite3"
|
||||||
@ -3223,7 +3224,13 @@ func (aH *APIHandler) queryRangeV4(ctx context.Context, queryRangeParams *v3.Que
|
|||||||
}
|
}
|
||||||
|
|
||||||
if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
||||||
postProcessResult(result, queryRangeParams)
|
result, err = postProcessResult(result, queryRangeParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
apiErrObj := &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||||
|
RespondError(w, apiErrObj, errQuriesByName)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resp := v3.QueryRangeResponse{
|
resp := v3.QueryRangeResponse{
|
||||||
@ -3260,7 +3267,7 @@ func (aH *APIHandler) QueryRangeV4(w http.ResponseWriter, r *http.Request) {
|
|||||||
// Much of this work can be done in the ClickHouse query, but we decided to do it here because:
|
// Much of this work can be done in the ClickHouse query, but we decided to do it here because:
|
||||||
// 1. Effective use of caching
|
// 1. Effective use of caching
|
||||||
// 2. Easier to add new functions
|
// 2. Easier to add new functions
|
||||||
func postProcessResult(result []*v3.Result, queryRangeParams *v3.QueryRangeParamsV3) {
|
func postProcessResult(result []*v3.Result, queryRangeParams *v3.QueryRangeParamsV3) ([]*v3.Result, error) {
|
||||||
// Having clause is not part of the clickhouse query, so we need to apply it here
|
// Having clause is not part of the clickhouse query, so we need to apply it here
|
||||||
// It's not included in the query because it doesn't work nicely with caching
|
// It's not included in the query because it doesn't work nicely with caching
|
||||||
// With this change, if you have a query with a having clause, and then you change the having clause
|
// With this change, if you have a query with a having clause, and then you change the having clause
|
||||||
@ -3281,6 +3288,27 @@ func postProcessResult(result []*v3.Result, queryRangeParams *v3.QueryRangeParam
|
|||||||
applyReduceTo(result, queryRangeParams)
|
applyReduceTo(result, queryRangeParams)
|
||||||
// We apply the functions here it's easier to add new functions
|
// We apply the functions here it's easier to add new functions
|
||||||
applyFunctions(result, queryRangeParams)
|
applyFunctions(result, queryRangeParams)
|
||||||
|
|
||||||
|
for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {
|
||||||
|
// The way we distinguish between a formula and a query is by checking if the expression
|
||||||
|
// is the same as the query name
|
||||||
|
// TODO(srikanthccv): Update the UI to send a flag to distinguish between a formula and a query
|
||||||
|
if query.Expression != query.QueryName {
|
||||||
|
expression, err := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, evalFuncs())
|
||||||
|
// This shouldn't happen here, because it should have been caught earlier in validation
|
||||||
|
if err != nil {
|
||||||
|
zap.S().Errorf("error in expression: %s", err.Error())
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
formulaResult, err := processResults(result, expression)
|
||||||
|
if err != nil {
|
||||||
|
zap.S().Errorf("error in expression: %s", err.Error())
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result = append(result, formulaResult)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyFunctions applies functions for each query in the composite query
|
// applyFunctions applies functions for each query in the composite query
|
||||||
|
@ -985,6 +985,41 @@ func ParseQueryRangeParams(r *http.Request) (*v3.QueryRangeParamsV3, *model.ApiE
|
|||||||
|
|
||||||
if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
||||||
for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {
|
for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {
|
||||||
|
// Formula query
|
||||||
|
if query.QueryName != query.Expression {
|
||||||
|
expression, err := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, evalFuncs())
|
||||||
|
if err != nil {
|
||||||
|
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get the group keys for the vars
|
||||||
|
groupKeys := make(map[string][]string)
|
||||||
|
for _, v := range expression.Vars() {
|
||||||
|
if varQuery, ok := queryRangeParams.CompositeQuery.BuilderQueries[v]; ok {
|
||||||
|
groupKeys[v] = []string{}
|
||||||
|
for _, key := range varQuery.GroupBy {
|
||||||
|
groupKeys[v] = append(groupKeys[v], key.Key)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("unknown variable %s", v)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
params := make(map[string]interface{})
|
||||||
|
for k, v := range groupKeys {
|
||||||
|
params[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
can, _, err := expression.CanJoin(params)
|
||||||
|
if err != nil {
|
||||||
|
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !can {
|
||||||
|
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("cannot join the given group keys")}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if query.Filters == nil || len(query.Filters.Items) == 0 {
|
if query.Filters == nil || len(query.Filters.Items) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -3,7 +3,6 @@ package app
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
@ -749,7 +748,6 @@ func TestParseQueryRangeParamsDashboardVarsSubstitution(t *testing.T) {
|
|||||||
require.Error(t, apiErr)
|
require.Error(t, apiErr)
|
||||||
require.Contains(t, apiErr.Error(), tc.errMsg)
|
require.Contains(t, apiErr.Error(), tc.errMsg)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println(apiErr)
|
|
||||||
require.Nil(t, apiErr)
|
require.Nil(t, apiErr)
|
||||||
require.Equal(t, parsedQueryRangeParams.CompositeQuery.BuilderQueries["A"].Filters.Items[0].Value, tc.expectedValue[0])
|
require.Equal(t, parsedQueryRangeParams.CompositeQuery.BuilderQueries["A"].Filters.Items[0].Value, tc.expectedValue[0])
|
||||||
require.Equal(t, parsedQueryRangeParams.CompositeQuery.BuilderQueries["A"].Filters.Items[1].Value, tc.expectedValue[1])
|
require.Equal(t, parsedQueryRangeParams.CompositeQuery.BuilderQueries["A"].Filters.Items[1].Value, tc.expectedValue[1])
|
||||||
@ -862,3 +860,368 @@ func TestParseQueryRangeParamsPromQLVars(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestQueryRangeFormula(t *testing.T) {
|
||||||
|
reqCases := []struct {
|
||||||
|
desc string
|
||||||
|
compositeQuery v3.CompositeQuery
|
||||||
|
variables map[string]interface{}
|
||||||
|
expectErr bool
|
||||||
|
errMsg string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
desc: "disjoint group by keys",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}},
|
||||||
|
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "operation_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "B/A",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
errMsg: "Group keys must match or be a subset of the other but found left: [operation_name], right: [service_name]",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "identical single group by key",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "B/A",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "identical multiple group by keys",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "B/A",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "identical multiple group by keys with different order",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "operation_name"}, {Key: "service_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "B/A",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "subset group by keys",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "A/B",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "empty keys on one side",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "A/B",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "empty keys on both sides",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "A/B",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "multiple group by keys with partial overlap",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "operation_name"}, {Key: "status_code"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "A/B",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
errMsg: "Group keys must match or be a subset of the other but found left: [service_name operation_name], right: [operation_name status_code]",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "Nested Expressions with Matching Keys - Testing expressions that involve operations (e.g., addition, division) with series whose keys match or are subsets.",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "A + B",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "Nested Expressions with Matching Keys - Testing expressions that involve operations (e.g., addition, division) with series whose keys match or are subsets.",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"A": {
|
||||||
|
QueryName: "A",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}},
|
||||||
|
Expression: "A",
|
||||||
|
},
|
||||||
|
"B": {
|
||||||
|
QueryName: "B",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}},
|
||||||
|
Expression: "B",
|
||||||
|
},
|
||||||
|
"C": {
|
||||||
|
QueryName: "C",
|
||||||
|
DataSource: v3.DataSourceMetrics,
|
||||||
|
AggregateOperator: v3.AggregateOperatorSum,
|
||||||
|
AggregateAttribute: v3.AttributeKey{Key: "signoz_calls_total"},
|
||||||
|
GroupBy: []v3.AttributeKey{{Key: "service_name"}, {Key: "operation_name"}, {Key: "status_code"}},
|
||||||
|
Expression: "C",
|
||||||
|
},
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "C/(A + B)",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
desc: "Unknow variable in expression",
|
||||||
|
compositeQuery: v3.CompositeQuery{
|
||||||
|
PanelType: v3.PanelTypeGraph,
|
||||||
|
QueryType: v3.QueryTypeBuilder,
|
||||||
|
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||||
|
"F1": {
|
||||||
|
QueryName: "F1",
|
||||||
|
Expression: "A + B",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
errMsg: "unknown variable",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range reqCases {
|
||||||
|
t.Run(tc.desc, func(t *testing.T) {
|
||||||
|
|
||||||
|
queryRangeParams := &v3.QueryRangeParamsV3{
|
||||||
|
Start: time.Now().Add(-time.Hour).UnixMilli(),
|
||||||
|
End: time.Now().UnixMilli(),
|
||||||
|
Step: time.Minute.Microseconds(),
|
||||||
|
CompositeQuery: &tc.compositeQuery,
|
||||||
|
Variables: tc.variables,
|
||||||
|
}
|
||||||
|
|
||||||
|
body := &bytes.Buffer{}
|
||||||
|
err := json.NewEncoder(body).Encode(queryRangeParams)
|
||||||
|
require.NoError(t, err)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/api/v4/query_range", body)
|
||||||
|
|
||||||
|
_, apiErr := ParseQueryRangeParams(req)
|
||||||
|
if tc.expectErr {
|
||||||
|
require.Error(t, apiErr)
|
||||||
|
require.Contains(t, apiErr.Error(), tc.errMsg)
|
||||||
|
} else {
|
||||||
|
if apiErr != nil {
|
||||||
|
if apiErr.Err != nil {
|
||||||
|
t.Fatalf("unexpected error for case: %s - %v", tc.desc, apiErr.Err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.Nil(t, apiErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user