From 1645523ae93cf168dcd80b9c3b2b1f602e2b50e6 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 11 Jun 2024 19:58:52 +0530 Subject: [PATCH] chore: add fill gaps for query range (#5155) * chore: add fill gaps for query range * chore: fix empty result --- .../common/{metrics.go => query_range.go} | 23 +++ pkg/query-service/model/v3/v3.go | 1 + pkg/query-service/postprocess/gaps.go | 70 ++++++++ pkg/query-service/postprocess/gaps_test.go | 166 ++++++++++++++++++ pkg/query-service/postprocess/query_range.go | 3 + 5 files changed, 263 insertions(+) rename pkg/query-service/common/{metrics.go => query_range.go} (73%) create mode 100644 pkg/query-service/postprocess/gaps.go create mode 100644 pkg/query-service/postprocess/gaps_test.go diff --git a/pkg/query-service/common/metrics.go b/pkg/query-service/common/query_range.go similarity index 73% rename from pkg/query-service/common/metrics.go rename to pkg/query-service/common/query_range.go index 3a14e06439..c3f95207d6 100644 --- a/pkg/query-service/common/metrics.go +++ b/pkg/query-service/common/query_range.go @@ -31,3 +31,26 @@ func MinAllowedStepInterval(start, end int64) int64 { // return the nearest lower multiple of 60 return step - step%60 } + +func GCD(a, b int64) int64 { + for b != 0 { + a, b = b, a%b + } + return a +} + +func LCM(a, b int64) int64 { + return (a * b) / GCD(a, b) +} + +// LCMList computes the LCM of a list of int64 numbers. +func LCMList(nums []int64) int64 { + if len(nums) == 0 { + return 1 + } + result := nums[0] + for _, num := range nums[1:] { + result = LCM(result, num) + } + return result +} diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 731a1edcac..61272ddb2d 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -400,6 +400,7 @@ type CompositeQuery struct { PanelType PanelType `json:"panelType"` QueryType QueryType `json:"queryType"` Unit string `json:"unit,omitempty"` + FillGaps bool `json:"fillGaps,omitempty"` } func (c *CompositeQuery) EnabledQueries() int { diff --git a/pkg/query-service/postprocess/gaps.go b/pkg/query-service/postprocess/gaps.go new file mode 100644 index 0000000000..ac80c4a180 --- /dev/null +++ b/pkg/query-service/postprocess/gaps.go @@ -0,0 +1,70 @@ +package postprocess + +import ( + "github.com/SigNoz/govaluate" + "go.signoz.io/signoz/pkg/query-service/common" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func stepIntervalForFunction(params *v3.QueryRangeParamsV3, query string) int64 { + q := params.CompositeQuery.BuilderQueries[query] + if q.QueryName != q.Expression { + expression, _ := govaluate.NewEvaluableExpressionWithFunctions(q.Expression, EvalFuncs()) + steps := []int64{} + for _, v := range expression.Vars() { + steps = append(steps, params.CompositeQuery.BuilderQueries[v].StepInterval) + } + return common.LCMList(steps) + } + return q.StepInterval +} + +func fillGap(series *v3.Series, start, end, step int64) *v3.Series { + v := make(map[int64]float64) + for _, point := range series.Points { + v[point.Timestamp] = point.Value + } + + // For all the values from start to end, find the timestamps + // that don't have value and add zero point + start = start - (start % (step * 1000)) + for i := start; i <= end; i += step * 1000 { + if _, ok := v[i]; !ok { + v[i] = 0 + } + } + newSeries := &v3.Series{ + Labels: series.Labels, + LabelsArray: series.LabelsArray, + Points: make([]v3.Point, 0), + } + for i := start; i <= end; i += step * 1000 { + newSeries.Points = append(newSeries.Points, v3.Point{Timestamp: i, Value: v[i]}) + } + return newSeries +} + +// TODO(srikanthccv): can WITH FILL be perfect substitute for all cases https://clickhouse.com/docs/en/sql-reference/statements/select/order-by#order-by-expr-with-fill-modifier +func FillGaps(results []*v3.Result, params *v3.QueryRangeParamsV3) { + for _, result := range results { + // A `result` item in `results` contains the query result for individual query. + // If there are no series in the result, we add empty series and `fillGap` adds all zeros + if len(result.Series) == 0 { + result.Series = []*v3.Series{ + { + Labels: make(map[string]string), + LabelsArray: make([]map[string]string, 0), + }, + } + } + + builderQueries := params.CompositeQuery.BuilderQueries + if builderQueries != nil { + // The values should be added at the intervals of `step` + step := stepIntervalForFunction(params, result.QueryName) + for idx := range result.Series { + result.Series[idx] = fillGap(result.Series[idx], params.Start, params.End, step) + } + } + } +} diff --git a/pkg/query-service/postprocess/gaps_test.go b/pkg/query-service/postprocess/gaps_test.go new file mode 100644 index 0000000000..999176a830 --- /dev/null +++ b/pkg/query-service/postprocess/gaps_test.go @@ -0,0 +1,166 @@ +package postprocess + +import ( + "testing" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func TestFillGaps(t *testing.T) { + // Helper function to create a sample series + createSeries := func(points []v3.Point) *v3.Series { + return &v3.Series{ + Points: points, + } + } + + // Helper function to create a sample result + createResult := func(queryName string, series []*v3.Series) *v3.Result { + return &v3.Result{ + QueryName: queryName, + Series: series, + } + } + + // Define test cases + tests := []struct { + name string + results []*v3.Result + params *v3.QueryRangeParamsV3 + expected []*v3.Result + }{ + { + name: "Single series with gaps", + results: []*v3.Result{ + createResult("query1", []*v3.Series{ + createSeries([]v3.Point{ + {Timestamp: 1000, Value: 1.0}, + {Timestamp: 3000, Value: 3.0}, + }), + }), + }, + params: &v3.QueryRangeParamsV3{ + Start: 1000, + End: 5000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "query1": { + QueryName: "query1", + Expression: "query1", + StepInterval: 1, + }, + }, + }, + }, + expected: []*v3.Result{ + createResult("query1", []*v3.Series{ + createSeries([]v3.Point{ + {Timestamp: 1000, Value: 1.0}, + {Timestamp: 2000, Value: 0.0}, + {Timestamp: 3000, Value: 3.0}, + {Timestamp: 4000, Value: 0.0}, + {Timestamp: 5000, Value: 0.0}, + }), + }), + }, + }, + { + name: "Multiple series with gaps", + results: []*v3.Result{ + createResult("query1", []*v3.Series{ + createSeries([]v3.Point{ + {Timestamp: 1000, Value: 1.0}, + {Timestamp: 3000, Value: 3.0}, + }), + createSeries([]v3.Point{ + {Timestamp: 2000, Value: 2.0}, + {Timestamp: 4000, Value: 4.0}, + }), + }), + }, + params: &v3.QueryRangeParamsV3{ + Start: 1000, + End: 5000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "query1": { + QueryName: "query1", + Expression: "query1", + StepInterval: 1, + }, + }, + }, + }, + expected: []*v3.Result{ + createResult("query1", []*v3.Series{ + createSeries([]v3.Point{ + {Timestamp: 1000, Value: 1.0}, + {Timestamp: 2000, Value: 0.0}, + {Timestamp: 3000, Value: 3.0}, + {Timestamp: 4000, Value: 0.0}, + {Timestamp: 5000, Value: 0.0}, + }), + createSeries([]v3.Point{ + {Timestamp: 1000, Value: 0.0}, + {Timestamp: 2000, Value: 2.0}, + {Timestamp: 3000, Value: 0.0}, + {Timestamp: 4000, Value: 4.0}, + {Timestamp: 5000, Value: 0.0}, + }), + }), + }, + }, + { + name: "Single series with no data", + results: []*v3.Result{ + createResult("query1", []*v3.Series{ + createSeries([]v3.Point{}), + }), + }, + params: &v3.QueryRangeParamsV3{ + Start: 1000, + End: 5000, + CompositeQuery: &v3.CompositeQuery{ + BuilderQueries: map[string]*v3.BuilderQuery{ + "query1": { + QueryName: "query1", + Expression: "query1", + StepInterval: 1, + }, + }, + }, + }, + expected: []*v3.Result{ + createResult("query1", []*v3.Series{ + createSeries([]v3.Point{ + {Timestamp: 1000, Value: 0.0}, + {Timestamp: 2000, Value: 0.0}, + {Timestamp: 3000, Value: 0.0}, + {Timestamp: 4000, Value: 0.0}, + {Timestamp: 5000, Value: 0.0}, + }), + }), + }, + }, + } + + // Execute test cases + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + FillGaps(tt.results, tt.params) + for i, result := range tt.results { + for j, series := range result.Series { + for k, point := range series.Points { + if point.Timestamp != tt.expected[i].Series[j].Points[k].Timestamp || + point.Value != tt.expected[i].Series[j].Points[k].Value { + t.Errorf("Test %s failed: expected (%v, %v), got (%v, %v)", tt.name, + tt.expected[i].Series[j].Points[k].Timestamp, + tt.expected[i].Series[j].Points[k].Value, + point.Timestamp, point.Value) + } + } + } + } + }) + } +} diff --git a/pkg/query-service/postprocess/query_range.go b/pkg/query-service/postprocess/query_range.go index b6bd6461f3..890c37148e 100644 --- a/pkg/query-service/postprocess/query_range.go +++ b/pkg/query-service/postprocess/query_range.go @@ -81,6 +81,9 @@ func PostProcessResult(result []*v3.Result, queryRangeParams *v3.QueryRangeParam if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder { result = removeDisabledQueries(result) } + if queryRangeParams.CompositeQuery.FillGaps { + FillGaps(result, queryRangeParams) + } return result, nil }