mirror of
https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz
synced 2025-08-16 12:45:53 +08:00
chore: add fill gaps for query range (#5155)
* chore: add fill gaps for query range * chore: fix empty result
This commit is contained in:
Srikanth Chekuri
GitHub
parent
a319d1ec53
commit
1645523ae9
@ -31,3 +31,26 @@ func MinAllowedStepInterval(start, end int64) int64 {
|
||||
// return the nearest lower multiple of 60
|
||||
return step - step%60
|
||||
}
|
||||
|
||||
func GCD(a, b int64) int64 {
|
||||
for b != 0 {
|
||||
a, b = b, a%b
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func LCM(a, b int64) int64 {
|
||||
return (a * b) / GCD(a, b)
|
||||
}
|
||||
|
||||
// LCMList computes the LCM of a list of int64 numbers.
|
||||
func LCMList(nums []int64) int64 {
|
||||
if len(nums) == 0 {
|
||||
return 1
|
||||
}
|
||||
result := nums[0]
|
||||
for _, num := range nums[1:] {
|
||||
result = LCM(result, num)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@ -400,6 +400,7 @@ type CompositeQuery struct {
|
||||
PanelType PanelType `json:"panelType"`
|
||||
QueryType QueryType `json:"queryType"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
FillGaps bool `json:"fillGaps,omitempty"`
|
||||
}
|
||||
|
||||
func (c *CompositeQuery) EnabledQueries() int {
|
||||
|
||||
70
pkg/query-service/postprocess/gaps.go
Normal file
70
pkg/query-service/postprocess/gaps.go
Normal file
@ -0,0 +1,70 @@
|
||||
package postprocess
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/govaluate"
|
||||
"go.signoz.io/signoz/pkg/query-service/common"
|
||||
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
|
||||
)
|
||||
|
||||
func stepIntervalForFunction(params *v3.QueryRangeParamsV3, query string) int64 {
|
||||
q := params.CompositeQuery.BuilderQueries[query]
|
||||
if q.QueryName != q.Expression {
|
||||
expression, _ := govaluate.NewEvaluableExpressionWithFunctions(q.Expression, EvalFuncs())
|
||||
steps := []int64{}
|
||||
for _, v := range expression.Vars() {
|
||||
steps = append(steps, params.CompositeQuery.BuilderQueries[v].StepInterval)
|
||||
}
|
||||
return common.LCMList(steps)
|
||||
}
|
||||
return q.StepInterval
|
||||
}
|
||||
|
||||
func fillGap(series *v3.Series, start, end, step int64) *v3.Series {
|
||||
v := make(map[int64]float64)
|
||||
for _, point := range series.Points {
|
||||
v[point.Timestamp] = point.Value
|
||||
}
|
||||
|
||||
// For all the values from start to end, find the timestamps
|
||||
// that don't have value and add zero point
|
||||
start = start - (start % (step * 1000))
|
||||
for i := start; i <= end; i += step * 1000 {
|
||||
if _, ok := v[i]; !ok {
|
||||
v[i] = 0
|
||||
}
|
||||
}
|
||||
newSeries := &v3.Series{
|
||||
Labels: series.Labels,
|
||||
LabelsArray: series.LabelsArray,
|
||||
Points: make([]v3.Point, 0),
|
||||
}
|
||||
for i := start; i <= end; i += step * 1000 {
|
||||
newSeries.Points = append(newSeries.Points, v3.Point{Timestamp: i, Value: v[i]})
|
||||
}
|
||||
return newSeries
|
||||
}
|
||||
|
||||
// TODO(srikanthccv): can WITH FILL be perfect substitute for all cases https://clickhouse.com/docs/en/sql-reference/statements/select/order-by#order-by-expr-with-fill-modifier
|
||||
func FillGaps(results []*v3.Result, params *v3.QueryRangeParamsV3) {
|
||||
for _, result := range results {
|
||||
// A `result` item in `results` contains the query result for individual query.
|
||||
// If there are no series in the result, we add empty series and `fillGap` adds all zeros
|
||||
if len(result.Series) == 0 {
|
||||
result.Series = []*v3.Series{
|
||||
{
|
||||
Labels: make(map[string]string),
|
||||
LabelsArray: make([]map[string]string, 0),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
builderQueries := params.CompositeQuery.BuilderQueries
|
||||
if builderQueries != nil {
|
||||
// The values should be added at the intervals of `step`
|
||||
step := stepIntervalForFunction(params, result.QueryName)
|
||||
for idx := range result.Series {
|
||||
result.Series[idx] = fillGap(result.Series[idx], params.Start, params.End, step)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
166
pkg/query-service/postprocess/gaps_test.go
Normal file
166
pkg/query-service/postprocess/gaps_test.go
Normal file
@ -0,0 +1,166 @@
|
||||
package postprocess
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
|
||||
)
|
||||
|
||||
func TestFillGaps(t *testing.T) {
|
||||
// Helper function to create a sample series
|
||||
createSeries := func(points []v3.Point) *v3.Series {
|
||||
return &v3.Series{
|
||||
Points: points,
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to create a sample result
|
||||
createResult := func(queryName string, series []*v3.Series) *v3.Result {
|
||||
return &v3.Result{
|
||||
QueryName: queryName,
|
||||
Series: series,
|
||||
}
|
||||
}
|
||||
|
||||
// Define test cases
|
||||
tests := []struct {
|
||||
name string
|
||||
results []*v3.Result
|
||||
params *v3.QueryRangeParamsV3
|
||||
expected []*v3.Result
|
||||
}{
|
||||
{
|
||||
name: "Single series with gaps",
|
||||
results: []*v3.Result{
|
||||
createResult("query1", []*v3.Series{
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 1000, Value: 1.0},
|
||||
{Timestamp: 3000, Value: 3.0},
|
||||
}),
|
||||
}),
|
||||
},
|
||||
params: &v3.QueryRangeParamsV3{
|
||||
Start: 1000,
|
||||
End: 5000,
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||
"query1": {
|
||||
QueryName: "query1",
|
||||
Expression: "query1",
|
||||
StepInterval: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: []*v3.Result{
|
||||
createResult("query1", []*v3.Series{
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 1000, Value: 1.0},
|
||||
{Timestamp: 2000, Value: 0.0},
|
||||
{Timestamp: 3000, Value: 3.0},
|
||||
{Timestamp: 4000, Value: 0.0},
|
||||
{Timestamp: 5000, Value: 0.0},
|
||||
}),
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Multiple series with gaps",
|
||||
results: []*v3.Result{
|
||||
createResult("query1", []*v3.Series{
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 1000, Value: 1.0},
|
||||
{Timestamp: 3000, Value: 3.0},
|
||||
}),
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 2000, Value: 2.0},
|
||||
{Timestamp: 4000, Value: 4.0},
|
||||
}),
|
||||
}),
|
||||
},
|
||||
params: &v3.QueryRangeParamsV3{
|
||||
Start: 1000,
|
||||
End: 5000,
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||
"query1": {
|
||||
QueryName: "query1",
|
||||
Expression: "query1",
|
||||
StepInterval: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: []*v3.Result{
|
||||
createResult("query1", []*v3.Series{
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 1000, Value: 1.0},
|
||||
{Timestamp: 2000, Value: 0.0},
|
||||
{Timestamp: 3000, Value: 3.0},
|
||||
{Timestamp: 4000, Value: 0.0},
|
||||
{Timestamp: 5000, Value: 0.0},
|
||||
}),
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 1000, Value: 0.0},
|
||||
{Timestamp: 2000, Value: 2.0},
|
||||
{Timestamp: 3000, Value: 0.0},
|
||||
{Timestamp: 4000, Value: 4.0},
|
||||
{Timestamp: 5000, Value: 0.0},
|
||||
}),
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Single series with no data",
|
||||
results: []*v3.Result{
|
||||
createResult("query1", []*v3.Series{
|
||||
createSeries([]v3.Point{}),
|
||||
}),
|
||||
},
|
||||
params: &v3.QueryRangeParamsV3{
|
||||
Start: 1000,
|
||||
End: 5000,
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
BuilderQueries: map[string]*v3.BuilderQuery{
|
||||
"query1": {
|
||||
QueryName: "query1",
|
||||
Expression: "query1",
|
||||
StepInterval: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: []*v3.Result{
|
||||
createResult("query1", []*v3.Series{
|
||||
createSeries([]v3.Point{
|
||||
{Timestamp: 1000, Value: 0.0},
|
||||
{Timestamp: 2000, Value: 0.0},
|
||||
{Timestamp: 3000, Value: 0.0},
|
||||
{Timestamp: 4000, Value: 0.0},
|
||||
{Timestamp: 5000, Value: 0.0},
|
||||
}),
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Execute test cases
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
FillGaps(tt.results, tt.params)
|
||||
for i, result := range tt.results {
|
||||
for j, series := range result.Series {
|
||||
for k, point := range series.Points {
|
||||
if point.Timestamp != tt.expected[i].Series[j].Points[k].Timestamp ||
|
||||
point.Value != tt.expected[i].Series[j].Points[k].Value {
|
||||
t.Errorf("Test %s failed: expected (%v, %v), got (%v, %v)", tt.name,
|
||||
tt.expected[i].Series[j].Points[k].Timestamp,
|
||||
tt.expected[i].Series[j].Points[k].Value,
|
||||
point.Timestamp, point.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -81,6 +81,9 @@ func PostProcessResult(result []*v3.Result, queryRangeParams *v3.QueryRangeParam
|
||||
if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
||||
result = removeDisabledQueries(result)
|
||||
}
|
||||
if queryRangeParams.CompositeQuery.FillGaps {
|
||||
FillGaps(result, queryRangeParams)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user