From fd6f0574f526136127a205ceb880509bbbc75c52 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 6 Aug 2024 20:24:06 +0530 Subject: [PATCH] fix: make timeshift work with cache (#5646) --- pkg/query-service/app/querier/querier_test.go | 28 +++++++++---------- pkg/query-service/app/querier/v2/helper.go | 2 +- .../app/querier/v2/querier_test.go | 24 ++++++++-------- .../app/queryBuilder/query_builder.go | 8 ++++++ 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 5160c564da..962ca3832a 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -758,8 +758,8 @@ func TestQueryRangeTimeShift(t *testing.T) { func TestQueryRangeTimeShiftWithCache(t *testing.T) { params := []*v3.QueryRangeParamsV3{ { - Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 - End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 + Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23 + End: 1675115596722 + 120*60*1000 - 86400*1000, //30th Jan, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, @@ -785,8 +785,8 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) { }, }, { - Start: 1675115596722, //31, 3:23 - End: 1675115596722 + 120*60*1000, //31, 5:23 + Start: 1675115596722, //31st Jan, 3:23 + End: 1675115596722 + 120*60*1000, //31st Jan, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, @@ -824,8 +824,8 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) { { Labels: map[string]string{}, Points: []v3.Point{ - {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, - {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, + {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23 + {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23 }, }, }, @@ -835,7 +835,7 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) { // logs queries are generates in ns expectedTimeRangeInQueryString := []string{ fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), - fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), + fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000), } for i, param := range params { @@ -856,8 +856,8 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) { func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) { params := []*v3.QueryRangeParamsV3{ { - Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 - End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 + Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23 + End: 1675115596722 + 120*60*1000 - 86400*1000, //30th Jan, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, @@ -884,8 +884,8 @@ func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) { }, }, { - Start: 1675115596722, //31, 3:23 - End: 1675115596722 + 120*60*1000, //31, 5:23 + Start: 1675115596722, //31st Jan, 3:23 + End: 1675115596722 + 120*60*1000, //31st Jan, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, @@ -924,8 +924,8 @@ func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) { { Labels: map[string]string{}, Points: []v3.Point{ - {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, - {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, + {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23 + {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23 }, }, }, @@ -935,7 +935,7 @@ func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) { // logs queries are generates in ns expectedTimeRangeInQueryString := []string{ fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), - fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), + fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000), } for i, param := range params { diff --git a/pkg/query-service/app/querier/v2/helper.go b/pkg/query-service/app/querier/v2/helper.go index 9df9965b5c..9ee90fb913 100644 --- a/pkg/query-service/app/querier/v2/helper.go +++ b/pkg/query-service/app/querier/v2/helper.go @@ -306,7 +306,7 @@ func (q *querier) runBuilderQuery( } // response doesn't need everything - filterCachedPoints(mergedSeries, params.Start, params.End) + filterCachedPoints(mergedSeries, start, end) ch <- channelResult{ Err: nil, diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index d29785b310..174f9ccd17 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -766,7 +766,7 @@ func TestV2QueryRangeTimeShift(t *testing.T) { func TestV2QueryRangeTimeShiftWithCache(t *testing.T) { params := []*v3.QueryRangeParamsV3{ { - Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 + Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23 End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ @@ -793,8 +793,8 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) { }, }, { - Start: 1675115596722, //31, 3:23 - End: 1675115596722 + 120*60*1000, //31, 5:23 + Start: 1675115596722, //31st Jan, 3:23 + End: 1675115596722 + 120*60*1000, //31st Jan, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, @@ -832,8 +832,8 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) { { Labels: map[string]string{}, Points: []v3.Point{ - {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, - {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, + {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23 + {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23 }, }, }, @@ -843,7 +843,7 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) { // logs queries are generates in ns expectedTimeRangeInQueryString := []string{ fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), - fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), + fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000), } for i, param := range params { @@ -864,7 +864,7 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) { func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) { params := []*v3.QueryRangeParamsV3{ { - Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 + Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23 End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ @@ -892,8 +892,8 @@ func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) { }, }, { - Start: 1675115596722, //31, 3:23 - End: 1675115596722 + 120*60*1000, //31, 5:23 + Start: 1675115596722, //31st Jan, 3:23 + End: 1675115596722 + 120*60*1000, //31st Jan, 5:23 Step: 5 * time.Minute.Milliseconds(), CompositeQuery: &v3.CompositeQuery{ QueryType: v3.QueryTypeBuilder, @@ -932,8 +932,8 @@ func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) { { Labels: map[string]string{}, Points: []v3.Point{ - {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, - {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, + {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23 + {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23 }, }, }, @@ -943,7 +943,7 @@ func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) { // logs queries are generates in ns expectedTimeRangeInQueryString := []string{ fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), - fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), + fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000), } for i, param := range params { diff --git a/pkg/query-service/app/queryBuilder/query_builder.go b/pkg/query-service/app/queryBuilder/query_builder.go index a3a957c985..2237c10f83 100644 --- a/pkg/query-service/app/queryBuilder/query_builder.go +++ b/pkg/query-service/app/queryBuilder/query_builder.go @@ -346,6 +346,10 @@ func (c *cacheKeyGenerator) GenerateKeys(params *v3.QueryRangeParamsV3) map[stri parts = append(parts, fmt.Sprintf("aggregate=%s", query.AggregateOperator)) parts = append(parts, fmt.Sprintf("limit=%d", query.Limit)) + if query.ShiftBy != 0 { + parts = append(parts, fmt.Sprintf("shiftBy=%d", query.ShiftBy)) + } + if query.AggregateAttribute.Key != "" { parts = append(parts, fmt.Sprintf("aggregateAttribute=%s", query.AggregateAttribute.CacheKey())) } @@ -387,6 +391,10 @@ func (c *cacheKeyGenerator) GenerateKeys(params *v3.QueryRangeParamsV3) map[stri parts = append(parts, fmt.Sprintf("timeAggregation=%s", query.TimeAggregation)) parts = append(parts, fmt.Sprintf("spaceAggregation=%s", query.SpaceAggregation)) + if query.ShiftBy != 0 { + parts = append(parts, fmt.Sprintf("shiftBy=%d", query.ShiftBy)) + } + if query.AggregateAttribute.Key != "" { parts = append(parts, fmt.Sprintf("aggregateAttribute=%s", query.AggregateAttribute.CacheKey())) }