GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-12 20:29:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: make timeshift work with cache (#5646)

Browse Source
This commit is contained in:
Srikanth Chekuri 2024-08-06 20:24:06 +05:30 committed by GitHub
parent b819a90c80
commit fd6f0574f5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 35 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/query-service/app/querier/querier_test.go
View File

@ -758,8 +758,8 @@ func TestQueryRangeTimeShift(t *testing.T) {
func TestQueryRangeTimeShiftWithCache(t *testing.T) { func TestQueryRangeTimeShiftWithCache(t *testing.T) {
params := []*v3.QueryRangeParamsV3{ params := []*v3.QueryRangeParamsV3{
{ {
Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23
End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 End: 1675115596722 + 120*60*1000 - 86400*1000, //30th Jan, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
@ -785,8 +785,8 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) {
}, },
}, },
{ {
Start: 1675115596722, //31, 3:23 Start: 1675115596722, //31st Jan, 3:23
End: 1675115596722 + 120*60*1000, //31, 5:23 End: 1675115596722 + 120*60*1000, //31st Jan, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
@ -824,8 +824,8 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) {
{ {
Labels: map[string]string{}, Labels: map[string]string{},
Points: []v3.Point{ Points: []v3.Point{
{Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23
{Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23
}, },
}, },
}, },
@ -835,7 +835,7 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) {
// logs queries are generates in ns // logs queries are generates in ns
expectedTimeRangeInQueryString := []string{ expectedTimeRangeInQueryString := []string{
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000),
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000),
} }
for i, param := range params { for i, param := range params {
@ -856,8 +856,8 @@ func TestQueryRangeTimeShiftWithCache(t *testing.T) {
func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) { func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
params := []*v3.QueryRangeParamsV3{ params := []*v3.QueryRangeParamsV3{
{ {
Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23
End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 End: 1675115596722 + 120*60*1000 - 86400*1000, //30th Jan, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
@ -884,8 +884,8 @@ func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
}, },
}, },
{ {
Start: 1675115596722, //31, 3:23 Start: 1675115596722, //31st Jan, 3:23
End: 1675115596722 + 120*60*1000, //31, 5:23 End: 1675115596722 + 120*60*1000, //31st Jan, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
@ -924,8 +924,8 @@ func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
{ {
Labels: map[string]string{}, Labels: map[string]string{},
Points: []v3.Point{ Points: []v3.Point{
{Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23
{Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23
}, },
}, },
}, },
@ -935,7 +935,7 @@ func TestQueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
// logs queries are generates in ns // logs queries are generates in ns
expectedTimeRangeInQueryString := []string{ expectedTimeRangeInQueryString := []string{
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000),
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000),
} }
for i, param := range params { for i, param := range params {

2
pkg/query-service/app/querier/v2/helper.go
View File

@ -306,7 +306,7 @@ func (q *querier) runBuilderQuery(
} }
// response doesn't need everything // response doesn't need everything
filterCachedPoints(mergedSeries, params.Start, params.End) filterCachedPoints(mergedSeries, start, end)
ch <- channelResult{ ch <- channelResult{
Err: nil, Err: nil,

24
pkg/query-service/app/querier/v2/querier_test.go
View File

@ -766,7 +766,7 @@ func TestV2QueryRangeTimeShift(t *testing.T) {
func TestV2QueryRangeTimeShiftWithCache(t *testing.T) { func TestV2QueryRangeTimeShiftWithCache(t *testing.T) {
params := []*v3.QueryRangeParamsV3{ params := []*v3.QueryRangeParamsV3{
{ {
Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23
End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
@ -793,8 +793,8 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) {
}, },
}, },
{ {
Start: 1675115596722, //31, 3:23 Start: 1675115596722, //31st Jan, 3:23
End: 1675115596722 + 120*60*1000, //31, 5:23 End: 1675115596722 + 120*60*1000, //31st Jan, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
@ -832,8 +832,8 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) {
{ {
Labels: map[string]string{}, Labels: map[string]string{},
Points: []v3.Point{ Points: []v3.Point{
{Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23
{Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23
}, },
}, },
}, },
@ -843,7 +843,7 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) {
// logs queries are generates in ns // logs queries are generates in ns
expectedTimeRangeInQueryString := []string{ expectedTimeRangeInQueryString := []string{
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000),
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000),
} }
for i, param := range params { for i, param := range params {
@ -864,7 +864,7 @@ func TestV2QueryRangeTimeShiftWithCache(t *testing.T) {
func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) { func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
params := []*v3.QueryRangeParamsV3{ params := []*v3.QueryRangeParamsV3{
{ {
Start: 1675115596722 + 60*60*1000 - 86400*1000, //30, 4:23 Start: 1675115596722 + 60*60*1000 - 86400*1000, //30th Jan, 4:23
End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23 End: 1675115596722 + 120*60*1000 - 86400*1000, //30, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
@ -892,8 +892,8 @@ func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
}, },
}, },
{ {
Start: 1675115596722, //31, 3:23 Start: 1675115596722, //31st Jan, 3:23
End: 1675115596722 + 120*60*1000, //31, 5:23 End: 1675115596722 + 120*60*1000, //31st Jan, 5:23
Step: 5 * time.Minute.Milliseconds(), Step: 5 * time.Minute.Milliseconds(),
CompositeQuery: &v3.CompositeQuery{ CompositeQuery: &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
@ -932,8 +932,8 @@ func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
{ {
Labels: map[string]string{}, Labels: map[string]string{},
Points: []v3.Point{ Points: []v3.Point{
{Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, {Timestamp: 1675115596722 + 60*60*1000 - 86400*1000, Value: 1}, // 30th Jan, 4:23
{Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, {Timestamp: 1675115596722 + 120*60*1000 - 86400*1000 + 60*60*1000, Value: 2}, // 30th Jan, 6:23
}, },
}, },
}, },
@ -943,7 +943,7 @@ func TestV2QueryRangeTimeShiftWithLimitAndCache(t *testing.T) {
// logs queries are generates in ns // logs queries are generates in ns
expectedTimeRangeInQueryString := []string{ expectedTimeRangeInQueryString := []string{
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722+60*60*1000-86400*1000)*1000000, (1675115596722+120*60*1000-86400*1000)*1000000),
fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+60*60*1000)-86400*1000-1)*1000000), fmt.Sprintf("timestamp >= %d AND timestamp <= %d", (1675115596722-86400*1000)*1000000, ((1675115596722+120*60*1000)-86400*1000)*1000000),
} }
for i, param := range params { for i, param := range params {

8
pkg/query-service/app/queryBuilder/query_builder.go
View File

@ -346,6 +346,10 @@ func (c *cacheKeyGenerator) GenerateKeys(params *v3.QueryRangeParamsV3) map[stri
parts = append(parts, fmt.Sprintf("aggregate=%s", query.AggregateOperator)) parts = append(parts, fmt.Sprintf("aggregate=%s", query.AggregateOperator))
parts = append(parts, fmt.Sprintf("limit=%d", query.Limit)) parts = append(parts, fmt.Sprintf("limit=%d", query.Limit))
if query.ShiftBy != 0 {
parts = append(parts, fmt.Sprintf("shiftBy=%d", query.ShiftBy))
}
if query.AggregateAttribute.Key != "" { if query.AggregateAttribute.Key != "" {
parts = append(parts, fmt.Sprintf("aggregateAttribute=%s", query.AggregateAttribute.CacheKey())) parts = append(parts, fmt.Sprintf("aggregateAttribute=%s", query.AggregateAttribute.CacheKey()))
} }
@ -387,6 +391,10 @@ func (c *cacheKeyGenerator) GenerateKeys(params *v3.QueryRangeParamsV3) map[stri
parts = append(parts, fmt.Sprintf("timeAggregation=%s", query.TimeAggregation)) parts = append(parts, fmt.Sprintf("timeAggregation=%s", query.TimeAggregation))
parts = append(parts, fmt.Sprintf("spaceAggregation=%s", query.SpaceAggregation)) parts = append(parts, fmt.Sprintf("spaceAggregation=%s", query.SpaceAggregation))
if query.ShiftBy != 0 {
parts = append(parts, fmt.Sprintf("shiftBy=%d", query.ShiftBy))
}
if query.AggregateAttribute.Key != "" { if query.AggregateAttribute.Key != "" {
parts = append(parts, fmt.Sprintf("aggregateAttribute=%s", query.AggregateAttribute.CacheKey())) parts = append(parts, fmt.Sprintf("aggregateAttribute=%s", query.AggregateAttribute.CacheKey()))
} }