GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-12 15:19:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: limit offset logic updated for logs list view (#3243)

Browse Source
This commit is contained in:
Nityananda Gohain 2023-08-02 13:06:41 +05:30 committed by GitHub
parent 2cdafa0564
commit 68ab022836
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 128 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/query-service/app/logs/v3/query_builder.go
View File

@ -413,6 +413,13 @@ type Options struct {
IsLivetailQuery bool IsLivetailQuery bool
} }
func isOrderByTs(orderBy []v3.OrderBy) bool {
if len(orderBy) == 1 && orderBy[0].Key == constants.TIMESTAMP {
return true
}
return false
}
func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) {
if options.IsLivetailQuery { if options.IsLivetailQuery {
query, err := buildLogsLiveTailQuery(mq) query, err := buildLogsLiveTailQuery(mq)
@ -446,12 +453,23 @@ func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.Pan
} }
if panelType == v3.PanelTypeList { if panelType == v3.PanelTypeList {
if mq.PageSize > 0 { // check if limit exceeded
if mq.Limit > 0 && mq.Offset > mq.Limit { if mq.Limit > 0 && mq.Offset >= mq.Limit {
return "", fmt.Errorf("max limit exceeded") return "", fmt.Errorf("max limit exceeded")
} }
if mq.PageSize > 0 {
if mq.Limit > 0 && mq.Offset+mq.PageSize > mq.Limit {
query = addLimitToQuery(query, mq.Limit-mq.Offset)
} else {
query = addLimitToQuery(query, mq.PageSize) query = addLimitToQuery(query, mq.PageSize)
}
// add offset to the query only if it is not orderd by timestamp.
if !isOrderByTs(mq.OrderBy) {
query = addOffsetToQuery(query, mq.Offset) query = addOffsetToQuery(query, mq.Offset)
}
} else { } else {
query = addLimitToQuery(query, mq.Limit) query = addLimitToQuery(query, mq.Limit)
} }

106
pkg/query-service/app/logs/v3/query_builder_test.go
View File

@ -1185,3 +1185,109 @@ func TestPrepareLogsQuery(t *testing.T) {
}) })
} }
} }
var testPrepLogsQueryLimitOffsetData = []struct {
Name string
PanelType v3.PanelType
Start int64
End int64
Step int64
BuilderQuery *v3.BuilderQuery
GroupByTags []v3.AttributeKey
TableName string
AggregateOperator v3.AggregateOperator
ExpectedQuery string
Options Options
}{
{
Name: "Test limit less than pageSize - order by ts",
PanelType: v3.PanelTypeList,
Start: 1680518666000000000,
End: 1691618704365000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateOperator: v3.AggregateOperatorNoOp,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
OrderBy: []v3.OrderBy{{ColumnName: constants.TIMESTAMP, Order: "desc", Key: constants.TIMESTAMP, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}},
Limit: 1,
Offset: 0,
PageSize: 5,
},
TableName: "logs",
ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) order by timestamp desc LIMIT 1",
},
{
Name: "Test limit greater than pageSize - order by ts",
PanelType: v3.PanelTypeList,
Start: 1680518666000000000,
End: 1691618704365000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateOperator: v3.AggregateOperatorNoOp,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "id", Type: v3.AttributeKeyTypeUnspecified, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Operator: v3.FilterOperatorLessThan, Value: "2TNh4vp2TpiWyLt3SzuadLJF2s4"},
}},
OrderBy: []v3.OrderBy{{ColumnName: constants.TIMESTAMP, Order: "desc", Key: constants.TIMESTAMP, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}},
Limit: 100,
Offset: 10,
PageSize: 10,
},
TableName: "logs",
ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by timestamp desc LIMIT 10",
},
{
Name: "Test limit less than pageSize - order by custom",
PanelType: v3.PanelTypeList,
Start: 1680518666000000000,
End: 1691618704365000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateOperator: v3.AggregateOperatorNoOp,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "desc", Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
Limit: 1,
Offset: 0,
PageSize: 5,
},
TableName: "logs",
ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 1 OFFSET 0",
},
{
Name: "Test limit greater than pageSize - order by custom",
PanelType: v3.PanelTypeList,
Start: 1680518666000000000,
End: 1691618704365000000,
Step: 60,
BuilderQuery: &v3.BuilderQuery{
QueryName: "A",
AggregateOperator: v3.AggregateOperatorNoOp,
Expression: "A",
Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "id", Type: v3.AttributeKeyTypeUnspecified, DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Operator: v3.FilterOperatorLessThan, Value: "2TNh4vp2TpiWyLt3SzuadLJF2s4"},
}},
OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "desc", Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
Limit: 100,
Offset: 50,
PageSize: 50,
},
TableName: "logs",
ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680518666000000000 AND timestamp <= 1691618704365000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 50 OFFSET 50",
},
}
func TestPrepareLogsQueryLimitOffset(t *testing.T) {
for _, tt := range testPrepLogsQueryLimitOffsetData {
Convey("TestBuildLogsQuery", t, func() {
query, err := PrepareLogsQuery(tt.Start, tt.End, "", tt.PanelType, tt.BuilderQuery, tt.Options)
So(err, ShouldBeNil)
So(query, ShouldEqual, tt.ExpectedQuery)
})
}
}