From eb28ece68049ca32548486129a684124def13dcf Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 10 Aug 2022 14:27:46 +0530 Subject: [PATCH] parser updated for pagination --- .../app/clickhouseReader/reader.go | 12 +++- pkg/query-service/app/http_handler.go | 2 +- pkg/query-service/app/logs/parser.go | 38 +++++++---- pkg/query-service/app/logs/parser_test.go | 65 ++++++++++++++++++- pkg/query-service/app/parser.go | 4 +- pkg/query-service/model/queryParams.go | 4 +- 6 files changed, 104 insertions(+), 21 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index dabdd3df20..1367c51a27 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3047,6 +3047,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter return nil, apiErr } + isPaginatePrev := logs.CheckIfPrevousPaginateAndModifyOrder(params) filterSql, err := logs.GenerateSQLWhere(fields, params) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} @@ -3064,7 +3065,12 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } - + if isPaginatePrev { + // rever the results from db + for i, j := 0, len(response)-1; i < j; i, j = i+1, j-1 { + response[i], response[j] = response[j], response[i] + } + } return &response, nil } @@ -3093,8 +3099,8 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } var idStart string - if client.Filter.IdStart != "" { - idStart = client.Filter.IdStart + if client.Filter.IdGt != "" { + idStart = client.Filter.IdGt } ticker := time.NewTicker(time.Duration(r.liveTailRefreshSeconds) * time.Second) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index ae90926421..86f74e9457 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1902,7 +1902,7 @@ func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter.HandleFunc("", ViewAccess(aH.getLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/tail", ViewAccess(aH.tailLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet) - subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost) + subRouter.HandleFunc("/fields", EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) subRouter.HandleFunc("/aggregate", ViewAccess(aH.logAggregate)).Methods(http.MethodGet) } diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 103e959d4a..53b168b17f 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -28,8 +28,11 @@ const ( ORDER_BY = "orderBy" TIMESTAMP_START = "timestampStart" TIMESTAMP_END = "timestampEnd" - IDSTART = "idStart" - IDEND = "idEnd" + IdGt = "idGt" + IdLT = "idLt" + TIMESTAMP = "timestamp" + ASC = "asc" + DESC = "desc" ) var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([^(]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) '[^']+')`) @@ -72,11 +75,11 @@ func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { } res.TimestampEnd = uint64(ts) } - if val, ok := params[IDSTART]; ok { - res.IdStart = val[0] + if val, ok := params[IdGt]; ok { + res.IdGt = val[0] } - if val, ok := params[IDEND]; ok { - res.IdEnd = val[0] + if val, ok := params[IdLT]; ok { + res.IdLT = val[0] } return &res, nil } @@ -94,8 +97,8 @@ func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error) } res.TimestampStart = uint64(ts) } - if val, ok := params[IDSTART]; ok { - res.IdStart = val[0] + if val, ok := params[IdGt]; ok { + res.IdGt = val[0] } return &res, nil } @@ -246,6 +249,17 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens [] return queryTokens, nil } +func CheckIfPrevousPaginateAndModifyOrder(params *model.LogsFilterParams) (isPaginatePrevious bool) { + if params.IdGt != "" && params.OrderBy == TIMESTAMP && params.Order == DESC { + isPaginatePrevious = true + params.Order = ASC + } else if params.IdLT != "" && params.OrderBy == TIMESTAMP && params.Order == ASC { + isPaginatePrevious = true + params.Order = DESC + } + return +} + func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, error) { var tokens []string var err error @@ -277,15 +291,15 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt } filterTokens = append(filterTokens, filter) } - if params.IdStart != "" { - filter := fmt.Sprintf("id > '%v' ", params.IdStart) + if params.IdGt != "" { + filter := fmt.Sprintf("id > '%v' ", params.IdGt) if len(filterTokens) > 0 { filter = "and " + filter } filterTokens = append(filterTokens, filter) } - if params.IdEnd != "" { - filter := fmt.Sprintf("id < '%v' ", params.IdEnd) + if params.IdLT != "" { + filter := fmt.Sprintf("id < '%v' ", params.IdLT) if len(filterTokens) > 0 { filter = "and " + filter } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index df35ffbe80..ff47632a4b 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -208,6 +208,69 @@ func TestReplaceInterestingFields(t *testing.T) { }) } +var previousPaginateTestCases = []struct { + Name string + Filter model.LogsFilterParams + IsPaginatePrev bool + Order string +}{ + { + Name: "empty", + Filter: model.LogsFilterParams{}, + IsPaginatePrev: false, + }, + { + Name: "next ordery by asc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: ASC, + IdGt: "myid", + }, + IsPaginatePrev: false, + Order: ASC, + }, + { + Name: "next ordery by desc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: DESC, + IdLT: "myid", + }, + IsPaginatePrev: false, + Order: DESC, + }, + { + Name: "prev ordery by desc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: DESC, + IdGt: "myid", + }, + IsPaginatePrev: true, + Order: ASC, + }, + { + Name: "prev ordery by asc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: ASC, + IdLT: "myid", + }, + IsPaginatePrev: true, + Order: DESC, + }, +} + +func TestCheckIfPrevousPaginateAndModifyOrder(t *testing.T) { + for _, test := range previousPaginateTestCases { + Convey(test.Name, t, func() { + isPrevPaginate := CheckIfPrevousPaginateAndModifyOrder(&test.Filter) + So(isPrevPaginate, ShouldEqual, test.IsPaginatePrev) + So(test.Order, ShouldEqual, test.Filter.Order) + }) + } +} + func TestGenerateSQLQuery(t *testing.T) { allFields := model.GetFieldsResponse{ Selected: []model.LogField{ @@ -233,7 +296,7 @@ func TestGenerateSQLQuery(t *testing.T) { idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU" sqlWhere := "timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' and id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 " Convey("testGenerateSQL", t, func() { - res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdStart: idStart, IdEnd: idEnd}) + res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdGt: idStart, IdLT: idEnd}) So(res, ShouldEqual, sqlWhere) }) } diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index 1e2935c06f..e7291e67a9 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -591,7 +591,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) { // Validate the type parameter if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL { - return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL) + return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL) } // Validate the TTL duration. @@ -630,7 +630,7 @@ func parseGetTTL(r *http.Request) (*model.GetTTLParams, error) { } else { // Validate the type parameter if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL { - return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL) + return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL) } } diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index c8cfa2dcee..33827d63c1 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -337,8 +337,8 @@ type LogsFilterParams struct { Query string `json:"q"` TimestampStart uint64 `json:"timestampStart"` TimestampEnd uint64 `json:"timestampEnd"` - IdStart string `json:"idStart"` - IdEnd string `json:"idEnd"` + IdGt string `json:"idGt"` + IdLT string `json:"idLt"` } type LogsAggregateParams struct {