GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-12 07:39:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

parser updated for pagination

Browse Source
This commit is contained in:
nityanandagohain 2022-08-10 14:27:46 +05:30
parent 0807a0ae26
commit eb28ece680
6 changed files with 104 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -3047,6 +3047,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter
return nil, apiErr
}
isPaginatePrev := logs.CheckIfPrevousPaginateAndModifyOrder(params)
filterSql, err := logs.GenerateSQLWhere(fields, params)
if err != nil {
return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData}
@ -3064,7 +3065,12 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter
if err != nil {
return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
if isPaginatePrev {
// rever the results from db
for i, j := 0, len(response)-1; i < j; i, j = i+1, j-1 {
response[i], response[j] = response[j], response[i]
}
}
return &response, nil
}
@ -3093,8 +3099,8 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC
}
var idStart string
if client.Filter.IdStart != "" {
idStart = client.Filter.IdStart
if client.Filter.IdGt != "" {
idStart = client.Filter.IdGt
}
ticker := time.NewTicker(time.Duration(r.liveTailRefreshSeconds) * time.Second)

2
pkg/query-service/app/http_handler.go
View File

@ -1902,7 +1902,7 @@ func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) {
subRouter.HandleFunc("", ViewAccess(aH.getLogs)).Methods(http.MethodGet)
subRouter.HandleFunc("/tail", ViewAccess(aH.tailLogs)).Methods(http.MethodGet)
subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet)
subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost)
subRouter.HandleFunc("/fields", EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost)
subRouter.HandleFunc("/aggregate", ViewAccess(aH.logAggregate)).Methods(http.MethodGet)
}

38
pkg/query-service/app/logs/parser.go
View File

@ -28,8 +28,11 @@ const (
ORDER_BY = "orderBy"
TIMESTAMP_START = "timestampStart"
TIMESTAMP_END = "timestampEnd"
IDSTART = "idStart"
IDEND = "idEnd"
IdGt = "idGt"
IdLT = "idLt"
TIMESTAMP = "timestamp"
ASC = "asc"
DESC = "desc"
)
var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([^(]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) '[^']+')`)
@ -72,11 +75,11 @@ func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) {
}
res.TimestampEnd = uint64(ts)
}
if val, ok := params[IDSTART]; ok {
res.IdStart = val[0]
if val, ok := params[IdGt]; ok {
res.IdGt = val[0]
}
if val, ok := params[IDEND]; ok {
res.IdEnd = val[0]
if val, ok := params[IdLT]; ok {
res.IdLT = val[0]
}
return &res, nil
}
@ -94,8 +97,8 @@ func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error)
}
res.TimestampStart = uint64(ts)
}
if val, ok := params[IDSTART]; ok {
res.IdStart = val[0]
if val, ok := params[IdGt]; ok {
res.IdGt = val[0]
}
return &res, nil
}
@ -246,6 +249,17 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens []
return queryTokens, nil
}
func CheckIfPrevousPaginateAndModifyOrder(params *model.LogsFilterParams) (isPaginatePrevious bool) {
if params.IdGt != "" && params.OrderBy == TIMESTAMP && params.Order == DESC {
isPaginatePrevious = true
params.Order = ASC
} else if params.IdLT != "" && params.OrderBy == TIMESTAMP && params.Order == ASC {
isPaginatePrevious = true
params.Order = DESC
}
return
}
func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, error) {
var tokens []string
var err error
@ -277,15 +291,15 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt
}
filterTokens = append(filterTokens, filter)
}
if params.IdStart != "" {
filter := fmt.Sprintf("id > '%v' ", params.IdStart)
if params.IdGt != "" {
filter := fmt.Sprintf("id > '%v' ", params.IdGt)
if len(filterTokens) > 0 {
filter = "and " + filter
}
filterTokens = append(filterTokens, filter)
}
if params.IdEnd != "" {
filter := fmt.Sprintf("id < '%v' ", params.IdEnd)
if params.IdLT != "" {
filter := fmt.Sprintf("id < '%v' ", params.IdLT)
if len(filterTokens) > 0 {
filter = "and " + filter
}

65
pkg/query-service/app/logs/parser_test.go
View File

@ -208,6 +208,69 @@ func TestReplaceInterestingFields(t *testing.T) {
})
}
var previousPaginateTestCases = []struct {
Name string
Filter model.LogsFilterParams
IsPaginatePrev bool
Order string
}{
{
Name: "empty",
Filter: model.LogsFilterParams{},
IsPaginatePrev: false,
},
{
Name: "next ordery by asc",
Filter: model.LogsFilterParams{
OrderBy: TIMESTAMP,
Order: ASC,
IdGt: "myid",
},
IsPaginatePrev: false,
Order: ASC,
},
{
Name: "next ordery by desc",
Filter: model.LogsFilterParams{
OrderBy: TIMESTAMP,
Order: DESC,
IdLT: "myid",
},
IsPaginatePrev: false,
Order: DESC,
},
{
Name: "prev ordery by desc",
Filter: model.LogsFilterParams{
OrderBy: TIMESTAMP,
Order: DESC,
IdGt: "myid",
},
IsPaginatePrev: true,
Order: ASC,
},
{
Name: "prev ordery by asc",
Filter: model.LogsFilterParams{
OrderBy: TIMESTAMP,
Order: ASC,
IdLT: "myid",
},
IsPaginatePrev: true,
Order: DESC,
},
}
func TestCheckIfPrevousPaginateAndModifyOrder(t *testing.T) {
for _, test := range previousPaginateTestCases {
Convey(test.Name, t, func() {
isPrevPaginate := CheckIfPrevousPaginateAndModifyOrder(&test.Filter)
So(isPrevPaginate, ShouldEqual, test.IsPaginatePrev)
So(test.Order, ShouldEqual, test.Filter.Order)
})
}
}
func TestGenerateSQLQuery(t *testing.T) {
allFields := model.GetFieldsResponse{
Selected: []model.LogField{
@ -233,7 +296,7 @@ func TestGenerateSQLQuery(t *testing.T) {
idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU"
sqlWhere := "timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' and id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 "
Convey("testGenerateSQL", t, func() {
res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdStart: idStart, IdEnd: idEnd})
res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdGt: idStart, IdLT: idEnd})
So(res, ShouldEqual, sqlWhere)
})
}

4
pkg/query-service/app/parser.go
View File

@ -591,7 +591,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) {
// Validate the type parameter
if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL {
return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL)
return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL)
}
// Validate the TTL duration.
@ -630,7 +630,7 @@ func parseGetTTL(r *http.Request) (*model.GetTTLParams, error) {
} else {
// Validate the type parameter
if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL {
return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL)
return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL)
}
}

4
pkg/query-service/model/queryParams.go
View File

@ -337,8 +337,8 @@ type LogsFilterParams struct {
Query string `json:"q"`
TimestampStart uint64 `json:"timestampStart"`
TimestampEnd uint64 `json:"timestampEnd"`
IdStart string `json:"idStart"`
IdEnd string `json:"idEnd"`
IdGt string `json:"idGt"`
IdLT string `json:"idLt"`
}
type LogsAggregateParams struct {