From ef141d2cee37f25148557b420a2f31e272a1daf6 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 12 Jul 2022 16:38:26 +0530 Subject: [PATCH 01/32] API for fields added --- .../app/clickhouseReader/options.go | 108 ++++++++------ .../app/clickhouseReader/reader.go | 138 +++++++++++++++--- pkg/query-service/app/http_handler.go | 41 ++++++ pkg/query-service/app/logs/validator.go | 40 +++++ pkg/query-service/app/server.go | 1 + pkg/query-service/constants/constants.go | 94 +++++++++--- pkg/query-service/interfaces/interface.go | 4 + pkg/query-service/model/queryParams.go | 9 ++ pkg/query-service/model/response.go | 15 ++ 9 files changed, 359 insertions(+), 91 deletions(-) create mode 100644 pkg/query-service/app/logs/validator.go diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 30f23b5cf3..fc25fb43ce 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -18,16 +18,20 @@ const ( ) const ( - defaultDatasource string = "tcp://localhost:9000" - defaultTraceDB string = "signoz_traces" - defaultOperationsTable string = "signoz_operations" - defaultIndexTable string = "signoz_index_v2" - defaultErrorTable string = "signoz_error_index" - defaulDurationTable string = "durationSortMV" - defaultSpansTable string = "signoz_spans" - defaultWriteBatchDelay time.Duration = 5 * time.Second - defaultWriteBatchSize int = 10000 - defaultEncoding Encoding = EncodingJSON + defaultDatasource string = "tcp://localhost:9000" + defaultTraceDB string = "signoz_traces" + defaultOperationsTable string = "signoz_operations" + defaultIndexTable string = "signoz_index_v2" + defaultErrorTable string = "signoz_error_index" + defaulDurationTable string = "durationSortMV" + defaultSpansTable string = "signoz_spans" + defaultLogsDB string = "signoz_logs" + defaultLogsTable string = "logs" + defaultLogAttributeKeysTable string = "logs_atrribute_keys" + defaultLogResourceKeysTable string = "logs_resource_keys" + defaultWriteBatchDelay time.Duration = 5 * time.Second + defaultWriteBatchSize int = 10000 + defaultEncoding Encoding = EncodingJSON ) const ( @@ -43,19 +47,23 @@ const ( // NamespaceConfig is Clickhouse's internal configuration data type namespaceConfig struct { - namespace string - Enabled bool - Datasource string - TraceDB string - OperationsTable string - IndexTable string - DurationTable string - SpansTable string - ErrorTable string - WriteBatchDelay time.Duration - WriteBatchSize int - Encoding Encoding - Connector Connector + namespace string + Enabled bool + Datasource string + TraceDB string + OperationsTable string + IndexTable string + DurationTable string + SpansTable string + ErrorTable string + LogsDB string + LogsTable string + LogsAttributeKeysTable string + LogsResourceKeysTable string + WriteBatchDelay time.Duration + WriteBatchSize int + Encoding Encoding + Connector Connector } // Connecto defines how to connect to the database @@ -102,19 +110,23 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s options := &Options{ primary: &namespaceConfig{ - namespace: primaryNamespace, - Enabled: true, - Datasource: datasource, - TraceDB: defaultTraceDB, - OperationsTable: defaultOperationsTable, - IndexTable: defaultIndexTable, - ErrorTable: defaultErrorTable, - DurationTable: defaulDurationTable, - SpansTable: defaultSpansTable, - WriteBatchDelay: defaultWriteBatchDelay, - WriteBatchSize: defaultWriteBatchSize, - Encoding: defaultEncoding, - Connector: defaultConnector, + namespace: primaryNamespace, + Enabled: true, + Datasource: datasource, + TraceDB: defaultTraceDB, + OperationsTable: defaultOperationsTable, + IndexTable: defaultIndexTable, + ErrorTable: defaultErrorTable, + DurationTable: defaulDurationTable, + SpansTable: defaultSpansTable, + LogsDB: defaultLogsDB, + LogsTable: defaultLogsTable, + LogsAttributeKeysTable: defaultLogAttributeKeysTable, + LogsResourceKeysTable: defaultLogResourceKeysTable, + WriteBatchDelay: defaultWriteBatchDelay, + WriteBatchSize: defaultWriteBatchSize, + Encoding: defaultEncoding, + Connector: defaultConnector, }, others: make(map[string]*namespaceConfig, len(otherNamespaces)), } @@ -122,16 +134,20 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s for _, namespace := range otherNamespaces { if namespace == archiveNamespace { options.others[namespace] = &namespaceConfig{ - namespace: namespace, - Datasource: datasource, - TraceDB: "", - OperationsTable: "", - IndexTable: "", - ErrorTable: "", - WriteBatchDelay: defaultWriteBatchDelay, - WriteBatchSize: defaultWriteBatchSize, - Encoding: defaultEncoding, - Connector: defaultConnector, + namespace: namespace, + Datasource: datasource, + TraceDB: "", + OperationsTable: "", + IndexTable: "", + ErrorTable: "", + LogsDB: "", + LogsTable: "", + LogsAttributeKeysTable: "", + LogsResourceKeysTable: "", + WriteBatchDelay: defaultWriteBatchDelay, + WriteBatchSize: defaultWriteBatchSize, + Encoding: defaultEncoding, + Connector: defaultConnector, } } else { options.others[namespace] = &namespaceConfig{namespace: namespace} diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 596354433e..bfcdbe4368 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -83,19 +83,23 @@ var ( // SpanWriter for reading spans from ClickHouse type ClickHouseReader struct { - db clickhouse.Conn - localDB *sqlx.DB - traceDB string - operationsTable string - durationTable string - indexTable string - errorTable string - spansTable string - queryEngine *promql.Engine - remoteStorage *remote.Storage - ruleManager *rules.Manager - promConfig *config.Config - alertManager am.Manager + db clickhouse.Conn + localDB *sqlx.DB + traceDB string + operationsTable string + durationTable string + indexTable string + errorTable string + spansTable string + logsDB string + logsTable string + logsAttributeKeys string + logsResourceKeys string + queryEngine *promql.Engine + remoteStorage *remote.Storage + ruleManager *rules.Manager + promConfig *config.Config + alertManager am.Manager } // NewTraceReader returns a TraceReader for the database @@ -113,15 +117,19 @@ func NewReader(localDB *sqlx.DB) *ClickHouseReader { alertManager := am.New("") return &ClickHouseReader{ - db: db, - localDB: localDB, - traceDB: options.primary.TraceDB, - alertManager: alertManager, - operationsTable: options.primary.OperationsTable, - indexTable: options.primary.IndexTable, - errorTable: options.primary.ErrorTable, - durationTable: options.primary.DurationTable, - spansTable: options.primary.SpansTable, + db: db, + localDB: localDB, + traceDB: options.primary.TraceDB, + alertManager: alertManager, + operationsTable: options.primary.OperationsTable, + indexTable: options.primary.IndexTable, + errorTable: options.primary.ErrorTable, + durationTable: options.primary.DurationTable, + spansTable: options.primary.SpansTable, + logsDB: options.primary.LogsDB, + logsTable: options.primary.LogsTable, + logsAttributeKeys: options.primary.LogsAttributeKeysTable, + logsResourceKeys: options.primary.LogsResourceKeysTable, } } @@ -2985,3 +2993,89 @@ func (r *ClickHouseReader) GetSamplesInfoInLastHeartBeatInterval(ctx context.Con return totalSamples, nil } + +func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) { + // response will contain top level fields from the otel log model + response := model.GetFieldsResponse{ + Selected: constants.StaticSelectedLogFields, + Interesting: []model.LogField{}, + } + + // get attribute keys + attributes := &[]model.LogField{} + query := fmt.Sprintf("SELECT DISTINCT name, datatype from %s.%s group by name, datatype", r.logsDB, r.logsAttributeKeys) + err := r.db.Select(ctx, attributes, query) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + // get resource keys + resources := &[]model.LogField{} + query = fmt.Sprintf("SELECT DISTINCT name, datatype from %s.%s group by name, datatype", r.logsDB, r.logsResourceKeys) + err = r.db.Select(ctx, resources, query) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + statements := []model.CreateTableStatement{} + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsTable) + err = r.db.Select(ctx, &statements, query) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + extractSelectedAndInterestingFields(statements[0].Statement, constants.Attributes, attributes, &response) + extractSelectedAndInterestingFields(statements[0].Statement, constants.Resources, resources, &response) + extractSelectedAndInterestingFields(statements[0].Statement, constants.Static, &constants.StaticInterestingLogFields, &response) + + return &response, nil +} + +func extractSelectedAndInterestingFields(tableStatement string, fieldType string, fields *[]model.LogField, response *model.GetFieldsResponse) { + for _, field := range *fields { + field.Type = fieldType + if strings.Contains(tableStatement, fmt.Sprintf("INDEX %s_idx", field.Name)) { + response.Selected = append(response.Selected, field) + } else { + response.Interesting = append(response.Interesting, field) + } + } +} + +func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError { + // if a field is selected it means that the field is indexed + if field.Selected { + // if the type is attribute or resource, create the materialized column first + if field.Type == constants.Attributes || field.Type == constants.Resources { + // create materialized + query := fmt.Sprintf("ALTER TABLE %s.%s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED %s_%s_value[indexOf(%s_%s_key, '%s')]", r.logsDB, r.logsTable, field.Name, field.DataType, field.Type, strings.ToLower(field.DataType), field.Type, strings.ToLower(field.DataType), field.Name) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } + + // create the index + if field.IndexType == nil { + iType := constants.DefaultLogSkipIndexType + field.IndexType = &iType + } + if field.IndexGranularity == nil { + granularity := constants.DefaultLogSkipIndexGranularity + field.IndexGranularity = &granularity + } + query := fmt.Sprintf("ALTER TABLE %s.%s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d", r.logsDB, r.logsTable, field.Name, field.Name, *field.IndexType, *field.IndexGranularity) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } else { + // remove index + query := fmt.Sprintf("ALTER TABLE %s.%s DROP INDEX IF EXISTS %s_idx", r.logsDB, r.logsTable, field.Name) + err := r.db.Exec(ctx, query) + if err != nil { + return &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + } + return nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 4e923af79c..dd752fa376 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -16,6 +16,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/prometheus/prometheus/promql" "go.signoz.io/query-service/app/dashboards" + "go.signoz.io/query-service/app/logs" "go.signoz.io/query-service/app/metrics" "go.signoz.io/query-service/app/parser" "go.signoz.io/query-service/auth" @@ -1816,3 +1817,43 @@ func (aH *APIHandler) writeJSON(w http.ResponseWriter, r *http.Request, response w.Header().Set("Content-Type", "application/json") w.Write(resp) } + +// logs +func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { + subRouter := router.PathPrefix("/api/v1/logs").Subrouter() + subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet) + subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost) +} + +func (aH *APIHandler) logFields(w http.ResponseWriter, r *http.Request) { + + fields, apiErr := (*aH.reader).GetLogFields(r.Context()) + if apiErr != nil { + respondError(w, apiErr, "Failed to fetch org from the DB") + return + } + aH.writeJSON(w, r, fields) +} + +func (aH *APIHandler) logFieldUpdate(w http.ResponseWriter, r *http.Request) { + field := model.UpdateField{} + if err := json.NewDecoder(r.Body).Decode(&field); err != nil { + apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} + respondError(w, apiErr, "Failed to decode payload") + return + } + + err := logs.ValidateUpdateFieldPayload(&field) + if err != nil { + apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} + respondError(w, apiErr, "Incorrect payload") + return + } + + apiErr := (*aH.reader).UpdateLogField(r.Context(), &field) + if apiErr != nil { + respondError(w, apiErr, "Failed to fetch org from the DB") + return + } + aH.writeJSON(w, r, field) +} diff --git a/pkg/query-service/app/logs/validator.go b/pkg/query-service/app/logs/validator.go new file mode 100644 index 0000000000..9277bda047 --- /dev/null +++ b/pkg/query-service/app/logs/validator.go @@ -0,0 +1,40 @@ +package logs + +import ( + "fmt" + "regexp" + + "go.signoz.io/query-service/constants" + "go.signoz.io/query-service/model" +) + +func ValidateUpdateFieldPayload(field *model.UpdateField) error { + if field.Name == "" { + return fmt.Errorf("name cannot be empty") + } + if field.Type == "" { + return fmt.Errorf("type cannot be empty") + } + if field.DataType == "" { + return fmt.Errorf("dataType cannot be empty") + } + + matched, err := regexp.MatchString(fmt.Sprintf("^(%s|%s|%s)$", constants.Static, constants.Attributes, constants.Resources), field.Type) + if err != nil { + return err + } + if !matched { + return fmt.Errorf("type %s not supported", field.Type) + } + + if field.IndexType != nil { + matched, err := regexp.MatchString(`^(minmax|set\([0-9]\)|bloom_filter\((0?.?[0-9]+|1)\)|tokenbf_v1\([0-9]+,[0-9]+,[0-9]+\)|ngrambf_v1\([0-9]+,[0-9]+,[0-9]+,[0-9]+\))$`, *field.IndexType) + if err != nil { + return err + } + if !matched { + return fmt.Errorf("index type %s not supported", *field.IndexType) + } + } + return nil +} diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 5bccea66e2..2d23531c41 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -145,6 +145,7 @@ func (s *Server) createPublicServer(api *APIHandler) (*http.Server, error) { api.RegisterRoutes(r) api.RegisterMetricsRoutes(r) + api.RegisterLogsRoutes(r) c := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index b4bc4b08ef..3de0f6ae7a 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -3,6 +3,8 @@ package constants import ( "os" "strconv" + + "go.signoz.io/query-service/model" ) const ( @@ -38,29 +40,34 @@ var AmChannelApiPath = GetOrDefaultEnv("ALERTMANAGER_API_CHANNEL_PATH", "v1/rout var RELATIONAL_DATASOURCE_PATH = GetOrDefaultEnv("SIGNOZ_LOCAL_DB_PATH", "/var/lib/signoz/signoz.db") const ( - ServiceName = "serviceName" - HttpRoute = "httpRoute" - HttpCode = "httpCode" - HttpHost = "httpHost" - HttpUrl = "httpUrl" - HttpMethod = "httpMethod" - Component = "component" - OperationDB = "name" - OperationRequest = "operation" - Status = "status" - Duration = "duration" - DBName = "dbName" - DBOperation = "dbOperation" - DBSystem = "dbSystem" - MsgSystem = "msgSystem" - MsgOperation = "msgOperation" - Timestamp = "timestamp" - Descending = "descending" - Ascending = "ascending" - ContextTimeout = 60 // seconds - StatusPending = "pending" - StatusFailed = "failed" - StatusSuccess = "success" + ServiceName = "serviceName" + HttpRoute = "httpRoute" + HttpCode = "httpCode" + HttpHost = "httpHost" + HttpUrl = "httpUrl" + HttpMethod = "httpMethod" + Component = "component" + OperationDB = "name" + OperationRequest = "operation" + Status = "status" + Duration = "duration" + DBName = "dbName" + DBOperation = "dbOperation" + DBSystem = "dbSystem" + MsgSystem = "msgSystem" + MsgOperation = "msgOperation" + Timestamp = "timestamp" + Descending = "descending" + Ascending = "ascending" + ContextTimeout = 60 // seconds + StatusPending = "pending" + StatusFailed = "failed" + StatusSuccess = "success" + Attributes = "attributes" + Resources = "resources" + Static = "static" + DefaultLogSkipIndexType = "bloom_filter(0.01)" + DefaultLogSkipIndexGranularity = 64 ) const ( SIGNOZ_METRIC_DBNAME = "signoz_metrics" @@ -75,3 +82,44 @@ func GetOrDefaultEnv(key string, fallback string) string { } return v } + +var StaticInterestingLogFields = []model.LogField{ + { + Name: "trace_id", + DataType: "String", + Type: Static, + }, + { + Name: "span_id", + DataType: "String", + Type: Static, + }, + { + Name: "trace_flags", + DataType: "UInt32", + Type: Static, + }, + { + Name: "severity_text", + DataType: "LowCardinality(String)", + Type: Static, + }, + { + Name: "severity_number", + DataType: "Int32", + Type: Static, + }, +} + +var StaticSelectedLogFields = []model.LogField{ + { + Name: "timestamp", + DataType: "UInt64", + Type: Static, + }, + { + Name: "id", + DataType: "String", + Type: Static, + }, +} diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 9c52a4497d..0bf1811b0c 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -59,4 +59,8 @@ type Reader interface { GetSpansInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetTimeSeriesInfo(ctx context.Context) (map[string]interface{}, error) GetSamplesInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) + + // Logs + GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) + UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError } diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index 69509849d2..ec2032a06e 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -303,3 +303,12 @@ type FilterSet struct { Operator string `json:"op,omitempty"` Items []FilterItem `json:"items"` } + +type UpdateField struct { + Name string `json:"name"` + DataType string `json:"dataType"` + Type string `json:"type"` + Selected bool `json:"selected"` + IndexType *string `json:"index"` + IndexGranularity *int `json:"indexGranularity"` +} diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 523ad7e96e..f61132bc94 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -373,3 +373,18 @@ func (p *MetricPoint) MarshalJSON() ([]byte, error) { v := strconv.FormatFloat(p.Value, 'f', -1, 64) return json.Marshal([...]interface{}{float64(p.Timestamp) / 1000, v}) } + +type CreateTableStatement struct { + Statement string `json:"statement" ch:"statement"` +} + +type LogField struct { + Name string `json:"name" ch:"name"` + DataType string `json:"dataType" ch:"datatype"` + Type string `json:"type"` +} + +type GetFieldsResponse struct { + Selected []LogField `json:"selected"` + Interesting []LogField `json:"interesting"` +} From ed5d217c762ec2f234ca3ed2d3ae8d6040de6424 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 13 Jul 2022 15:42:13 +0530 Subject: [PATCH 02/32] API for filtering and paginating logs added --- .../app/clickhouseReader/reader.go | 35 ++++++ pkg/query-service/app/http_handler.go | 29 ++++- pkg/query-service/app/logs/parser.go | 100 ++++++++++++++++++ pkg/query-service/app/logs/validator.go | 25 +++++ pkg/query-service/interfaces/interface.go | 1 + pkg/query-service/model/queryParams.go | 13 +++ pkg/query-service/model/response.go | 16 +++ 7 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 pkg/query-service/app/logs/parser.go diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index bfcdbe4368..ac41d049b0 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -47,6 +47,7 @@ import ( "github.com/jmoiron/sqlx" promModel "github.com/prometheus/common/model" + "go.signoz.io/query-service/app/logs" "go.signoz.io/query-service/constants" am "go.signoz.io/query-service/integrations/alertManager" "go.signoz.io/query-service/model" @@ -3079,3 +3080,37 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda } return nil } + +func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.GetLogsResponse, *model.ApiError) { + response := &[]model.GetLogsResponse{} + fields, apiErr := r.GetLogFields(ctx) + if apiErr != nil { + return nil, apiErr + } + + filterSql, err := logs.ParseLogFilter(fields, ¶ms.Filters) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} + } + + query := fmt.Sprintf("SELECT "+ + "timestamp, observed_timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ + "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,"+ + "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,"+ + "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,"+ + "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ + "from %s.%s", r.logsDB, r.logsTable) + + if filterSql != nil && *filterSql != "" { + query += fmt.Sprintf(" where %s", *filterSql) + } + + query = fmt.Sprintf("%s order by %s %s limit %d", query, params.OrderBy, params.Order, params.Limit) + zap.S().Debug(query) + err = r.db.Select(ctx, response, query) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + return response, nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index dd752fa376..c4ebeccaa9 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1821,15 +1821,15 @@ func (aH *APIHandler) writeJSON(w http.ResponseWriter, r *http.Request, response // logs func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v1/logs").Subrouter() + subRouter.HandleFunc("", ViewAccess(aH.getLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost) } func (aH *APIHandler) logFields(w http.ResponseWriter, r *http.Request) { - fields, apiErr := (*aH.reader).GetLogFields(r.Context()) if apiErr != nil { - respondError(w, apiErr, "Failed to fetch org from the DB") + respondError(w, apiErr, "Failed to fetch fields from the DB") return } aH.writeJSON(w, r, fields) @@ -1852,8 +1852,31 @@ func (aH *APIHandler) logFieldUpdate(w http.ResponseWriter, r *http.Request) { apiErr := (*aH.reader).UpdateLogField(r.Context(), &field) if apiErr != nil { - respondError(w, apiErr, "Failed to fetch org from the DB") + respondError(w, apiErr, "Failed to update filed in the DB") return } aH.writeJSON(w, r, field) } + +func (aH *APIHandler) getLogs(w http.ResponseWriter, r *http.Request) { + params, err := logs.ParseFilterParams(r) + if err != nil { + apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} + respondError(w, apiErr, "Incorrect params") + return + } + + err = logs.ValidateFilters(¶ms.Filters) + if err != nil { + apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} + respondError(w, apiErr, "Incorrect filter") + return + } + + res, apiErr := (*aH.reader).GetLogs(r.Context(), params) + if apiErr != nil { + respondError(w, apiErr, "Failed to fetch logs from the DB") + return + } + aH.writeJSON(w, r, map[string]interface{}{"results": res}) +} diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go new file mode 100644 index 0000000000..a7975cdd8d --- /dev/null +++ b/pkg/query-service/app/logs/parser.go @@ -0,0 +1,100 @@ +package logs + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "go.signoz.io/query-service/model" +) + +var operatorMapping = map[string]string{ + "eq": "=", + "neq": "!=", + "lt": "<", + "gt": ">", + "lte": "<=", + "gte": ">=", + "in": "in", + "like": "like", + "ilike": "ilike", +} + +func arrayToMap(fields []model.LogField) map[string]model.LogField { + res := map[string]model.LogField{} + for _, field := range fields { + res[field.Name] = field + } + return res +} + +func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { + res := model.LogsFilterParams{ + Limit: 30, + OrderBy: "timestamp", + Order: "desc", + } + var err error + params := r.URL.Query() + filters := []model.LogFilter{} + if val, ok := params["limit"]; ok { + res.Limit, err = strconv.Atoi(val[0]) + if err != nil { + return nil, err + } + } + if val, ok := params["orderBy"]; ok { + res.OrderBy = val[0] + } + if val, ok := params["order"]; ok { + res.Order = val[0] + } + if val, ok := params["filter"]; ok { + err := json.Unmarshal([]byte(val[0]), &filters) + if err != nil { + return nil, err + } + } + res.Filters = filters + return &res, nil +} + +func ParseLogFilter(allFields *model.GetFieldsResponse, filters *[]model.LogFilter) (*string, error) { + fLen := len(*filters) + if fLen <= 0 { + return nil, nil + } + + selectedFieldsLookup := arrayToMap(allFields.Selected) + interestingFieldLookup := arrayToMap(allFields.Interesting) + filterSql := "" + for fIndx := 0; fIndx < fLen; fIndx++ { + filter := (*filters)[fIndx] + fieldSQLName := filter.Column + if _, ok := selectedFieldsLookup[filter.Column]; !ok { + if field, ok := interestingFieldLookup[filter.Column]; ok { + fieldSQLName = fmt.Sprintf("%s_%s_value[indexOf(%s_%s_key, '%s')]", field.Type, strings.ToLower(field.DataType), field.Type, strings.ToLower(field.DataType), filter.Column) + } else { + return nil, fmt.Errorf("field not found for filtering") + } + } + + filterSql += "(" + vLen := len(filter.Value) + for i := 0; i < vLen; i++ { + filterSql += fmt.Sprintf("%s%s'%v'", fieldSQLName, operatorMapping[filter.Operation], filter.Value[i]) + if i != vLen-1 { + filterSql += " or " + } + } + filterSql += ")" + + if fIndx != fLen-1 { + filterSql += " and " + } + } + + return &filterSql, nil +} diff --git a/pkg/query-service/app/logs/validator.go b/pkg/query-service/app/logs/validator.go index 9277bda047..99858969e8 100644 --- a/pkg/query-service/app/logs/validator.go +++ b/pkg/query-service/app/logs/validator.go @@ -38,3 +38,28 @@ func ValidateUpdateFieldPayload(field *model.UpdateField) error { } return nil } + +func ValidateFilters(filters *[]model.LogFilter) error { + opsRegex := "^(gte|lte|gt|lt|eq|neq|in|like|ilike)$" + regex, err := regexp.Compile(opsRegex) + if err != nil { + return err + } + for _, val := range *filters { + if val.Column == "" { + return fmt.Errorf("col cannot be empty") + } + if val.Operation == "" { + return fmt.Errorf("op cannot be empty") + } + if len(val.Value) == 0 { + return fmt.Errorf("val cannot be empty") + } + + matched := regex.MatchString(val.Operation) + if !matched { + return fmt.Errorf("op type %s not supported", val.Operation) + } + } + return nil +} diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 0bf1811b0c..bb5a428a0e 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -63,4 +63,5 @@ type Reader interface { // Logs GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError + GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.GetLogsResponse, *model.ApiError) } diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index ec2032a06e..c39c3eb802 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -312,3 +312,16 @@ type UpdateField struct { IndexType *string `json:"index"` IndexGranularity *int `json:"indexGranularity"` } + +type LogFilter struct { + Column string `json:"col"` + Operation string `json:"op"` + Value []interface{} `json:"val"` +} + +type LogsFilterParams struct { + Limit int `json:"limit"` + OrderBy string `json:"orderBy"` + Order string `json:"order"` + Filters []LogFilter `json:"filters"` +} diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index f61132bc94..f6120f34af 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -388,3 +388,19 @@ type GetFieldsResponse struct { Selected []LogField `json:"selected"` Interesting []LogField `json:"interesting"` } + +type GetLogsResponse struct { + Timestamp uint64 `json:"timestamp" ch:"timestamp"` + ObservedTimestamp uint64 `json:"observedTimestamp" ch:"observed_timestamp"` + ID string `json:"id" ch:"id"` + TraceID string `json:"traceId" ch:"trace_id"` + SpanID string `json:"spanId" ch:"span_id"` + TraceFlags uint32 `json:"traceFlags" ch:"trace_flags"` + SeverityText string `json:"severityText" ch:"severity_text"` + SeverityNumber int32 `json:"severityNumber" ch:"severity_number"` + Body string `json:"body" ch:"body"` + Resources_string map[string]string `json:"resourcesString" ch:"resources_string"` + Attributes_string map[string]string `json:"attributesString" ch:"attributes_string"` + Attributes_int64 map[string]int64 `json:"attributesInt" ch:"attributes_int64"` + Attributes_float64 map[string]float64 `json:"attributesFloat" ch:"attributes_float64"` +} From 2e9affa80cc647cb41014eafbce992041a6b1f3f Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Jul 2022 16:37:46 +0530 Subject: [PATCH 03/32] Filtering logic updated --- .../app/clickhouseReader/reader.go | 20 +- pkg/query-service/app/http_handler.go | 43 +++- pkg/query-service/app/logs/parser.go | 179 +++++++++++---- pkg/query-service/app/logs/parser_test.go | 209 ++++++++++++++++++ pkg/query-service/app/logs/validator.go | 25 --- pkg/query-service/app/server.go | 5 + pkg/query-service/interfaces/interface.go | 1 + pkg/query-service/model/queryParams.go | 18 +- pkg/query-service/model/response.go | 31 ++- 9 files changed, 430 insertions(+), 101 deletions(-) create mode 100644 pkg/query-service/app/logs/parser_test.go diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ac41d049b0..687fcb3e46 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3088,7 +3088,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter return nil, apiErr } - filterSql, err := logs.ParseLogFilter(fields, ¶ms.Filters) + filterSql, err := logs.GenerateSQLWhere(fields, params) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} } @@ -3114,3 +3114,21 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter return response, nil } + +func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailClient) *model.ApiError { + for i := 0; i < 10; i++ { + select { + case <-ctx.Done(): + return nil + default: + data := fmt.Sprintf("hello log %d", i) + client.Logs <- &data + time.Sleep(time.Second) + } + } + done := true + client.Done <- &done + fmt.Println("done in the tail logs") + + return nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index c4ebeccaa9..92caf4ed69 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1,6 +1,7 @@ package app import ( + "bytes" "context" "encoding/json" "errors" @@ -1822,6 +1823,7 @@ func (aH *APIHandler) writeJSON(w http.ResponseWriter, r *http.Request, response func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v1/logs").Subrouter() subRouter.HandleFunc("", ViewAccess(aH.getLogs)).Methods(http.MethodGet) + subRouter.HandleFunc("/tail", ViewAccess(aH.tailLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost) } @@ -1866,13 +1868,6 @@ func (aH *APIHandler) getLogs(w http.ResponseWriter, r *http.Request) { return } - err = logs.ValidateFilters(¶ms.Filters) - if err != nil { - apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} - respondError(w, apiErr, "Incorrect filter") - return - } - res, apiErr := (*aH.reader).GetLogs(r.Context(), params) if apiErr != nil { respondError(w, apiErr, "Failed to fetch logs from the DB") @@ -1880,3 +1875,37 @@ func (aH *APIHandler) getLogs(w http.ResponseWriter, r *http.Request) { } aH.writeJSON(w, r, map[string]interface{}{"results": res}) } + +func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { + client := &model.LogsTailClient{Name: r.RemoteAddr, Logs: make(chan *string, 100), Done: make(chan *bool)} + go (*aH.reader).TailLogs(r.Context(), client) + + w.Header().Set("Connection", "keep-alive") + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Access-Control-Allow-Origin", "*") + w.WriteHeader(200) + + flusher, ok := w.(http.Flusher) + if !ok { + err := model.ApiError{Typ: model.ErrorStreamingNotSupported, Err: nil} + respondError(w, &err, "streaming is not supported") + return + } + + loop := true + for loop { + select { + case ev := <-client.Logs: + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.Encode(ev) + fmt.Fprintf(w, "data: %v\n\n", buf.String()) + fmt.Printf("data: %v\n", buf.String()) + flusher.Flush() + case <-client.Done: + fmt.Println("done!") + return + } + } +} diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index a7975cdd8d..61d1d2840b 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -1,9 +1,9 @@ package logs import ( - "encoding/json" "fmt" "net/http" + "regexp" "strconv" "strings" @@ -11,24 +11,18 @@ import ( ) var operatorMapping = map[string]string{ - "eq": "=", - "neq": "!=", - "lt": "<", - "gt": ">", - "lte": "<=", - "gte": ">=", - "in": "in", - "like": "like", - "ilike": "ilike", + "lt": "<", + "gt": ">", + "lte": "<=", + "gte": ">=", + "in": "IN", + "nin": "NOT IN", + "contains": "ILIKE", + "ncontains": "NOT ILIKE", } -func arrayToMap(fields []model.LogField) map[string]model.LogField { - res := map[string]model.LogField{} - for _, field := range fields { - res[field.Name] = field - } - return res -} +var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \(["\w.,' \-><]+\))|([\w.-]+ (gt|lt|gte|lte|contains|ncontains) ("|')?\S+("|')?))`) +var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { res := model.LogsFilterParams{ @@ -38,7 +32,6 @@ func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { } var err error params := r.URL.Query() - filters := []model.LogFilter{} if val, ok := params["limit"]; ok { res.Limit, err = strconv.Atoi(val[0]) if err != nil { @@ -51,50 +44,144 @@ func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { if val, ok := params["order"]; ok { res.Order = val[0] } - if val, ok := params["filter"]; ok { - err := json.Unmarshal([]byte(val[0]), &filters) + if val, ok := params["q"]; ok { + res.Query = &val[0] + } + if val, ok := params["timestampStart"]; ok { + ts, err := strconv.Atoi(val[0]) if err != nil { return nil, err } + ts64 := int64(ts) + res.TimestampStart = &ts64 + } + if val, ok := params["timestampEnd"]; ok { + ts, err := strconv.Atoi(val[0]) + if err != nil { + return nil, err + } + ts64 := int64(ts) + res.TimestampEnd = &ts64 + } + if val, ok := params["idStart"]; ok { + res.IdStart = &val[0] + } + if val, ok := params["idEnd"]; ok { + res.IdEnd = &val[0] } - res.Filters = filters return &res, nil } -func ParseLogFilter(allFields *model.GetFieldsResponse, filters *[]model.LogFilter) (*string, error) { - fLen := len(*filters) - if fLen <= 0 { - return nil, nil +func parseLogQuery(query string) ([]string, error) { + sqlQueryTokens := []string{} + filterTokens := tokenRegex.FindAllString(query, -1) + + if len(filterTokens) == 0 { + sqlQueryTokens = append(sqlQueryTokens, fmt.Sprintf("body ILIKE '%%%s%%' ", query)) + return sqlQueryTokens, nil } + // replace and check if there is something that is lying around + if len(strings.TrimSpace(tokenRegex.ReplaceAllString(query, ""))) > 0 { + return nil, fmt.Errorf("failed to parse query, contains unknown tokens") + } + + for _, v := range filterTokens { + op := strings.TrimSpace(operatorRegex.FindString(v)) + + if strings.ToLower(op) == "contains" { + searchString := strings.TrimSpace(strings.Split(v, op)[1]) + sqlQueryTokens = append(sqlQueryTokens, fmt.Sprintf(`AND body ILIKE '%%%s%%' `, searchString[1:len(searchString)-1])) + } else if strings.ToLower(op) == "ncontains" { + searchString := strings.TrimSpace(strings.Split(v, op)[1]) + sqlQueryTokens = append(sqlQueryTokens, fmt.Sprintf(`AND body NOT ILIKE '%%%s%%' `, searchString[1:len(searchString)-1])) + } else { + symbol := operatorMapping[strings.ToLower(op)] + sqlQueryTokens = append(sqlQueryTokens, strings.Replace(v, " "+op+" ", " "+symbol+" ", 1)+" ") + } + } + + return sqlQueryTokens, nil +} + +func parseColumn(s string) (*string, error) { + s = strings.ToLower(s) + + colName := "" + + // if has and/or as prefix + filter := strings.Split(s, " ") + if len(filter) < 3 { + return nil, fmt.Errorf("incorrect filter") + } + + if strings.HasPrefix(s, "and") { + colName = filter[1] + } else { + colName = filter[0] + } + + return &colName, nil +} + +func arrayToMap(fields []model.LogField) map[string]model.LogField { + res := map[string]model.LogField{} + for _, field := range fields { + res[field.Name] = field + } + return res +} + +func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens []string) ([]string, error) { + // check if cols selectedFieldsLookup := arrayToMap(allFields.Selected) interestingFieldLookup := arrayToMap(allFields.Interesting) - filterSql := "" - for fIndx := 0; fIndx < fLen; fIndx++ { - filter := (*filters)[fIndx] - fieldSQLName := filter.Column - if _, ok := selectedFieldsLookup[filter.Column]; !ok { - if field, ok := interestingFieldLookup[filter.Column]; ok { - fieldSQLName = fmt.Sprintf("%s_%s_value[indexOf(%s_%s_key, '%s')]", field.Type, strings.ToLower(field.DataType), field.Type, strings.ToLower(field.DataType), filter.Column) + + for index := 0; index < len(queryTokens); index++ { + queryToken := queryTokens[index] + col, err := parseColumn(queryToken) + if err != nil { + return nil, err + } + + sqlColName := *col + if _, ok := selectedFieldsLookup[*col]; !ok && *col != "body" { + if field, ok := interestingFieldLookup[*col]; ok { + sqlColName = fmt.Sprintf("%s_%s_value[indexOf(%s_%s_key, '%s')]", field.Type, strings.ToLower(field.DataType), field.Type, strings.ToLower(field.DataType), *col) } else { return nil, fmt.Errorf("field not found for filtering") } } + queryTokens[index] = strings.Replace(queryToken, *col, sqlColName, 1) + } + return queryTokens, nil +} - filterSql += "(" - vLen := len(filter.Value) - for i := 0; i < vLen; i++ { - filterSql += fmt.Sprintf("%s%s'%v'", fieldSQLName, operatorMapping[filter.Operation], filter.Value[i]) - if i != vLen-1 { - filterSql += " or " - } - } - filterSql += ")" - - if fIndx != fLen-1 { - filterSql += " and " - } +func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (*string, error) { + tokens, err := parseLogQuery(*params.Query) + if err != nil { + return nil, err } - return &filterSql, nil + tokens, err = replaceInterestingFields(allFields, tokens) + if err != nil { + return nil, err + } + + if params.TimestampStart != nil { + tokens = append(tokens, fmt.Sprintf("and timestamp >= '%d' ", *params.TimestampStart)) + } + if params.TimestampEnd != nil { + tokens = append(tokens, fmt.Sprintf("and timestamp <= '%d' ", *params.TimestampEnd)) + } + if params.IdStart != nil { + tokens = append(tokens, fmt.Sprintf("and id > '%v' ", *params.IdStart)) + } + if params.IdEnd != nil { + tokens = append(tokens, fmt.Sprintf("and id < '%v' ", *params.IdEnd)) + } + + sqlWhere := strings.Join(tokens, "") + + return &sqlWhere, nil } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go new file mode 100644 index 0000000000..d1467b4ee3 --- /dev/null +++ b/pkg/query-service/app/logs/parser_test.go @@ -0,0 +1,209 @@ +package logs + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" + "go.signoz.io/query-service/model" +) + +var correctQueriesTest = []struct { + Name string + InputQuery string + WantSqlTokens []string +}{ + { + `filter with fulltext`, + `OPERATION in ('bcd') AND FULLTEXT contains 'searchstring'`, + []string{`OPERATION IN ('bcd') `, `AND body ILIKE '%searchstring%' `}, + }, + { + `fulltext`, + `searchstring`, + []string{`body ILIKE '%searchstring%' `}, + }, + { + `filters with lt,gt,lte,gte operators`, + `id lt 100 and id gt 50 and code lte 500 and code gte 400`, + []string{`id < 100 `, `and id > 50 `, `and code <= 500 `, `and code >= 400 `}, + }, + { + `filter with number`, + `status gte 200 AND FULLTEXT ncontains '"key"'`, + []string{`status >= 200 `, `AND body NOT ILIKE '%"key"%' `}, + }, + { + `characters inside string`, + `service NIN ('name > 100') AND length gt 100`, + []string{`service NOT IN ('name > 100') `, `AND length > 100 `}, + }, + { + `fulltext with in`, + `key in 2`, + []string{`body ILIKE '%key in 2%' `}, + }, + { + `not valid fulltext but a filter`, + `key in (2,3)`, + []string{`key IN (2,3) `}, + }, + { + `filters with extra spaces`, + `service IN ('name > 100') AND length gt 100`, + []string{`service IN ('name > 100') `, `AND length > 100 `}, + }, + { + `filters with special characters in key name`, + `id.userid in (100) and id_userid gt 50`, + []string{`id.userid IN (100) `, `and id_userid > 50 `}, + }, +} + +func TestParseLogQueryCorrect(t *testing.T) { + for _, test := range correctQueriesTest { + Convey(test.Name, t, func() { + query, _ := parseLogQuery(test.InputQuery) + + So(query, ShouldResemble, test.WantSqlTokens) + }) + } +} + +var incorrectQueries = []struct { + Name string + Query string +}{ + { + "filter without a key", + "OPERATION in ('bcd') AND 'abcd' FULLTEXT contains 'helloxyz'", + }, + { + "fulltext without fulltext keyword", + "OPERATION in ('bcd') AND 'searchstring'", + }, + { + "fulltext in the beginning without keyword", + "'searchstring and OPERATION in ('bcd')", + }, +} + +func TestParseLogQueryInCorrect(t *testing.T) { + for _, test := range incorrectQueries { + Convey(test.Name, t, func() { + _, err := parseLogQuery(test.Query) + So(err, ShouldBeError) + }) + } +} + +var parseCorrectColumns = []struct { + Name string + Filter string + Column string +}{ + { + "column with IN operator", + "id.userid IN (100) ", + "id.userid", + }, + { + "column with NOT IN operator", + "service NOT IN ('name > 100') ", + "service", + }, + { + "column with > operator", + "and id_userid > 50 ", + "id_userid", + }, + { + "column with < operator", + "and id_userid < 50 ", + "id_userid", + }, + { + "column with <= operator", + "and id_userid <= 50 ", + "id_userid", + }, + { + "column with >= operator", + "and id_userid >= 50 ", + "id_userid", + }, + { + "column with ilike", + `AND body ILIKE '%searchstring%' `, + "body", + }, + { + "column with not ilike", + `AND body ILIKE '%searchstring%' `, + "body", + }, +} + +func TestParseColumn(t *testing.T) { + for _, test := range parseCorrectColumns { + Convey(test.Name, t, func() { + column, _ := parseColumn(test.Filter) + So(*column, ShouldEqual, test.Column) + }) + } +} + +func TestReplaceInterestingFields(t *testing.T) { + queryTokens := []string{"id.userid IN (100) ", "and id_key >= 50 ", `AND body ILIKE '%searchstring%'`} + allFields := model.GetFieldsResponse{ + Selected: []model.LogField{ + model.LogField{ + Name: "id_key", + DataType: "int64", + Type: "attributes", + }, + }, + Interesting: []model.LogField{ + model.LogField{ + Name: "id.userid", + DataType: "int64", + Type: "attributes", + }, + }, + } + + expectedTokens := []string{"attributes_int64_value[indexOf(attributes_int64_key, 'id.userid')] IN (100) ", "and id_key >= 50 ", `AND body ILIKE '%searchstring%'`} + Convey("testInterestingFields", t, func() { + tokens, _ := replaceInterestingFields(&allFields, queryTokens) + So(tokens, ShouldResemble, expectedTokens) + }) +} + +func TestGenerateSQLQuery(t *testing.T) { + allFields := model.GetFieldsResponse{ + Selected: []model.LogField{ + { + Name: "id", + DataType: "int64", + Type: "attributes", + }, + }, + Interesting: []model.LogField{ + { + Name: "code", + DataType: "int64", + Type: "attributes", + }, + }, + } + + query := "id lt 100 and id gt 50 and code lte 500 and code gte 400" + tsStart := int64(1657689292000) + tsEnd := int64(1657689294000) + idStart := "2BsKLKv8cZrLCn6rkOcRGkdjBdM" + idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU" + sqlWhere := "id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 and timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' " + Convey("testInterestingFields", t, func() { + res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: &query, TimestampStart: &tsStart, TimestampEnd: &tsEnd, IdStart: &idStart, IdEnd: &idEnd}) + So(*res, ShouldEqual, sqlWhere) + }) +} diff --git a/pkg/query-service/app/logs/validator.go b/pkg/query-service/app/logs/validator.go index 99858969e8..9277bda047 100644 --- a/pkg/query-service/app/logs/validator.go +++ b/pkg/query-service/app/logs/validator.go @@ -38,28 +38,3 @@ func ValidateUpdateFieldPayload(field *model.UpdateField) error { } return nil } - -func ValidateFilters(filters *[]model.LogFilter) error { - opsRegex := "^(gte|lte|gt|lt|eq|neq|in|like|ilike)$" - regex, err := regexp.Compile(opsRegex) - if err != nil { - return err - } - for _, val := range *filters { - if val.Column == "" { - return fmt.Errorf("col cannot be empty") - } - if val.Operation == "" { - return fmt.Errorf("op cannot be empty") - } - if len(val.Value) == 0 { - return fmt.Errorf("val cannot be empty") - } - - matched := regex.MatchString(val.Operation) - if !matched { - return fmt.Errorf("op type %s not supported", val.Operation) - } - } - return nil -} diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 2d23531c41..ea9a182786 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -201,6 +201,11 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) { lrw.ResponseWriter.WriteHeader(code) } +// Flush implements the http.Flush interface. +func (lrw *loggingResponseWriter) Flush() { + lrw.ResponseWriter.(http.Flusher).Flush() +} + func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route := mux.CurrentRoute(r) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index bb5a428a0e..c8d9b1cfcf 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -64,4 +64,5 @@ type Reader interface { GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.GetLogsResponse, *model.ApiError) + TailLogs(ctx context.Context, client *model.LogsTailClient) *model.ApiError } diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index c39c3eb802..1df7813679 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -313,15 +313,13 @@ type UpdateField struct { IndexGranularity *int `json:"indexGranularity"` } -type LogFilter struct { - Column string `json:"col"` - Operation string `json:"op"` - Value []interface{} `json:"val"` -} - type LogsFilterParams struct { - Limit int `json:"limit"` - OrderBy string `json:"orderBy"` - Order string `json:"order"` - Filters []LogFilter `json:"filters"` + Limit int `json:"limit"` + OrderBy string `json:"orderBy"` + Order string `json:"order"` + Query *string `json:"q"` + TimestampStart *int64 `json:"timestampStart"` + TimestampEnd *int64 `json:"timestampEnd"` + IdStart *string `json:"idStart"` + IdEnd *string `json:"idEnd"` } diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index f6120f34af..7f6020c114 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -18,18 +18,19 @@ type ApiError struct { type ErrorType string const ( - ErrorNone ErrorType = "" - ErrorTimeout ErrorType = "timeout" - ErrorCanceled ErrorType = "canceled" - ErrorExec ErrorType = "execution" - ErrorBadData ErrorType = "bad_data" - ErrorInternal ErrorType = "internal" - ErrorUnavailable ErrorType = "unavailable" - ErrorNotFound ErrorType = "not_found" - ErrorNotImplemented ErrorType = "not_implemented" - ErrorUnauthorized ErrorType = "unauthorized" - ErrorForbidden ErrorType = "forbidden" - ErrorConflict ErrorType = "conflict" + ErrorNone ErrorType = "" + ErrorTimeout ErrorType = "timeout" + ErrorCanceled ErrorType = "canceled" + ErrorExec ErrorType = "execution" + ErrorBadData ErrorType = "bad_data" + ErrorInternal ErrorType = "internal" + ErrorUnavailable ErrorType = "unavailable" + ErrorNotFound ErrorType = "not_found" + ErrorNotImplemented ErrorType = "not_implemented" + ErrorUnauthorized ErrorType = "unauthorized" + ErrorForbidden ErrorType = "forbidden" + ErrorConflict ErrorType = "conflict" + ErrorStreamingNotSupported ErrorType = "streaming is not supported" ) type QueryDataV2 struct { @@ -404,3 +405,9 @@ type GetLogsResponse struct { Attributes_int64 map[string]int64 `json:"attributesInt" ch:"attributes_int64"` Attributes_float64 map[string]float64 `json:"attributesFloat" ch:"attributes_float64"` } + +type LogsTailClient struct { + Name string + Logs chan *string + Done chan *bool +} From 2450fff34d27a27749149e4297ae3932df1c3ab8 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Jul 2022 18:55:52 +0530 Subject: [PATCH 04/32] live tail v1 --- .../app/clickhouseReader/reader.go | 88 ++++++++++++++++--- pkg/query-service/app/http_handler.go | 15 +++- pkg/query-service/app/logs/parser.go | 36 ++++++-- pkg/query-service/interfaces/interface.go | 2 +- pkg/query-service/model/queryParams.go | 4 +- pkg/query-service/model/response.go | 8 +- 6 files changed, 128 insertions(+), 25 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 4060b08eed..0c8581e8f6 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2883,20 +2883,86 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter return response, nil } -func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailClient) *model.ApiError { - for i := 0; i < 10; i++ { +func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailClient) { + response := &[]model.GetLogsResponse{} + fields, apiErr := r.GetLogFields(ctx) + if apiErr != nil { + client.Error <- apiErr.Err + return + } + + filterSql, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ + Query: client.Filter.Query, + }) + + if err != nil { + client.Error <- err + return + } + + query := fmt.Sprintf("SELECT "+ + "timestamp, observed_timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ + "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,"+ + "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,"+ + "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,"+ + "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ + "from %s.%s", r.logsDB, r.logsTable) + + currentTime := uint64(time.Now().UnixNano() / int64(time.Millisecond)) + tsStart := ¤tTime + if client.Filter.TimestampStart != nil { + tsStart = client.Filter.TimestampStart + } + + var idStart *string + if client.Filter.IdStart != nil { + idStart = client.Filter.IdStart + } + + for { select { case <-ctx.Done(): - return nil + done := true + client.Done <- &done + zap.S().Debug("closing go routine : " + client.Name) + return default: - data := fmt.Sprintf("hello log %d", i) - client.Logs <- &data - time.Sleep(time.Second) + tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, *tsStart) + if filterSql != nil && *filterSql != "" { + tmpQuery += fmt.Sprintf(" and %s", *filterSql) + } + if idStart != nil { + tmpQuery += fmt.Sprintf(" and id > '%s'", *idStart) + } + tmpQuery = fmt.Sprintf("%s order by timestamp asc limit 1000", tmpQuery) + zap.S().Debug(tmpQuery) + err := r.db.Select(ctx, response, tmpQuery) + if err != nil { + zap.S().Debug(err) + client.Error <- err + return + } + len := len(*response) + for i := 0; i < len; i++ { + select { + case <-ctx.Done(): + done := true + client.Done <- &done + zap.S().Debug("closing go routine while sending logs : " + client.Name) + return + default: + client.Logs <- &(*response)[i] + if i == len-1 { + tsStart = &(*response)[i].Timestamp + idStart = &(*response)[i].ID + } + } + } + if len == 0 { + currentTime := uint64(time.Now().UnixNano() / int64(time.Millisecond)) + tsStart = ¤tTime + } + time.Sleep(2 * time.Second) } } - done := true - client.Done <- &done - fmt.Println("done in the tail logs") - - return nil } diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 8e5233d744..32113d801a 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1903,7 +1903,7 @@ func (aH *APIHandler) logFieldUpdate(w http.ResponseWriter, r *http.Request) { } func (aH *APIHandler) getLogs(w http.ResponseWriter, r *http.Request) { - params, err := logs.ParseFilterParams(r) + params, err := logs.ParseLogFilterParams(r) if err != nil { apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} respondError(w, apiErr, "Incorrect params") @@ -1919,7 +1919,15 @@ func (aH *APIHandler) getLogs(w http.ResponseWriter, r *http.Request) { } func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { - client := &model.LogsTailClient{Name: r.RemoteAddr, Logs: make(chan *string, 100), Done: make(chan *bool)} + params, err := logs.ParseLogFilterParams(r) + if err != nil { + apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} + respondError(w, apiErr, "Incorrect params") + return + } + + // create the client + client := &model.LogsTailClient{Name: r.RemoteAddr, Logs: make(chan *model.GetLogsResponse, 1000), Done: make(chan *bool), Error: make(chan error), Filter: *params} go (*aH.reader).TailLogs(r.Context(), client) w.Header().Set("Connection", "keep-alive") @@ -1948,6 +1956,9 @@ func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { case <-client.Done: fmt.Println("done!") return + case <-client.Error: + fmt.Println("error occured!") + return } } } diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 61d1d2840b..1bbab36466 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -24,7 +24,7 @@ var operatorMapping = map[string]string{ var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \(["\w.,' \-><]+\))|([\w.-]+ (gt|lt|gte|lte|contains|ncontains) ("|')?\S+("|')?))`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) -func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { +func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { res := model.LogsFilterParams{ Limit: 30, OrderBy: "timestamp", @@ -52,7 +52,7 @@ func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { if err != nil { return nil, err } - ts64 := int64(ts) + ts64 := uint64(ts) res.TimestampStart = &ts64 } if val, ok := params["timestampEnd"]; ok { @@ -60,7 +60,7 @@ func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { if err != nil { return nil, err } - ts64 := int64(ts) + ts64 := uint64(ts) res.TimestampEnd = &ts64 } if val, ok := params["idStart"]; ok { @@ -72,6 +72,26 @@ func ParseFilterParams(r *http.Request) (*model.LogsFilterParams, error) { return &res, nil } +func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error) { + res := model.LogsFilterParams{} + params := r.URL.Query() + if val, ok := params["q"]; ok { + res.Query = &val[0] + } + if val, ok := params["timestampStart"]; ok { + ts, err := strconv.Atoi(val[0]) + if err != nil { + return nil, err + } + ts64 := uint64(ts) + res.TimestampStart = &ts64 + } + if val, ok := params["idStart"]; ok { + res.IdStart = &val[0] + } + return &res, nil +} + func parseLogQuery(query string) ([]string, error) { sqlQueryTokens := []string{} filterTokens := tokenRegex.FindAllString(query, -1) @@ -158,9 +178,13 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens [] } func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (*string, error) { - tokens, err := parseLogQuery(*params.Query) - if err != nil { - return nil, err + var tokens []string + var err error + if params.Query != nil { + tokens, err = parseLogQuery(*params.Query) + if err != nil { + return nil, err + } } tokens, err = replaceInterestingFields(allFields, tokens) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 8519b09b04..b0429db9bb 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -62,7 +62,7 @@ type Reader interface { GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.GetLogsResponse, *model.ApiError) - TailLogs(ctx context.Context, client *model.LogsTailClient) *model.ApiError + TailLogs(ctx context.Context, client *model.LogsTailClient) // Connection needed for rules, not ideal but required GetConn() clickhouse.Conn diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index d63a07eb71..b6059107d9 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -335,8 +335,8 @@ type LogsFilterParams struct { OrderBy string `json:"orderBy"` Order string `json:"order"` Query *string `json:"q"` - TimestampStart *int64 `json:"timestampStart"` - TimestampEnd *int64 `json:"timestampEnd"` + TimestampStart *uint64 `json:"timestampStart"` + TimestampEnd *uint64 `json:"timestampEnd"` IdStart *string `json:"idStart"` IdEnd *string `json:"idEnd"` } diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 8efb4f1069..b85955da74 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -437,7 +437,9 @@ type GetLogsResponse struct { } type LogsTailClient struct { - Name string - Logs chan *string - Done chan *bool + Name string + Logs chan *GetLogsResponse + Done chan *bool + Error chan error + Filter LogsFilterParams } From 8e4fbbe77004ae34d9bf2739e5713e2fd2c9cb75 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 19 Jul 2022 10:40:19 +0530 Subject: [PATCH 05/32] parsing logic and test updated --- pkg/query-service/app/logs/parser.go | 14 ++++++++------ pkg/query-service/app/logs/parser_test.go | 9 +++++++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 1bbab36466..4c8649b5fd 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -21,7 +21,7 @@ var operatorMapping = map[string]string{ "ncontains": "NOT ILIKE", } -var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \(["\w.,' \-><]+\))|([\w.-]+ (gt|lt|gte|lte|contains|ncontains) ("|')?\S+("|')?))`) +var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[\S ]+(')?)`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { @@ -108,13 +108,15 @@ func parseLogQuery(query string) ([]string, error) { for _, v := range filterTokens { op := strings.TrimSpace(operatorRegex.FindString(v)) + opLower := strings.ToLower(op) - if strings.ToLower(op) == "contains" { + if opLower == "contains" || opLower == "ncontains" { searchString := strings.TrimSpace(strings.Split(v, op)[1]) - sqlQueryTokens = append(sqlQueryTokens, fmt.Sprintf(`AND body ILIKE '%%%s%%' `, searchString[1:len(searchString)-1])) - } else if strings.ToLower(op) == "ncontains" { - searchString := strings.TrimSpace(strings.Split(v, op)[1]) - sqlQueryTokens = append(sqlQueryTokens, fmt.Sprintf(`AND body NOT ILIKE '%%%s%%' `, searchString[1:len(searchString)-1])) + f := fmt.Sprintf(`body %s '%%%s%%' `, operatorMapping[opLower], searchString[1:len(searchString)-1]) + if strings.HasPrefix(strings.ToLower(v), "and") { + f = "AND " + f + } + sqlQueryTokens = append(sqlQueryTokens, f) } else { symbol := operatorMapping[strings.ToLower(op)] sqlQueryTokens = append(sqlQueryTokens, strings.Replace(v, " "+op+" ", " "+symbol+" ", 1)+" ") diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index d1467b4ee3..b06b2ccec5 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -22,6 +22,11 @@ var correctQueriesTest = []struct { `searchstring`, []string{`body ILIKE '%searchstring%' `}, }, + { + `fulltext with quotes and space`, + `FULLTEXT contains 'Hello, "World"'`, + []string{`body ILIKE '%Hello, "World"%' `}, + }, { `filters with lt,gt,lte,gte operators`, `id lt 100 and id gt 50 and code lte 500 and code gte 400`, @@ -197,8 +202,8 @@ func TestGenerateSQLQuery(t *testing.T) { } query := "id lt 100 and id gt 50 and code lte 500 and code gte 400" - tsStart := int64(1657689292000) - tsEnd := int64(1657689294000) + tsStart := uint64(1657689292000) + tsEnd := uint64(1657689294000) idStart := "2BsKLKv8cZrLCn6rkOcRGkdjBdM" idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU" sqlWhere := "id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 and timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' " From b5c876460509901b98cf4fc79a978537d4eea6a8 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 19 Jul 2022 16:34:33 +0530 Subject: [PATCH 06/32] changes added for live tail api --- pkg/query-service/app/clickhouseReader/reader.go | 5 +++-- pkg/query-service/app/http_handler.go | 5 ++--- pkg/query-service/app/server.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 0c8581e8f6..4a57376b26 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2884,7 +2884,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter } func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailClient) { - response := &[]model.GetLogsResponse{} + fields, apiErr := r.GetLogFields(ctx) if apiErr != nil { client.Error <- apiErr.Err @@ -2934,8 +2934,9 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC if idStart != nil { tmpQuery += fmt.Sprintf(" and id > '%s'", *idStart) } - tmpQuery = fmt.Sprintf("%s order by timestamp asc limit 1000", tmpQuery) + tmpQuery = fmt.Sprintf("%s order by timestamp asc, id asc limit 1000", tmpQuery) zap.S().Debug(tmpQuery) + response := &[]model.GetLogsResponse{} err := r.db.Select(ctx, response, tmpQuery) if err != nil { zap.S().Debug(err) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 32113d801a..d5ef57164c 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1951,13 +1951,12 @@ func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { enc := json.NewEncoder(&buf) enc.Encode(ev) fmt.Fprintf(w, "data: %v\n\n", buf.String()) - fmt.Printf("data: %v\n", buf.String()) flusher.Flush() case <-client.Done: - fmt.Println("done!") + zap.S().Debug("done!") return case <-client.Error: - fmt.Println("error occured!") + zap.S().Debug("error occured!") return } } diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index b735f1b657..77ab3fc9b9 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -166,8 +166,8 @@ func (s *Server) createPublicServer(api *APIHandler) (*http.Server, error) { c := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, - AllowedMethods: []string{"GET", "DELETE", "POST", "PUT"}, - AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"}, + AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "cache-control"}, }) handler := c.Handler(r) From 051f640100646b40ce1d65a1bf4af85c9bfca212 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 19 Jul 2022 16:38:28 +0530 Subject: [PATCH 07/32] correct var names in live tail --- pkg/query-service/app/http_handler.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index d5ef57164c..a2f55fec60 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1943,13 +1943,12 @@ func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { return } - loop := true - for loop { + for { select { - case ev := <-client.Logs: + case log := <-client.Logs: var buf bytes.Buffer enc := json.NewEncoder(&buf) - enc.Encode(ev) + enc.Encode(log) fmt.Fprintf(w, "data: %v\n\n", buf.String()) flusher.Flush() case <-client.Done: From c24bdfc8cf70d33d8dee71dc65451cc35396c893 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Jul 2022 12:11:03 +0530 Subject: [PATCH 08/32] aggregate function added --- .../app/clickhouseReader/reader.go | 81 +++++++++++++++++++ pkg/query-service/app/http_handler.go | 17 ++++ pkg/query-service/app/logs/parser.go | 48 +++++++++++ pkg/query-service/interfaces/interface.go | 1 + pkg/query-service/model/queryParams.go | 9 +++ pkg/query-service/model/response.go | 16 ++++ 6 files changed, 172 insertions(+) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 4a57376b26..9cd521e3dc 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2967,3 +2967,84 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } } } + +func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.LogsAggregateParams) (*model.GetLogsAggregatesResponse, *model.ApiError) { + logAggregatesDBResponseItems := &[]model.LogsAggregatesDBResponseItem{} + + groupBy := "" + if params.GroupBy != nil { + groupBy = *params.GroupBy + } + + function := "toFloat64(count()) as value" + if params.Function != nil { + function = fmt.Sprintf("toFloat64(%s) as value", *params.Function) + } + + fields, apiErr := r.GetLogFields(ctx) + if apiErr != nil { + return nil, apiErr + } + + filterSql, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ + Query: params.Query, + }) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} + } + + query := "" + if groupBy != "" { + query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000), INTERVAL %d minute))*1000) as time, toString(%s) as groupBy, "+ + "%s "+ + "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", + *params.StepSeconds/60, groupBy, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) + } else { + query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000), INTERVAL %d minute))*1000) as time, "+ + "%s "+ + "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", + *params.StepSeconds/60, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) + } + if filterSql != nil && *filterSql != "" { + query += fmt.Sprintf(" AND %s ", *filterSql) + } + if groupBy != "" { + query += fmt.Sprintf("GROUP BY time, toString(%s) as groupBy ORDER BY time", groupBy) + } else { + query += "GROUP BY time ORDER BY time" + } + + zap.S().Debug(query) + err = r.db.Select(ctx, logAggregatesDBResponseItems, query) + if err != nil { + return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} + } + + aggregateResponse := model.GetLogsAggregatesResponse{ + Items: make(map[int64]model.LogsAggregatesResponseItem), + } + + for i := range *logAggregatesDBResponseItems { + if elem, ok := aggregateResponse.Items[int64((*logAggregatesDBResponseItems)[i].Timestamp)]; ok { + if groupBy != "" && (*logAggregatesDBResponseItems)[i].GroupBy != "" { + elem.GroupBy[(*logAggregatesDBResponseItems)[i].GroupBy] = (*logAggregatesDBResponseItems)[i].Value + } + aggregateResponse.Items[(*logAggregatesDBResponseItems)[i].Timestamp] = elem + } else { + if groupBy != "" && (*logAggregatesDBResponseItems)[i].GroupBy != "" { + aggregateResponse.Items[(*logAggregatesDBResponseItems)[i].Timestamp] = model.LogsAggregatesResponseItem{ + Timestamp: (*logAggregatesDBResponseItems)[i].Timestamp, + GroupBy: map[string]interface{}{(*logAggregatesDBResponseItems)[i].GroupBy: (*logAggregatesDBResponseItems)[i].Value}, + } + } else if groupBy == "" { + aggregateResponse.Items[(*logAggregatesDBResponseItems)[i].Timestamp] = model.LogsAggregatesResponseItem{ + Timestamp: (*logAggregatesDBResponseItems)[i].Timestamp, + Value: (*logAggregatesDBResponseItems)[i].Value, + } + } + } + + } + + return &aggregateResponse, nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index a2f55fec60..c97ffd932f 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1868,6 +1868,7 @@ func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter.HandleFunc("/tail", ViewAccess(aH.tailLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost) + subRouter.HandleFunc("/aggregate", ViewAccess(aH.logAggregate)).Methods(http.MethodGet) } func (aH *APIHandler) logFields(w http.ResponseWriter, r *http.Request) { @@ -1960,3 +1961,19 @@ func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { } } } + +func (aH *APIHandler) logAggregate(w http.ResponseWriter, r *http.Request) { + params, err := logs.ParseLogAggregateParams(r) + if err != nil { + apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err} + respondError(w, apiErr, "Incorrect params") + return + } + + res, apiErr := (*aH.reader).AggregateLogs(r.Context(), params) + if apiErr != nil { + respondError(w, apiErr, "Failed to fetch logs aggregate from the DB") + return + } + aH.writeJSON(w, r, res) +} diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 4c8649b5fd..7d9438976b 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -92,6 +92,54 @@ func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error) return &res, nil } +func ParseLogAggregateParams(r *http.Request) (*model.LogsAggregateParams, error) { + res := model.LogsAggregateParams{} + params := r.URL.Query() + if val, ok := params["timestampStart"]; ok { + ts, err := strconv.Atoi(val[0]) + if err != nil { + return nil, err + } + ts64 := uint64(ts) + res.TimestampStart = &ts64 + } else { + return nil, fmt.Errorf("timestampStart is required") + } + if val, ok := params["timestampEnd"]; ok { + ts, err := strconv.Atoi(val[0]) + if err != nil { + return nil, err + } + ts64 := uint64(ts) + res.TimestampEnd = &ts64 + } else { + return nil, fmt.Errorf("timestampEnd is required") + } + + if val, ok := params["q"]; ok { + res.Query = &val[0] + } + + if val, ok := params["groupBy"]; ok { + res.GroupBy = &val[0] + } + + if val, ok := params["function"]; ok { + res.Function = &val[0] + } + + if val, ok := params["step"]; ok { + step, err := strconv.Atoi(val[0]) + if err != nil { + return nil, err + } + res.StepSeconds = &step + } else { + return nil, fmt.Errorf("step is required") + } + return &res, nil +} + func parseLogQuery(query string) ([]string, error) { sqlQueryTokens := []string{} filterTokens := tokenRegex.FindAllString(query, -1) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index b0429db9bb..c96a1af470 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -63,6 +63,7 @@ type Reader interface { UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.GetLogsResponse, *model.ApiError) TailLogs(ctx context.Context, client *model.LogsTailClient) + AggregateLogs(ctx context.Context, params *model.LogsAggregateParams) (*model.GetLogsAggregatesResponse, *model.ApiError) // Connection needed for rules, not ideal but required GetConn() clickhouse.Conn diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index b6059107d9..ce9c4f53c3 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -340,3 +340,12 @@ type LogsFilterParams struct { IdStart *string `json:"idStart"` IdEnd *string `json:"idEnd"` } + +type LogsAggregateParams struct { + Query *string `json:"q"` + TimestampStart *uint64 `json:"timestampStart"` + TimestampEnd *uint64 `json:"timestampEnd"` + GroupBy *string `json:"groupBy"` + Function *string `json:"function"` + StepSeconds *int `json:"step"` +} diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index b85955da74..b802e3c4f7 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -443,3 +443,19 @@ type LogsTailClient struct { Error chan error Filter LogsFilterParams } + +type GetLogsAggregatesResponse struct { + Items map[int64]LogsAggregatesResponseItem `json:"items"` +} + +type LogsAggregatesResponseItem struct { + Timestamp int64 `json:"timestamp,omitempty" ` + Value interface{} `json:"value,omitempty"` + GroupBy map[string]interface{} `json:"groupBy,omitempty"` +} + +type LogsAggregatesDBResponseItem struct { + Timestamp int64 `ch:"time"` + Value float64 `ch:"value"` + GroupBy string `ch:"groupBy"` +} From a527c33c7d5bc739da64a81b9859f803657df29c Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Jul 2022 13:05:24 +0530 Subject: [PATCH 09/32] timestamp in ns from ms --- pkg/query-service/app/clickhouseReader/reader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 9cd521e3dc..770d2058c5 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2995,12 +2995,12 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs query := "" if groupBy != "" { - query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000), INTERVAL %d minute))*1000) as time, toString(%s) as groupBy, "+ + query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000000000), INTERVAL %d minute))*1000000000) as time, toString(%s) as groupBy, "+ "%s "+ "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", *params.StepSeconds/60, groupBy, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) } else { - query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000), INTERVAL %d minute))*1000) as time, "+ + query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000000000), INTERVAL %d minute))*1000000000) as time, "+ "%s "+ "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", *params.StepSeconds/60, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) From 5912d3a4a0f3385c144c43851f008b57f1554f26 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Jul 2022 14:52:16 +0530 Subject: [PATCH 10/32] observed timestamp removed --- pkg/query-service/app/clickhouseReader/reader.go | 4 ++-- pkg/query-service/model/response.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 770d2058c5..9cfa81b647 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2862,7 +2862,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter } query := fmt.Sprintf("SELECT "+ - "timestamp, observed_timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ + "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,"+ "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,"+ "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,"+ @@ -2901,7 +2901,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } query := fmt.Sprintf("SELECT "+ - "timestamp, observed_timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ + "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,"+ "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,"+ "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,"+ diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index b802e3c4f7..2232633471 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -422,7 +422,6 @@ type GetFieldsResponse struct { type GetLogsResponse struct { Timestamp uint64 `json:"timestamp" ch:"timestamp"` - ObservedTimestamp uint64 `json:"observedTimestamp" ch:"observed_timestamp"` ID string `json:"id" ch:"id"` TraceID string `json:"traceId" ch:"trace_id"` SpanID string `json:"spanId" ch:"span_id"` From 2132d1059cc4f1a4079d147149f6d4bbfb5535ac Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 21 Jul 2022 17:55:08 +0530 Subject: [PATCH 11/32] live tail api excluded from timeout middleware --- pkg/query-service/app/server.go | 10 ++++++++-- pkg/query-service/constants/constants.go | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 77ab3fc9b9..b2fbe2df5a 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -242,8 +242,14 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { func setTimeoutMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), constants.ContextTimeout*time.Second) - defer cancel() + ctx := r.Context() + var cancel context.CancelFunc + // check if route is not excluded + url := r.URL.Path + if _, ok := constants.TimeoutExcludedRoutes[url]; !ok { + ctx, cancel = context.WithTimeout(r.Context(), constants.ContextTimeout*time.Second) + defer cancel() + } r = r.WithContext(ctx) next.ServeHTTP(w, r) diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 8f590d81ba..e5e727de0b 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -81,6 +81,10 @@ const ( SIGNOZ_TIMESERIES_TABLENAME = "time_series_v2" ) +var TimeoutExcludedRoutes = map[string]bool{ + "/api/v1/logs/tail": true, +} + // alert related constants const ( // AlertHelpPage is used in case default alert repo url is not set From 6ac7cb10228dbb9297992b34e2943d1394d2e6a4 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 21 Jul 2022 18:32:11 +0530 Subject: [PATCH 12/32] parser updated --- pkg/query-service/app/logs/parser.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 7d9438976b..1ea0dc9ac7 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -243,16 +243,32 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt } if params.TimestampStart != nil { - tokens = append(tokens, fmt.Sprintf("and timestamp >= '%d' ", *params.TimestampStart)) + filter := fmt.Sprintf("timestamp >= '%d' ", *params.TimestampStart) + if len(tokens) > 0 { + filter = "and " + filter + } + tokens = append(tokens, filter) } if params.TimestampEnd != nil { - tokens = append(tokens, fmt.Sprintf("and timestamp <= '%d' ", *params.TimestampEnd)) + filter := fmt.Sprintf("timestamp <= '%d' ", *params.TimestampEnd) + if len(tokens) > 0 { + filter = "and " + filter + } + tokens = append(tokens, filter) } if params.IdStart != nil { - tokens = append(tokens, fmt.Sprintf("and id > '%v' ", *params.IdStart)) + filter := fmt.Sprintf("id > '%v' ", *params.IdStart) + if len(tokens) > 0 { + filter = "and " + filter + } + tokens = append(tokens, filter) } if params.IdEnd != nil { - tokens = append(tokens, fmt.Sprintf("and id < '%v' ", *params.IdEnd)) + filter := fmt.Sprintf("id < '%v' ", *params.IdEnd) + if len(tokens) > 0 { + filter = "and " + filter + } + tokens = append(tokens, filter) } sqlWhere := strings.Join(tokens, "") From 448e14b32fe93bf812919dc97e5b6a71298ffcf5 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 15:17:46 +0530 Subject: [PATCH 13/32] parser updated to support contians operator for other fields --- pkg/query-service/app/logs/parser.go | 21 ++++++++++++++++++--- pkg/query-service/app/logs/parser_test.go | 5 +++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 1ea0dc9ac7..3c776b84ad 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -21,6 +21,10 @@ var operatorMapping = map[string]string{ "ncontains": "NOT ILIKE", } +const ( + AND = "and" +) + var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[\S ]+(')?)`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) @@ -160,8 +164,19 @@ func parseLogQuery(query string) ([]string, error) { if opLower == "contains" || opLower == "ncontains" { searchString := strings.TrimSpace(strings.Split(v, op)[1]) - f := fmt.Sprintf(`body %s '%%%s%%' `, operatorMapping[opLower], searchString[1:len(searchString)-1]) - if strings.HasPrefix(strings.ToLower(v), "and") { + + operatorRemovedTokens := strings.Split(operatorRegex.ReplaceAllString(v, " "), " ") + searchCol := strings.ToLower(operatorRemovedTokens[0]) + if searchCol == AND { + searchCol = strings.ToLower(operatorRemovedTokens[1]) + } + col := searchCol + if strings.ToLower(searchCol) == "fulltext" { + col = "body" + } + + f := fmt.Sprintf(`%s %s '%%%s%%' `, col, operatorMapping[opLower], searchString[1:len(searchString)-1]) + if strings.HasPrefix(strings.ToLower(v), AND) { f = "AND " + f } sqlQueryTokens = append(sqlQueryTokens, f) @@ -185,7 +200,7 @@ func parseColumn(s string) (*string, error) { return nil, fmt.Errorf("incorrect filter") } - if strings.HasPrefix(s, "and") { + if strings.HasPrefix(s, AND) { colName = filter[1] } else { colName = filter[0] diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index b06b2ccec5..65fb89921b 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -27,6 +27,11 @@ var correctQueriesTest = []struct { `FULLTEXT contains 'Hello, "World"'`, []string{`body ILIKE '%Hello, "World"%' `}, }, + { + `contains search with a different attributes`, + `resource contains 'Hello, "World"'`, + []string{`resource ILIKE '%Hello, "World"%' `}, + }, { `filters with lt,gt,lte,gte operators`, `id lt 100 and id gt 50 and code lte 500 and code gte 400`, From 94cde111647ed99256ee31627518f8449ddfbc59 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 15:27:52 +0530 Subject: [PATCH 14/32] consistant response value instead of pointer --- .../app/clickhouseReader/reader.go | 50 +++++++++---------- pkg/query-service/model/response.go | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 9cfa81b647..8a45521ef7 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2771,30 +2771,30 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe } // get attribute keys - attributes := &[]model.LogField{} + attributes := []model.LogField{} query := fmt.Sprintf("SELECT DISTINCT name, datatype from %s.%s group by name, datatype", r.logsDB, r.logsAttributeKeys) - err := r.db.Select(ctx, attributes, query) + err := r.db.Select(ctx, &attributes, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } // get resource keys - resources := &[]model.LogField{} + resources := []model.LogField{} query = fmt.Sprintf("SELECT DISTINCT name, datatype from %s.%s group by name, datatype", r.logsDB, r.logsResourceKeys) - err = r.db.Select(ctx, resources, query) + err = r.db.Select(ctx, &resources, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } - statements := []model.CreateTableStatement{} + statements := []model.ShowCreateTableStatement{} query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.logsDB, r.logsTable) err = r.db.Select(ctx, &statements, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } - extractSelectedAndInterestingFields(statements[0].Statement, constants.Attributes, attributes, &response) - extractSelectedAndInterestingFields(statements[0].Statement, constants.Resources, resources, &response) + extractSelectedAndInterestingFields(statements[0].Statement, constants.Attributes, &attributes, &response) + extractSelectedAndInterestingFields(statements[0].Statement, constants.Resources, &resources, &response) extractSelectedAndInterestingFields(statements[0].Statement, constants.Static, &constants.StaticInterestingLogFields, &response) return &response, nil @@ -2812,7 +2812,7 @@ func extractSelectedAndInterestingFields(tableStatement string, fieldType string } func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError { - // if a field is selected it means that the field is indexed + // if a field is selected it means that the field needs to be indexed if field.Selected { // if the type is attribute or resource, create the materialized column first if field.Type == constants.Attributes || field.Type == constants.Resources { @@ -2850,7 +2850,7 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda } func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.GetLogsResponse, *model.ApiError) { - response := &[]model.GetLogsResponse{} + response := []model.GetLogsResponse{} fields, apiErr := r.GetLogFields(ctx) if apiErr != nil { return nil, apiErr @@ -2875,12 +2875,12 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter query = fmt.Sprintf("%s order by %s %s limit %d", query, params.OrderBy, params.Order, params.Limit) zap.S().Debug(query) - err = r.db.Select(ctx, response, query) + err = r.db.Select(ctx, &response, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } - return response, nil + return &response, nil } func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailClient) { @@ -2969,7 +2969,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.LogsAggregateParams) (*model.GetLogsAggregatesResponse, *model.ApiError) { - logAggregatesDBResponseItems := &[]model.LogsAggregatesDBResponseItem{} + logAggregatesDBResponseItems := []model.LogsAggregatesDBResponseItem{} groupBy := "" if params.GroupBy != nil { @@ -3015,7 +3015,7 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs } zap.S().Debug(query) - err = r.db.Select(ctx, logAggregatesDBResponseItems, query) + err = r.db.Select(ctx, &logAggregatesDBResponseItems, query) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } @@ -3024,22 +3024,22 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs Items: make(map[int64]model.LogsAggregatesResponseItem), } - for i := range *logAggregatesDBResponseItems { - if elem, ok := aggregateResponse.Items[int64((*logAggregatesDBResponseItems)[i].Timestamp)]; ok { - if groupBy != "" && (*logAggregatesDBResponseItems)[i].GroupBy != "" { - elem.GroupBy[(*logAggregatesDBResponseItems)[i].GroupBy] = (*logAggregatesDBResponseItems)[i].Value + for i := range logAggregatesDBResponseItems { + if elem, ok := aggregateResponse.Items[int64(logAggregatesDBResponseItems[i].Timestamp)]; ok { + if groupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { + elem.GroupBy[logAggregatesDBResponseItems[i].GroupBy] = logAggregatesDBResponseItems[i].Value } - aggregateResponse.Items[(*logAggregatesDBResponseItems)[i].Timestamp] = elem + aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = elem } else { - if groupBy != "" && (*logAggregatesDBResponseItems)[i].GroupBy != "" { - aggregateResponse.Items[(*logAggregatesDBResponseItems)[i].Timestamp] = model.LogsAggregatesResponseItem{ - Timestamp: (*logAggregatesDBResponseItems)[i].Timestamp, - GroupBy: map[string]interface{}{(*logAggregatesDBResponseItems)[i].GroupBy: (*logAggregatesDBResponseItems)[i].Value}, + if groupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { + aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = model.LogsAggregatesResponseItem{ + Timestamp: logAggregatesDBResponseItems[i].Timestamp, + GroupBy: map[string]interface{}{logAggregatesDBResponseItems[i].GroupBy: (*logAggregatesDBResponseItems)[i].Value}, } } else if groupBy == "" { - aggregateResponse.Items[(*logAggregatesDBResponseItems)[i].Timestamp] = model.LogsAggregatesResponseItem{ - Timestamp: (*logAggregatesDBResponseItems)[i].Timestamp, - Value: (*logAggregatesDBResponseItems)[i].Value, + aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = model.LogsAggregatesResponseItem{ + Timestamp: logAggregatesDBResponseItems[i].Timestamp, + Value: logAggregatesDBResponseItems[i].Value, } } } diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 2232633471..ea4abe53d4 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -405,7 +405,7 @@ func (p *MetricPoint) MarshalJSON() ([]byte, error) { return json.Marshal([...]interface{}{float64(p.Timestamp) / 1000, v}) } -type CreateTableStatement struct { +type ShowCreateTableStatement struct { Statement string `json:"statement" ch:"statement"` } From bdb6901c74b604789be011323d8741cd5a52d1e9 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 15:39:43 +0530 Subject: [PATCH 15/32] generateSql returns value insted of pointer --- pkg/query-service/app/clickhouseReader/reader.go | 14 +++++++------- pkg/query-service/app/logs/parser.go | 11 ++++++----- pkg/query-service/app/logs/parser_test.go | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 8a45521ef7..4bb74c0ccb 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2869,8 +2869,8 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ "from %s.%s", r.logsDB, r.logsTable) - if filterSql != nil && *filterSql != "" { - query += fmt.Sprintf(" where %s", *filterSql) + if filterSql != "" { + query += fmt.Sprintf(" where %s", filterSql) } query = fmt.Sprintf("%s order by %s %s limit %d", query, params.OrderBy, params.Order, params.Limit) @@ -2928,8 +2928,8 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC return default: tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, *tsStart) - if filterSql != nil && *filterSql != "" { - tmpQuery += fmt.Sprintf(" and %s", *filterSql) + if filterSql != "" { + tmpQuery += fmt.Sprintf(" and %s", filterSql) } if idStart != nil { tmpQuery += fmt.Sprintf(" and id > '%s'", *idStart) @@ -3005,8 +3005,8 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", *params.StepSeconds/60, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) } - if filterSql != nil && *filterSql != "" { - query += fmt.Sprintf(" AND %s ", *filterSql) + if filterSql != "" { + query += fmt.Sprintf(" AND %s ", filterSql) } if groupBy != "" { query += fmt.Sprintf("GROUP BY time, toString(%s) as groupBy ORDER BY time", groupBy) @@ -3034,7 +3034,7 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs if groupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = model.LogsAggregatesResponseItem{ Timestamp: logAggregatesDBResponseItems[i].Timestamp, - GroupBy: map[string]interface{}{logAggregatesDBResponseItems[i].GroupBy: (*logAggregatesDBResponseItems)[i].Value}, + GroupBy: map[string]interface{}{logAggregatesDBResponseItems[i].GroupBy: logAggregatesDBResponseItems[i].Value}, } } else if groupBy == "" { aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = model.LogsAggregatesResponseItem{ diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 3c776b84ad..3a2a7dd1e7 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -242,19 +242,20 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens [] return queryTokens, nil } -func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (*string, error) { +func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, error) { var tokens []string var err error + var sqlWhere string if params.Query != nil { tokens, err = parseLogQuery(*params.Query) if err != nil { - return nil, err + return sqlWhere, err } } tokens, err = replaceInterestingFields(allFields, tokens) if err != nil { - return nil, err + return sqlWhere, err } if params.TimestampStart != nil { @@ -286,7 +287,7 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt tokens = append(tokens, filter) } - sqlWhere := strings.Join(tokens, "") + sqlWhere = strings.Join(tokens, "") - return &sqlWhere, nil + return sqlWhere, nil } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index 65fb89921b..d2855ef1f8 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -214,6 +214,6 @@ func TestGenerateSQLQuery(t *testing.T) { sqlWhere := "id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 and timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' " Convey("testInterestingFields", t, func() { res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: &query, TimestampStart: &tsStart, TimestampEnd: &tsEnd, IdStart: &idStart, IdEnd: &idEnd}) - So(*res, ShouldEqual, sqlWhere) + So(res, ShouldEqual, sqlWhere) }) } From 420d46ab01b0165475bef309e8d1b4ec449512a0 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 15:44:07 +0530 Subject: [PATCH 16/32] tail function updated to use values instead of pointers --- .../app/clickhouseReader/reader.go | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 4bb74c0ccb..ff20d22cfd 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2908,15 +2908,14 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ "from %s.%s", r.logsDB, r.logsTable) - currentTime := uint64(time.Now().UnixNano() / int64(time.Millisecond)) - tsStart := ¤tTime + tsStart := uint64(time.Now().UnixNano() / int64(time.Millisecond)) if client.Filter.TimestampStart != nil { - tsStart = client.Filter.TimestampStart + tsStart = *client.Filter.TimestampStart } - var idStart *string + var idStart string if client.Filter.IdStart != nil { - idStart = client.Filter.IdStart + idStart = *client.Filter.IdStart } for { @@ -2927,23 +2926,23 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC zap.S().Debug("closing go routine : " + client.Name) return default: - tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, *tsStart) + tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, tsStart) if filterSql != "" { tmpQuery += fmt.Sprintf(" and %s", filterSql) } - if idStart != nil { - tmpQuery += fmt.Sprintf(" and id > '%s'", *idStart) + if idStart != "" { + tmpQuery += fmt.Sprintf(" and id > '%s'", idStart) } tmpQuery = fmt.Sprintf("%s order by timestamp asc, id asc limit 1000", tmpQuery) zap.S().Debug(tmpQuery) - response := &[]model.GetLogsResponse{} - err := r.db.Select(ctx, response, tmpQuery) + response := []model.GetLogsResponse{} + err := r.db.Select(ctx, &response, tmpQuery) if err != nil { zap.S().Debug(err) client.Error <- err return } - len := len(*response) + len := len(response) for i := 0; i < len; i++ { select { case <-ctx.Done(): @@ -2952,16 +2951,15 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC zap.S().Debug("closing go routine while sending logs : " + client.Name) return default: - client.Logs <- &(*response)[i] + client.Logs <- &response[i] if i == len-1 { - tsStart = &(*response)[i].Timestamp - idStart = &(*response)[i].ID + tsStart = response[i].Timestamp + idStart = response[i].ID } } } if len == 0 { - currentTime := uint64(time.Now().UnixNano() / int64(time.Millisecond)) - tsStart = ¤tTime + tsStart = uint64(time.Now().UnixNano() / int64(time.Millisecond)) } time.Sleep(2 * time.Second) } From f8be4a6d5bf6c2280604dd78c01b954f9977a698 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 15:49:50 +0530 Subject: [PATCH 17/32] livetail timestamp correction --- pkg/query-service/app/clickhouseReader/reader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ff20d22cfd..4428ed38cf 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2908,7 +2908,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ "from %s.%s", r.logsDB, r.logsTable) - tsStart := uint64(time.Now().UnixNano() / int64(time.Millisecond)) + tsStart := uint64(time.Now().UnixNano()) if client.Filter.TimestampStart != nil { tsStart = *client.Filter.TimestampStart } @@ -2959,7 +2959,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } } if len == 0 { - tsStart = uint64(time.Now().UnixNano() / int64(time.Millisecond)) + tsStart = uint64(time.Now().UnixNano()) } time.Sleep(2 * time.Second) } From 373cbbc3758e70dfc650257465a2fec0ca722272 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 16:07:19 +0530 Subject: [PATCH 18/32] logs select statement converted to a const --- .../app/clickhouseReader/reader.go | 21 +++---------------- pkg/query-service/constants/constants.go | 9 ++++++++ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 4428ed38cf..368ecf7944 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2861,13 +2861,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} } - query := fmt.Sprintf("SELECT "+ - "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ - "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,"+ - "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,"+ - "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,"+ - "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ - "from %s.%s", r.logsDB, r.logsTable) + query := fmt.Sprintf("%s from %s.%s", constants.LogsSQLSelect, r.logsDB, r.logsTable) if filterSql != "" { query += fmt.Sprintf(" where %s", filterSql) @@ -2900,13 +2894,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC return } - query := fmt.Sprintf("SELECT "+ - "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,"+ - "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,"+ - "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,"+ - "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,"+ - "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string "+ - "from %s.%s", r.logsDB, r.logsTable) + query := fmt.Sprintf("%s from %s.%s", constants.LogsSQLSelect, r.logsDB, r.logsTable) tsStart := uint64(time.Now().UnixNano()) if client.Filter.TimestampStart != nil { @@ -2958,10 +2946,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } } } - if len == 0 { - tsStart = uint64(time.Now().UnixNano()) - } - time.Sleep(2 * time.Second) + time.Sleep(10 * time.Second) } } } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index e5e727de0b..af96d2571f 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -139,3 +139,12 @@ var StaticSelectedLogFields = []model.LogField{ Type: Static, }, } + +const ( + LogsSQLSelect = "SELECT " + + "timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body," + + "CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string," + + "CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64," + + "CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64," + + "CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string " +) From 2f178983901f4700976460ba5c9af13427ffc309 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 16:49:40 +0530 Subject: [PATCH 19/32] primitive type pointers removed --- .../app/clickhouseReader/reader.go | 45 +++++------ pkg/query-service/app/logs/parser.go | 81 ++++++++++--------- pkg/query-service/app/logs/parser_test.go | 2 +- pkg/query-service/app/logs/validator.go | 6 +- pkg/query-service/model/queryParams.go | 40 ++++----- 5 files changed, 84 insertions(+), 90 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 368ecf7944..830db0043e 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2825,15 +2825,13 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda } // create the index - if field.IndexType == nil { - iType := constants.DefaultLogSkipIndexType - field.IndexType = &iType + if field.IndexType == "" { + field.IndexType = constants.DefaultLogSkipIndexType } - if field.IndexGranularity == nil { - granularity := constants.DefaultLogSkipIndexGranularity - field.IndexGranularity = &granularity + if field.IndexGranularity == 0 { + field.IndexGranularity = constants.DefaultLogSkipIndexGranularity } - query := fmt.Sprintf("ALTER TABLE %s.%s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d", r.logsDB, r.logsTable, field.Name, field.Name, *field.IndexType, *field.IndexGranularity) + query := fmt.Sprintf("ALTER TABLE %s.%s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d", r.logsDB, r.logsTable, field.Name, field.Name, field.IndexType, field.IndexGranularity) err := r.db.Exec(ctx, query) if err != nil { return &model.ApiError{Err: err, Typ: model.ErrorInternal} @@ -2897,13 +2895,13 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC query := fmt.Sprintf("%s from %s.%s", constants.LogsSQLSelect, r.logsDB, r.logsTable) tsStart := uint64(time.Now().UnixNano()) - if client.Filter.TimestampStart != nil { - tsStart = *client.Filter.TimestampStart + if client.Filter.TimestampStart != 0 { + tsStart = client.Filter.TimestampStart } var idStart string - if client.Filter.IdStart != nil { - idStart = *client.Filter.IdStart + if client.Filter.IdStart != "" { + idStart = client.Filter.IdStart } for { @@ -2954,14 +2952,9 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.LogsAggregateParams) (*model.GetLogsAggregatesResponse, *model.ApiError) { logAggregatesDBResponseItems := []model.LogsAggregatesDBResponseItem{} - groupBy := "" - if params.GroupBy != nil { - groupBy = *params.GroupBy - } - function := "toFloat64(count()) as value" - if params.Function != nil { - function = fmt.Sprintf("toFloat64(%s) as value", *params.Function) + if params.Function != "" { + function = fmt.Sprintf("toFloat64(%s) as value", params.Function) } fields, apiErr := r.GetLogFields(ctx) @@ -2977,22 +2970,22 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs } query := "" - if groupBy != "" { + if params.GroupBy != "" { query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000000000), INTERVAL %d minute))*1000000000) as time, toString(%s) as groupBy, "+ "%s "+ "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", - *params.StepSeconds/60, groupBy, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) + params.StepSeconds/60, params.GroupBy, function, r.logsDB, r.logsTable, params.TimestampStart, params.TimestampEnd) } else { query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000000000), INTERVAL %d minute))*1000000000) as time, "+ "%s "+ "FROM %s.%s WHERE timestamp >= '%d' AND timestamp <= '%d' ", - *params.StepSeconds/60, function, r.logsDB, r.logsTable, *params.TimestampStart, *params.TimestampEnd) + params.StepSeconds/60, function, r.logsDB, r.logsTable, params.TimestampStart, params.TimestampEnd) } if filterSql != "" { query += fmt.Sprintf(" AND %s ", filterSql) } - if groupBy != "" { - query += fmt.Sprintf("GROUP BY time, toString(%s) as groupBy ORDER BY time", groupBy) + if params.GroupBy != "" { + query += fmt.Sprintf("GROUP BY time, toString(%s) as groupBy ORDER BY time", params.GroupBy) } else { query += "GROUP BY time ORDER BY time" } @@ -3009,17 +3002,17 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs for i := range logAggregatesDBResponseItems { if elem, ok := aggregateResponse.Items[int64(logAggregatesDBResponseItems[i].Timestamp)]; ok { - if groupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { + if params.GroupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { elem.GroupBy[logAggregatesDBResponseItems[i].GroupBy] = logAggregatesDBResponseItems[i].Value } aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = elem } else { - if groupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { + if params.GroupBy != "" && logAggregatesDBResponseItems[i].GroupBy != "" { aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = model.LogsAggregatesResponseItem{ Timestamp: logAggregatesDBResponseItems[i].Timestamp, GroupBy: map[string]interface{}{logAggregatesDBResponseItems[i].GroupBy: logAggregatesDBResponseItems[i].Value}, } - } else if groupBy == "" { + } else if params.GroupBy == "" { aggregateResponse.Items[logAggregatesDBResponseItems[i].Timestamp] = model.LogsAggregatesResponseItem{ Timestamp: logAggregatesDBResponseItems[i].Timestamp, Value: logAggregatesDBResponseItems[i].Value, diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 3a2a7dd1e7..fb819337ce 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -22,7 +22,13 @@ var operatorMapping = map[string]string{ } const ( - AND = "and" + AND = "and" + ORDER = "order" + ORDER_BY = "orderBy" + TIMESTAMP_START = "timestampStart" + TIMESTAMP_END = "timestampEnd" + IDSTART = "idStart" + IDEND = "idEnd" ) var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[\S ]+(')?)`) @@ -42,36 +48,34 @@ func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { return nil, err } } - if val, ok := params["orderBy"]; ok { + if val, ok := params[ORDER_BY]; ok { res.OrderBy = val[0] } - if val, ok := params["order"]; ok { + if val, ok := params[ORDER]; ok { res.Order = val[0] } if val, ok := params["q"]; ok { - res.Query = &val[0] + res.Query = val[0] } - if val, ok := params["timestampStart"]; ok { + if val, ok := params[TIMESTAMP_START]; ok { ts, err := strconv.Atoi(val[0]) if err != nil { return nil, err } - ts64 := uint64(ts) - res.TimestampStart = &ts64 + res.TimestampStart = uint64(ts) } - if val, ok := params["timestampEnd"]; ok { + if val, ok := params[TIMESTAMP_END]; ok { ts, err := strconv.Atoi(val[0]) if err != nil { return nil, err } - ts64 := uint64(ts) - res.TimestampEnd = &ts64 + res.TimestampEnd = uint64(ts) } - if val, ok := params["idStart"]; ok { - res.IdStart = &val[0] + if val, ok := params[IDSTART]; ok { + res.IdStart = val[0] } - if val, ok := params["idEnd"]; ok { - res.IdEnd = &val[0] + if val, ok := params[IDEND]; ok { + res.IdEnd = val[0] } return &res, nil } @@ -80,18 +84,17 @@ func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error) res := model.LogsFilterParams{} params := r.URL.Query() if val, ok := params["q"]; ok { - res.Query = &val[0] + res.Query = val[0] } - if val, ok := params["timestampStart"]; ok { + if val, ok := params[TIMESTAMP_START]; ok { ts, err := strconv.Atoi(val[0]) if err != nil { return nil, err } - ts64 := uint64(ts) - res.TimestampStart = &ts64 + res.TimestampStart = uint64(ts) } - if val, ok := params["idStart"]; ok { - res.IdStart = &val[0] + if val, ok := params[IDSTART]; ok { + res.IdStart = val[0] } return &res, nil } @@ -99,37 +102,35 @@ func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error) func ParseLogAggregateParams(r *http.Request) (*model.LogsAggregateParams, error) { res := model.LogsAggregateParams{} params := r.URL.Query() - if val, ok := params["timestampStart"]; ok { + if val, ok := params[TIMESTAMP_START]; ok { ts, err := strconv.Atoi(val[0]) if err != nil { return nil, err } - ts64 := uint64(ts) - res.TimestampStart = &ts64 + res.TimestampStart = uint64(ts) } else { return nil, fmt.Errorf("timestampStart is required") } - if val, ok := params["timestampEnd"]; ok { + if val, ok := params[TIMESTAMP_END]; ok { ts, err := strconv.Atoi(val[0]) if err != nil { return nil, err } - ts64 := uint64(ts) - res.TimestampEnd = &ts64 + res.TimestampEnd = uint64(ts) } else { return nil, fmt.Errorf("timestampEnd is required") } if val, ok := params["q"]; ok { - res.Query = &val[0] + res.Query = val[0] } if val, ok := params["groupBy"]; ok { - res.GroupBy = &val[0] + res.GroupBy = val[0] } if val, ok := params["function"]; ok { - res.Function = &val[0] + res.Function = val[0] } if val, ok := params["step"]; ok { @@ -137,7 +138,7 @@ func ParseLogAggregateParams(r *http.Request) (*model.LogsAggregateParams, error if err != nil { return nil, err } - res.StepSeconds = &step + res.StepSeconds = step } else { return nil, fmt.Errorf("step is required") } @@ -246,8 +247,8 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt var tokens []string var err error var sqlWhere string - if params.Query != nil { - tokens, err = parseLogQuery(*params.Query) + if params.Query != "" { + tokens, err = parseLogQuery(params.Query) if err != nil { return sqlWhere, err } @@ -258,29 +259,29 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt return sqlWhere, err } - if params.TimestampStart != nil { - filter := fmt.Sprintf("timestamp >= '%d' ", *params.TimestampStart) + if params.TimestampStart != 0 { + filter := fmt.Sprintf("timestamp >= '%d' ", params.TimestampStart) if len(tokens) > 0 { filter = "and " + filter } tokens = append(tokens, filter) } - if params.TimestampEnd != nil { - filter := fmt.Sprintf("timestamp <= '%d' ", *params.TimestampEnd) + if params.TimestampEnd != 0 { + filter := fmt.Sprintf("timestamp <= '%d' ", params.TimestampEnd) if len(tokens) > 0 { filter = "and " + filter } tokens = append(tokens, filter) } - if params.IdStart != nil { - filter := fmt.Sprintf("id > '%v' ", *params.IdStart) + if params.IdStart != "" { + filter := fmt.Sprintf("id > '%v' ", params.IdStart) if len(tokens) > 0 { filter = "and " + filter } tokens = append(tokens, filter) } - if params.IdEnd != nil { - filter := fmt.Sprintf("id < '%v' ", *params.IdEnd) + if params.IdEnd != "" { + filter := fmt.Sprintf("id < '%v' ", params.IdEnd) if len(tokens) > 0 { filter = "and " + filter } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index d2855ef1f8..84984acb3b 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -213,7 +213,7 @@ func TestGenerateSQLQuery(t *testing.T) { idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU" sqlWhere := "id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 and timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' " Convey("testInterestingFields", t, func() { - res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: &query, TimestampStart: &tsStart, TimestampEnd: &tsEnd, IdStart: &idStart, IdEnd: &idEnd}) + res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdStart: idStart, IdEnd: idEnd}) So(res, ShouldEqual, sqlWhere) }) } diff --git a/pkg/query-service/app/logs/validator.go b/pkg/query-service/app/logs/validator.go index 9277bda047..0a27a11b15 100644 --- a/pkg/query-service/app/logs/validator.go +++ b/pkg/query-service/app/logs/validator.go @@ -27,13 +27,13 @@ func ValidateUpdateFieldPayload(field *model.UpdateField) error { return fmt.Errorf("type %s not supported", field.Type) } - if field.IndexType != nil { - matched, err := regexp.MatchString(`^(minmax|set\([0-9]\)|bloom_filter\((0?.?[0-9]+|1)\)|tokenbf_v1\([0-9]+,[0-9]+,[0-9]+\)|ngrambf_v1\([0-9]+,[0-9]+,[0-9]+,[0-9]+\))$`, *field.IndexType) + if field.IndexType != "" { + matched, err := regexp.MatchString(`^(minmax|set\([0-9]\)|bloom_filter\((0?.?[0-9]+|1)\)|tokenbf_v1\([0-9]+,[0-9]+,[0-9]+\)|ngrambf_v1\([0-9]+,[0-9]+,[0-9]+,[0-9]+\))$`, field.IndexType) if err != nil { return err } if !matched { - return fmt.Errorf("index type %s not supported", *field.IndexType) + return fmt.Errorf("index type %s not supported", field.IndexType) } } return nil diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index ce9c4f53c3..f020a429ea 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -322,30 +322,30 @@ type FilterSet struct { } type UpdateField struct { - Name string `json:"name"` - DataType string `json:"dataType"` - Type string `json:"type"` - Selected bool `json:"selected"` - IndexType *string `json:"index"` - IndexGranularity *int `json:"indexGranularity"` + Name string `json:"name"` + DataType string `json:"dataType"` + Type string `json:"type"` + Selected bool `json:"selected"` + IndexType string `json:"index"` + IndexGranularity int `json:"indexGranularity"` } type LogsFilterParams struct { - Limit int `json:"limit"` - OrderBy string `json:"orderBy"` - Order string `json:"order"` - Query *string `json:"q"` - TimestampStart *uint64 `json:"timestampStart"` - TimestampEnd *uint64 `json:"timestampEnd"` - IdStart *string `json:"idStart"` - IdEnd *string `json:"idEnd"` + Limit int `json:"limit"` + OrderBy string `json:"orderBy"` + Order string `json:"order"` + Query string `json:"q"` + TimestampStart uint64 `json:"timestampStart"` + TimestampEnd uint64 `json:"timestampEnd"` + IdStart string `json:"idStart"` + IdEnd string `json:"idEnd"` } type LogsAggregateParams struct { - Query *string `json:"q"` - TimestampStart *uint64 `json:"timestampStart"` - TimestampEnd *uint64 `json:"timestampEnd"` - GroupBy *string `json:"groupBy"` - Function *string `json:"function"` - StepSeconds *int `json:"step"` + Query string `json:"q"` + TimestampStart uint64 `json:"timestampStart"` + TimestampEnd uint64 `json:"timestampEnd"` + GroupBy string `json:"groupBy"` + Function string `json:"function"` + StepSeconds int `json:"step"` } From 4825ed6e5ffc9266daa5be9f19d1f288f607c9a6 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 22 Jul 2022 17:19:55 +0530 Subject: [PATCH 20/32] dataType constant strings --- pkg/query-service/constants/constants.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index af96d2571f..b77d8dd7bf 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -99,30 +99,37 @@ func GetOrDefaultEnv(key string, fallback string) string { return v } +const ( + STRING = "String" + UINT32 = "UInt32" + LOWCARDINALITY_STRING = "LowCardinality(String)" + INT32 = "Int32" +) + var StaticInterestingLogFields = []model.LogField{ { Name: "trace_id", - DataType: "String", + DataType: STRING, Type: Static, }, { Name: "span_id", - DataType: "String", + DataType: STRING, Type: Static, }, { Name: "trace_flags", - DataType: "UInt32", + DataType: UINT32, Type: Static, }, { Name: "severity_text", - DataType: "LowCardinality(String)", + DataType: LOWCARDINALITY_STRING, Type: Static, }, { Name: "severity_number", - DataType: "Int32", + DataType: INT32, Type: Static, }, } @@ -130,12 +137,12 @@ var StaticInterestingLogFields = []model.LogField{ var StaticSelectedLogFields = []model.LogField{ { Name: "timestamp", - DataType: "UInt64", + DataType: UINT32, Type: Static, }, { Name: "id", - DataType: "String", + DataType: STRING, Type: Static, }, } From 0fe4327877c3e1382074391562e49fa61a1552d8 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 25 Jul 2022 14:42:58 +0530 Subject: [PATCH 21/32] live tail fetch only recent 100 logs every 10s --- .../app/clickhouseReader/options.go | 32 ++++++++------- .../app/clickhouseReader/reader.go | 40 ++++++++++--------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 930a5e9d70..aba35490cc 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -18,20 +18,21 @@ const ( ) const ( - defaultDatasource string = "tcp://localhost:9000" - defaultTraceDB string = "signoz_traces" - defaultOperationsTable string = "signoz_operations" - defaultIndexTable string = "signoz_index_v2" - defaultErrorTable string = "signoz_error_index_v2" - defaulDurationTable string = "durationSortMV" - defaultSpansTable string = "signoz_spans" - defaultLogsDB string = "signoz_logs" - defaultLogsTable string = "logs" - defaultLogAttributeKeysTable string = "logs_atrribute_keys" - defaultLogResourceKeysTable string = "logs_resource_keys" - defaultWriteBatchDelay time.Duration = 5 * time.Second - defaultWriteBatchSize int = 10000 - defaultEncoding Encoding = EncodingJSON + defaultDatasource string = "tcp://localhost:9000" + defaultTraceDB string = "signoz_traces" + defaultOperationsTable string = "signoz_operations" + defaultIndexTable string = "signoz_index_v2" + defaultErrorTable string = "signoz_error_index_v2" + defaulDurationTable string = "durationSortMV" + defaultSpansTable string = "signoz_spans" + defaultLogsDB string = "signoz_logs" + defaultLogsTable string = "logs" + defaultLogAttributeKeysTable string = "logs_atrribute_keys" + defaultLogResourceKeysTable string = "logs_resource_keys" + defaultLiveTailRefreshSeconds int = 10 + defaultWriteBatchDelay time.Duration = 5 * time.Second + defaultWriteBatchSize int = 10000 + defaultEncoding Encoding = EncodingJSON ) const ( @@ -60,6 +61,7 @@ type namespaceConfig struct { LogsTable string LogsAttributeKeysTable string LogsResourceKeysTable string + LiveTailRefreshSeconds int WriteBatchDelay time.Duration WriteBatchSize int Encoding Encoding @@ -123,6 +125,7 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s LogsTable: defaultLogsTable, LogsAttributeKeysTable: defaultLogAttributeKeysTable, LogsResourceKeysTable: defaultLogResourceKeysTable, + LiveTailRefreshSeconds: defaultLiveTailRefreshSeconds, WriteBatchDelay: defaultWriteBatchDelay, WriteBatchSize: defaultWriteBatchSize, Encoding: defaultEncoding, @@ -144,6 +147,7 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s LogsTable: "", LogsAttributeKeysTable: "", LogsResourceKeysTable: "", + LiveTailRefreshSeconds: defaultLiveTailRefreshSeconds, WriteBatchDelay: defaultWriteBatchDelay, WriteBatchSize: defaultWriteBatchSize, Encoding: defaultEncoding, diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 830db0043e..acaa52e5d4 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -94,6 +94,8 @@ type ClickHouseReader struct { promConfigFile string promConfig *config.Config alertManager am.Manager + + liveTailRefreshSeconds int } // NewTraceReader returns a TraceReader for the database @@ -116,20 +118,21 @@ func NewReader(localDB *sqlx.DB, configFile string) *ClickHouseReader { } return &ClickHouseReader{ - db: db, - localDB: localDB, - traceDB: options.primary.TraceDB, - alertManager: alertManager, - operationsTable: options.primary.OperationsTable, - indexTable: options.primary.IndexTable, - errorTable: options.primary.ErrorTable, - durationTable: options.primary.DurationTable, - spansTable: options.primary.SpansTable, - logsDB: options.primary.LogsDB, - logsTable: options.primary.LogsTable, - logsAttributeKeys: options.primary.LogsAttributeKeysTable, - logsResourceKeys: options.primary.LogsResourceKeysTable, - promConfigFile: configFile, + db: db, + localDB: localDB, + traceDB: options.primary.TraceDB, + alertManager: alertManager, + operationsTable: options.primary.OperationsTable, + indexTable: options.primary.IndexTable, + errorTable: options.primary.ErrorTable, + durationTable: options.primary.DurationTable, + spansTable: options.primary.SpansTable, + logsDB: options.primary.LogsDB, + logsTable: options.primary.LogsTable, + logsAttributeKeys: options.primary.LogsAttributeKeysTable, + logsResourceKeys: options.primary.LogsResourceKeysTable, + liveTailRefreshSeconds: options.primary.LiveTailRefreshSeconds, + promConfigFile: configFile, } } @@ -2912,6 +2915,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC zap.S().Debug("closing go routine : " + client.Name) return default: + // get the new 100 logs as anything more older won't make sense tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, tsStart) if filterSql != "" { tmpQuery += fmt.Sprintf(" and %s", filterSql) @@ -2919,7 +2923,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC if idStart != "" { tmpQuery += fmt.Sprintf(" and id > '%s'", idStart) } - tmpQuery = fmt.Sprintf("%s order by timestamp asc, id asc limit 1000", tmpQuery) + tmpQuery = fmt.Sprintf("%s order by timestamp desc, id desc limit 100", tmpQuery) zap.S().Debug(tmpQuery) response := []model.GetLogsResponse{} err := r.db.Select(ctx, &response, tmpQuery) @@ -2929,7 +2933,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC return } len := len(response) - for i := 0; i < len; i++ { + for i := len - 1; i >= 0; i-- { select { case <-ctx.Done(): done := true @@ -2938,13 +2942,13 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC return default: client.Logs <- &response[i] - if i == len-1 { + if i == 0 { tsStart = response[i].Timestamp idStart = response[i].ID } } } - time.Sleep(10 * time.Second) + time.Sleep(time.Duration(r.liveTailRefreshSeconds) * time.Second) } } } From 294d527a0e33f4f0189f69fb8ebd3ffa72d03e8b Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 26 Jul 2022 14:45:20 +0530 Subject: [PATCH 22/32] parser updated to support more than one contains --- pkg/query-service/app/logs/parser.go | 2 +- pkg/query-service/app/logs/parser_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index fb819337ce..ec1fb773d4 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -31,7 +31,7 @@ const ( IDEND = "idEnd" ) -var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[\S ]+(')?)`) +var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[^']+(')?)`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index 84984acb3b..517247122e 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -32,6 +32,11 @@ var correctQueriesTest = []struct { `resource contains 'Hello, "World"'`, []string{`resource ILIKE '%Hello, "World"%' `}, }, + { + `more than one continas`, + `resource contains 'Hello, "World"' and myresource contains 'abcd'`, + []string{`resource ILIKE '%Hello, "World"%' `, `AND myresource ILIKE '%abcd%' `}, + }, { `filters with lt,gt,lte,gte operators`, `id lt 100 and id gt 50 and code lte 500 and code gte 400`, From baf72610d6d516d941aee1515f6e0c1c7fc47f1d Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Wed, 27 Jul 2022 10:39:08 +0530 Subject: [PATCH 23/32] Update pkg/query-service/app/clickhouseReader/reader.go Co-authored-by: Srikanth Chekuri --- pkg/query-service/app/clickhouseReader/reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index acaa52e5d4..a2f56ad7f3 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2928,7 +2928,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC response := []model.GetLogsResponse{} err := r.db.Select(ctx, &response, tmpQuery) if err != nil { - zap.S().Debug(err) + zap.S().Error(err) client.Error <- err return } From 7b6a086b37e967e77bd5f750512f10f1883667c6 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 27 Jul 2022 10:46:33 +0530 Subject: [PATCH 24/32] consistant query formatting --- pkg/query-service/app/clickhouseReader/reader.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index a2f56ad7f3..f67fa559eb 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2865,7 +2865,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter query := fmt.Sprintf("%s from %s.%s", constants.LogsSQLSelect, r.logsDB, r.logsTable) if filterSql != "" { - query += fmt.Sprintf(" where %s", filterSql) + query = fmt.Sprintf("%s where %s", query, filterSql) } query = fmt.Sprintf("%s order by %s %s limit %d", query, params.OrderBy, params.Order, params.Limit) @@ -2918,10 +2918,10 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC // get the new 100 logs as anything more older won't make sense tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, tsStart) if filterSql != "" { - tmpQuery += fmt.Sprintf(" and %s", filterSql) + tmpQuery = fmt.Sprintf("%s and %s", tmpQuery, filterSql) } if idStart != "" { - tmpQuery += fmt.Sprintf(" and id > '%s'", idStart) + tmpQuery = fmt.Sprintf("%s and id > '%s'", tmpQuery, idStart) } tmpQuery = fmt.Sprintf("%s order by timestamp desc, id desc limit 100", tmpQuery) zap.S().Debug(tmpQuery) @@ -2932,8 +2932,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC client.Error <- err return } - len := len(response) - for i := len - 1; i >= 0; i-- { + for i := len(response) - 1; i >= 0; i-- { select { case <-ctx.Done(): done := true @@ -2986,12 +2985,12 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs params.StepSeconds/60, function, r.logsDB, r.logsTable, params.TimestampStart, params.TimestampEnd) } if filterSql != "" { - query += fmt.Sprintf(" AND %s ", filterSql) + query = fmt.Sprintf("%s AND %s ", query, filterSql) } if params.GroupBy != "" { - query += fmt.Sprintf("GROUP BY time, toString(%s) as groupBy ORDER BY time", params.GroupBy) + query = fmt.Sprintf("%s GROUP BY time, toString(%s) as groupBy ORDER BY time", query, params.GroupBy) } else { - query += "GROUP BY time ORDER BY time" + query = fmt.Sprintf("%s GROUP BY time ORDER BY time", query) } zap.S().Debug(query) From 047227ad188bbc50cd7ea9bd9d01b61166a869b6 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 27 Jul 2022 11:47:35 +0530 Subject: [PATCH 25/32] use ticker for polling db in live tail. --- pkg/query-service/app/clickhouseReader/reader.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index f67fa559eb..b5e6948197 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2907,6 +2907,8 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC idStart = client.Filter.IdStart } + ticker := time.NewTicker(time.Duration(r.liveTailRefreshSeconds) * time.Second) + defer ticker.Stop() for { select { case <-ctx.Done(): @@ -2914,7 +2916,7 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC client.Done <- &done zap.S().Debug("closing go routine : " + client.Name) return - default: + case <-ticker.C: // get the new 100 logs as anything more older won't make sense tmpQuery := fmt.Sprintf("%s where timestamp >='%d'", query, tsStart) if filterSql != "" { @@ -2947,7 +2949,6 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } } } - time.Sleep(time.Duration(r.liveTailRefreshSeconds) * time.Second) } } } From d15f9a170994a83268180cabfa44a9ce516f1d46 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 27 Jul 2022 15:58:58 +0530 Subject: [PATCH 26/32] log statement corrected --- pkg/query-service/app/http_handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index c97ffd932f..c90893a55d 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1955,8 +1955,8 @@ func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { case <-client.Done: zap.S().Debug("done!") return - case <-client.Error: - zap.S().Debug("error occured!") + case err := <-client.Error: + zap.S().Error("error occured!", err) return } } From 6eb9389e81630dd26bb7775c201bd03d7188ab8d Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 1 Aug 2022 12:17:15 +0530 Subject: [PATCH 27/32] parser updated to include or as well --- pkg/query-service/app/logs/parser.go | 27 +++++++++++++++-------- pkg/query-service/app/logs/parser_test.go | 9 ++++++-- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index ec1fb773d4..8b24c940ac 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -31,7 +31,7 @@ const ( IDEND = "idEnd" ) -var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[^']+(')?)`) +var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[^']+(')?)`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { @@ -259,33 +259,42 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt return sqlWhere, err } + filterTokens := []string{} if params.TimestampStart != 0 { filter := fmt.Sprintf("timestamp >= '%d' ", params.TimestampStart) - if len(tokens) > 0 { + if len(filterTokens) > 0 { filter = "and " + filter } - tokens = append(tokens, filter) + filterTokens = append(filterTokens, filter) } if params.TimestampEnd != 0 { filter := fmt.Sprintf("timestamp <= '%d' ", params.TimestampEnd) - if len(tokens) > 0 { + if len(filterTokens) > 0 { filter = "and " + filter } - tokens = append(tokens, filter) + filterTokens = append(filterTokens, filter) } if params.IdStart != "" { filter := fmt.Sprintf("id > '%v' ", params.IdStart) - if len(tokens) > 0 { + if len(filterTokens) > 0 { filter = "and " + filter } - tokens = append(tokens, filter) + filterTokens = append(filterTokens, filter) } if params.IdEnd != "" { filter := fmt.Sprintf("id < '%v' ", params.IdEnd) - if len(tokens) > 0 { + if len(filterTokens) > 0 { filter = "and " + filter } - tokens = append(tokens, filter) + filterTokens = append(filterTokens, filter) + } + + if len(filterTokens) > 0 { + if len(tokens) > 0 { + tokens[0] = fmt.Sprintf("and %s", tokens[0]) + } + filterTokens = append(filterTokens, tokens...) + tokens = filterTokens } sqlWhere = strings.Join(tokens, "") diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index 517247122e..819f9d0c9e 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -42,6 +42,11 @@ var correctQueriesTest = []struct { `id lt 100 and id gt 50 and code lte 500 and code gte 400`, []string{`id < 100 `, `and id > 50 `, `and code <= 500 `, `and code >= 400 `}, }, + { + `filters with lt,gt,lte,gte operators seprated by OR`, + `id lt 100 or id gt 50 or code lte 500 or code gte 400`, + []string{`id < 100 `, `or id > 50 `, `or code <= 500 `, `or code >= 400 `}, + }, { `filter with number`, `status gte 200 AND FULLTEXT ncontains '"key"'`, @@ -216,8 +221,8 @@ func TestGenerateSQLQuery(t *testing.T) { tsEnd := uint64(1657689294000) idStart := "2BsKLKv8cZrLCn6rkOcRGkdjBdM" idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU" - sqlWhere := "id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 and timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' " - Convey("testInterestingFields", t, func() { + sqlWhere := "timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' and id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 " + Convey("testGenerateSQL", t, func() { res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdStart: idStart, IdEnd: idEnd}) So(res, ShouldEqual, sqlWhere) }) From 5894acdb2d484a2350ea08b508d302f1b0605fc6 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 1 Aug 2022 12:30:11 +0530 Subject: [PATCH 28/32] OR support added with contains --- pkg/query-service/app/logs/parser.go | 5 ++++- pkg/query-service/app/logs/parser_test.go | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 8b24c940ac..d709036cf3 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -23,6 +23,7 @@ var operatorMapping = map[string]string{ const ( AND = "and" + OR = "or" ORDER = "order" ORDER_BY = "orderBy" TIMESTAMP_START = "timestampStart" @@ -168,7 +169,7 @@ func parseLogQuery(query string) ([]string, error) { operatorRemovedTokens := strings.Split(operatorRegex.ReplaceAllString(v, " "), " ") searchCol := strings.ToLower(operatorRemovedTokens[0]) - if searchCol == AND { + if searchCol == AND || searchCol == OR { searchCol = strings.ToLower(operatorRemovedTokens[1]) } col := searchCol @@ -179,6 +180,8 @@ func parseLogQuery(query string) ([]string, error) { f := fmt.Sprintf(`%s %s '%%%s%%' `, col, operatorMapping[opLower], searchString[1:len(searchString)-1]) if strings.HasPrefix(strings.ToLower(v), AND) { f = "AND " + f + } else if strings.HasPrefix(strings.ToLower(v), OR) { + f = "OR " + f } sqlQueryTokens = append(sqlQueryTokens, f) } else { diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index 819f9d0c9e..cf8d12a803 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -37,6 +37,11 @@ var correctQueriesTest = []struct { `resource contains 'Hello, "World"' and myresource contains 'abcd'`, []string{`resource ILIKE '%Hello, "World"%' `, `AND myresource ILIKE '%abcd%' `}, }, + { + "contains with or", + `id in ('2CkBCauK8m3nkyKR19YhCw6WbdY') or fulltext contains 'OPTIONS /api/v1/logs'`, + []string{`id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY') `, `OR body ILIKE '%OPTIONS /api/v1/logs%' `}, + }, { `filters with lt,gt,lte,gte operators`, `id lt 100 and id gt 50 and code lte 500 and code gte 400`, From 594bfc256c4e1da1a6c9d335fa73a8f1d409e976 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 1 Aug 2022 13:02:00 +0530 Subject: [PATCH 29/32] fulltext validation updated --- pkg/query-service/app/logs/parser.go | 6 +++--- pkg/query-service/app/logs/parser_test.go | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index d709036cf3..103e959d4a 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -32,7 +32,7 @@ const ( IDEND = "idEnd" ) -var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([\S ]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) (')?[^']+(')?)`) +var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([^(]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) '[^']+')`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { @@ -204,7 +204,7 @@ func parseColumn(s string) (*string, error) { return nil, fmt.Errorf("incorrect filter") } - if strings.HasPrefix(s, AND) { + if strings.HasPrefix(s, AND) || strings.HasPrefix(s, OR) { colName = filter[1] } else { colName = filter[0] @@ -237,7 +237,7 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens [] if _, ok := selectedFieldsLookup[*col]; !ok && *col != "body" { if field, ok := interestingFieldLookup[*col]; ok { sqlColName = fmt.Sprintf("%s_%s_value[indexOf(%s_%s_key, '%s')]", field.Type, strings.ToLower(field.DataType), field.Type, strings.ToLower(field.DataType), *col) - } else { + } else if strings.Compare(strings.ToLower(*col), "fulltext") != 0 { return nil, fmt.Errorf("field not found for filtering") } } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index cf8d12a803..df35ffbe80 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -42,6 +42,11 @@ var correctQueriesTest = []struct { `id in ('2CkBCauK8m3nkyKR19YhCw6WbdY') or fulltext contains 'OPTIONS /api/v1/logs'`, []string{`id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY') `, `OR body ILIKE '%OPTIONS /api/v1/logs%' `}, }, + { + "mixing and or", + `id in ('2CkBCauK8m3nkyKR19YhCw6WbdY') and id in ('2CkBCauK8m3nkyKR19YhCw6WbdY','2CkBCauK8m3nkyKR19YhCw6WbdY') or fulltext contains 'OPTIONS /api/v1/logs'`, + []string{`id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY') `, `and id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY','2CkBCauK8m3nkyKR19YhCw6WbdY') `, `OR body ILIKE '%OPTIONS /api/v1/logs%' `}, + }, { `filters with lt,gt,lte,gte operators`, `id lt 100 and id gt 50 and code lte 500 and code gte 400`, From 9dcf913a74c7719be9cf9ec2ef877e2d1c14adb1 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 3 Aug 2022 12:23:00 +0530 Subject: [PATCH 30/32] severity_number type changed to int8 --- pkg/query-service/constants/constants.go | 3 ++- pkg/query-service/model/response.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index b77d8dd7bf..a730c1ee2f 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -104,6 +104,7 @@ const ( UINT32 = "UInt32" LOWCARDINALITY_STRING = "LowCardinality(String)" INT32 = "Int32" + UINT8 = "Uint8" ) var StaticInterestingLogFields = []model.LogField{ @@ -129,7 +130,7 @@ var StaticInterestingLogFields = []model.LogField{ }, { Name: "severity_number", - DataType: INT32, + DataType: UINT8, Type: Static, }, } diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index ea4abe53d4..b6d4a056f9 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -427,7 +427,7 @@ type GetLogsResponse struct { SpanID string `json:"spanId" ch:"span_id"` TraceFlags uint32 `json:"traceFlags" ch:"trace_flags"` SeverityText string `json:"severityText" ch:"severity_text"` - SeverityNumber int32 `json:"severityNumber" ch:"severity_number"` + SeverityNumber uint8 `json:"severityNumber" ch:"severity_number"` Body string `json:"body" ch:"body"` Resources_string map[string]string `json:"resourcesString" ch:"resources_string"` Attributes_string map[string]string `json:"attributesString" ch:"attributes_string"` From 61ebd3aded491a984ad423edfb3de2bc85b7d9df Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 4 Aug 2022 14:28:10 +0530 Subject: [PATCH 31/32] logs ttl support added in ttl api --- .../app/clickhouseReader/reader.go | 96 ++++++++++++++++++- pkg/query-service/app/parser.go | 8 +- pkg/query-service/constants/constants.go | 1 + pkg/query-service/model/response.go | 4 + 4 files changed, 104 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index b5e6948197..49fe724df5 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1888,7 +1888,7 @@ func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, query return &GetFilteredSpansAggregatesResponse, nil } -// SetTTL sets the TTL for traces or metrics tables. +// SetTTL sets the TTL for traces or metrics or logs tables. // This is an async API which creates goroutines to set TTL. // Status of TTL update is tracked with ttl_status table in sqlite db. func (r *ClickHouseReader) SetTTL(ctx context.Context, @@ -2017,6 +2017,59 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, return } }(tableName) + case constants.LogsTTL: + tableName = r.logsDB + "." + r.logsTable + statusItem, err := r.checkTTLStatusItem(ctx, tableName) + if err != nil { + return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")} + } + if statusItem.Status == constants.StatusPending { + return nil, &model.ApiError{Typ: model.ErrorConflict, Err: fmt.Errorf("TTL is already running")} + } + go func(tableName string) { + _, dbErr := r.localDB.Exec("INSERT INTO ttl_status (transaction_id, created_at, updated_at, table_name, ttl, status, cold_storage_ttl) VALUES (?, ?, ?, ?, ?, ?, ?)", uuid, time.Now(), time.Now(), tableName, params.DelDuration, constants.StatusPending, coldStorageDuration) + if dbErr != nil { + zap.S().Error(fmt.Errorf("error in inserting to ttl_status table: %s", dbErr.Error())) + return + } + req = fmt.Sprintf( + "ALTER TABLE %v MODIFY TTL toDateTime(timestamp / 1000000000) + "+ + "INTERVAL %v SECOND DELETE", tableName, params.DelDuration) + if len(params.ColdStorageVolume) > 0 { + req += fmt.Sprintf(", toDateTime(timestamp / 1000000000)"+ + " + INTERVAL %v SECOND TO VOLUME '%s'", + params.ToColdStorageDuration, params.ColdStorageVolume) + } + err := r.setColdStorage(context.Background(), tableName, params.ColdStorageVolume) + if err != nil { + zap.S().Error(fmt.Errorf("error in setting cold storage: %s", err.Err.Error())) + statusItem, err := r.checkTTLStatusItem(ctx, tableName) + if err == nil { + _, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id) + if dbErr != nil { + zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr) + return + } + } + return + } + zap.S().Debugf("Executing TTL request: %s\n", req) + statusItem, _ := r.checkTTLStatusItem(ctx, tableName) + if err := r.db.Exec(ctx, req); err != nil { + zap.S().Error(fmt.Errorf("error while setting ttl. Err=%v", err)) + _, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id) + if dbErr != nil { + zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr) + return + } + return + } + _, dbErr = r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusSuccess, statusItem.Id) + if dbErr != nil { + zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr) + return + } + }(tableName) default: return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while setting ttl. ttl type should be , got %v", @@ -2180,6 +2233,24 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa } } + getLogsTTL := func() (*model.DBResponseTTL, *model.ApiError) { + var dbResp []model.DBResponseTTL + + query := fmt.Sprintf("SELECT engine_full FROM system.tables WHERE name='%v' AND database='%v'", r.logsTable, r.logsDB) + + err := r.db.Select(ctx, &dbResp, query) + + if err != nil { + zap.S().Error(fmt.Errorf("error while getting ttl. Err=%v", err)) + return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while getting ttl. Err=%v", err)} + } + if len(dbResp) == 0 { + return nil, nil + } else { + return &dbResp[0], nil + } + } + switch ttlParams.Type { case constants.TraceTTL: tableNameArray := []string{signozTraceDBName + "." + signozTraceTableName, signozTraceDBName + "." + signozDurationMVTable, signozTraceDBName + "." + signozSpansTable, signozTraceDBName + "." + signozErrorIndexTable} @@ -2224,6 +2295,29 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa delTTL, moveTTL := parseTTL(dbResp.EngineFull) return &model.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL, ExpectedMetricsTime: ttlQuery.TTL, ExpectedMetricsMoveTime: ttlQuery.ColdStorageTtl, Status: status}, nil + + case constants.LogsTTL: + tableNameArray := []string{r.logsDB + "." + r.logsTable} + status, err := r.setTTLQueryStatus(ctx, tableNameArray) + if err != nil { + return nil, err + } + dbResp, err := getLogsTTL() + if err != nil { + return nil, err + } + ttlQuery, err := r.checkTTLStatusItem(ctx, tableNameArray[0]) + if err != nil { + return nil, err + } + ttlQuery.TTL = ttlQuery.TTL / 3600 // convert to hours + if ttlQuery.ColdStorageTtl != -1 { + ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600 // convert to hours + } + + delTTL, moveTTL := parseTTL(dbResp.EngineFull) + return &model.GetTTLResponseItem{LogsTime: delTTL, LogsMoveTime: moveTTL, ExpectedLogsTime: ttlQuery.TTL, ExpectedLogsMoveTime: ttlQuery.ColdStorageTtl, Status: status}, nil + default: return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while getting ttl. ttl type should be metrics|traces, got %v", ttlParams.Type)} diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index e81b986a3d..ddd0ebd42d 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -467,8 +467,8 @@ func parseCountErrorsRequest(r *http.Request) (*model.CountErrorsParams, error) } params := &model.CountErrorsParams{ - Start: startTime, - End: endTime, + Start: startTime, + End: endTime, } return params, nil @@ -590,7 +590,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) { } // Validate the type parameter - if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL { + if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL { return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL) } @@ -629,7 +629,7 @@ func parseGetTTL(r *http.Request) (*model.GetTTLParams, error) { return nil, fmt.Errorf("type param cannot be empty from the query") } else { // Validate the type parameter - if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL { + if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL { return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL) } } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index a730c1ee2f..2e01c976cb 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -26,6 +26,7 @@ func IsTelemetryEnabled() bool { const TraceTTL = "traces" const MetricsTTL = "metrics" +const LogsTTL = "logs" func GetAlertManagerApiPrefix() string { if os.Getenv("ALERTMANAGER_API_PREFIX") != "" { diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index b6d4a056f9..feca7de5aa 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -274,10 +274,14 @@ type GetTTLResponseItem struct { MetricsMoveTime int `json:"metrics_move_ttl_duration_hrs,omitempty"` TracesTime int `json:"traces_ttl_duration_hrs,omitempty"` TracesMoveTime int `json:"traces_move_ttl_duration_hrs,omitempty"` + LogsTime int `json:"logs_ttl_duration_hrs,omitempty"` + LogsMoveTime int `json:"logs_move_ttl_duration_hrs,omitempty"` ExpectedMetricsTime int `json:"expected_metrics_ttl_duration_hrs,omitempty"` ExpectedMetricsMoveTime int `json:"expected_metrics_move_ttl_duration_hrs,omitempty"` ExpectedTracesTime int `json:"expected_traces_ttl_duration_hrs,omitempty"` ExpectedTracesMoveTime int `json:"expected_traces_move_ttl_duration_hrs,omitempty"` + ExpectedLogsTime int `json:"expected_logs_ttl_duration_hrs,omitempty"` + ExpectedLogsMoveTime int `json:"expected_logs_move_ttl_duration_hrs,omitempty"` Status string `json:"status"` } From eb28ece68049ca32548486129a684124def13dcf Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 10 Aug 2022 14:27:46 +0530 Subject: [PATCH 32/32] parser updated for pagination --- .../app/clickhouseReader/reader.go | 12 +++- pkg/query-service/app/http_handler.go | 2 +- pkg/query-service/app/logs/parser.go | 38 +++++++---- pkg/query-service/app/logs/parser_test.go | 65 ++++++++++++++++++- pkg/query-service/app/parser.go | 4 +- pkg/query-service/model/queryParams.go | 4 +- 6 files changed, 104 insertions(+), 21 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index dabdd3df20..1367c51a27 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3047,6 +3047,7 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter return nil, apiErr } + isPaginatePrev := logs.CheckIfPrevousPaginateAndModifyOrder(params) filterSql, err := logs.GenerateSQLWhere(fields, params) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} @@ -3064,7 +3065,12 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal} } - + if isPaginatePrev { + // rever the results from db + for i, j := 0, len(response)-1; i < j; i, j = i+1, j-1 { + response[i], response[j] = response[j], response[i] + } + } return &response, nil } @@ -3093,8 +3099,8 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC } var idStart string - if client.Filter.IdStart != "" { - idStart = client.Filter.IdStart + if client.Filter.IdGt != "" { + idStart = client.Filter.IdGt } ticker := time.NewTicker(time.Duration(r.liveTailRefreshSeconds) * time.Second) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index ae90926421..86f74e9457 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1902,7 +1902,7 @@ func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter.HandleFunc("", ViewAccess(aH.getLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/tail", ViewAccess(aH.tailLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", ViewAccess(aH.logFields)).Methods(http.MethodGet) - subRouter.HandleFunc("/fields", ViewAccess(aH.logFieldUpdate)).Methods(http.MethodPost) + subRouter.HandleFunc("/fields", EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) subRouter.HandleFunc("/aggregate", ViewAccess(aH.logAggregate)).Methods(http.MethodGet) } diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 103e959d4a..53b168b17f 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -28,8 +28,11 @@ const ( ORDER_BY = "orderBy" TIMESTAMP_START = "timestampStart" TIMESTAMP_END = "timestampEnd" - IDSTART = "idStart" - IDEND = "idEnd" + IdGt = "idGt" + IdLT = "idLt" + TIMESTAMP = "timestamp" + ASC = "asc" + DESC = "desc" ) var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([^(]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) '[^']+')`) @@ -72,11 +75,11 @@ func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { } res.TimestampEnd = uint64(ts) } - if val, ok := params[IDSTART]; ok { - res.IdStart = val[0] + if val, ok := params[IdGt]; ok { + res.IdGt = val[0] } - if val, ok := params[IDEND]; ok { - res.IdEnd = val[0] + if val, ok := params[IdLT]; ok { + res.IdLT = val[0] } return &res, nil } @@ -94,8 +97,8 @@ func ParseLiveTailFilterParams(r *http.Request) (*model.LogsFilterParams, error) } res.TimestampStart = uint64(ts) } - if val, ok := params[IDSTART]; ok { - res.IdStart = val[0] + if val, ok := params[IdGt]; ok { + res.IdGt = val[0] } return &res, nil } @@ -246,6 +249,17 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens [] return queryTokens, nil } +func CheckIfPrevousPaginateAndModifyOrder(params *model.LogsFilterParams) (isPaginatePrevious bool) { + if params.IdGt != "" && params.OrderBy == TIMESTAMP && params.Order == DESC { + isPaginatePrevious = true + params.Order = ASC + } else if params.IdLT != "" && params.OrderBy == TIMESTAMP && params.Order == ASC { + isPaginatePrevious = true + params.Order = DESC + } + return +} + func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, error) { var tokens []string var err error @@ -277,15 +291,15 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt } filterTokens = append(filterTokens, filter) } - if params.IdStart != "" { - filter := fmt.Sprintf("id > '%v' ", params.IdStart) + if params.IdGt != "" { + filter := fmt.Sprintf("id > '%v' ", params.IdGt) if len(filterTokens) > 0 { filter = "and " + filter } filterTokens = append(filterTokens, filter) } - if params.IdEnd != "" { - filter := fmt.Sprintf("id < '%v' ", params.IdEnd) + if params.IdLT != "" { + filter := fmt.Sprintf("id < '%v' ", params.IdLT) if len(filterTokens) > 0 { filter = "and " + filter } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index df35ffbe80..ff47632a4b 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -208,6 +208,69 @@ func TestReplaceInterestingFields(t *testing.T) { }) } +var previousPaginateTestCases = []struct { + Name string + Filter model.LogsFilterParams + IsPaginatePrev bool + Order string +}{ + { + Name: "empty", + Filter: model.LogsFilterParams{}, + IsPaginatePrev: false, + }, + { + Name: "next ordery by asc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: ASC, + IdGt: "myid", + }, + IsPaginatePrev: false, + Order: ASC, + }, + { + Name: "next ordery by desc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: DESC, + IdLT: "myid", + }, + IsPaginatePrev: false, + Order: DESC, + }, + { + Name: "prev ordery by desc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: DESC, + IdGt: "myid", + }, + IsPaginatePrev: true, + Order: ASC, + }, + { + Name: "prev ordery by asc", + Filter: model.LogsFilterParams{ + OrderBy: TIMESTAMP, + Order: ASC, + IdLT: "myid", + }, + IsPaginatePrev: true, + Order: DESC, + }, +} + +func TestCheckIfPrevousPaginateAndModifyOrder(t *testing.T) { + for _, test := range previousPaginateTestCases { + Convey(test.Name, t, func() { + isPrevPaginate := CheckIfPrevousPaginateAndModifyOrder(&test.Filter) + So(isPrevPaginate, ShouldEqual, test.IsPaginatePrev) + So(test.Order, ShouldEqual, test.Filter.Order) + }) + } +} + func TestGenerateSQLQuery(t *testing.T) { allFields := model.GetFieldsResponse{ Selected: []model.LogField{ @@ -233,7 +296,7 @@ func TestGenerateSQLQuery(t *testing.T) { idEnd := "2BsKG6tRpFWjYMcWsAGKfSxoQdU" sqlWhere := "timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' and id < 100 and id > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 " Convey("testGenerateSQL", t, func() { - res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdStart: idStart, IdEnd: idEnd}) + res, _ := GenerateSQLWhere(&allFields, &model.LogsFilterParams{Query: query, TimestampStart: tsStart, TimestampEnd: tsEnd, IdGt: idStart, IdLT: idEnd}) So(res, ShouldEqual, sqlWhere) }) } diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index 1e2935c06f..e7291e67a9 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -591,7 +591,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) { // Validate the type parameter if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL { - return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL) + return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL) } // Validate the TTL duration. @@ -630,7 +630,7 @@ func parseGetTTL(r *http.Request) (*model.GetTTLParams, error) { } else { // Validate the type parameter if typeTTL != constants.TraceTTL && typeTTL != constants.MetricsTTL && typeTTL != constants.LogsTTL { - return nil, fmt.Errorf("type param should be metrics|traces, got %v", typeTTL) + return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL) } } diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index c8cfa2dcee..33827d63c1 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -337,8 +337,8 @@ type LogsFilterParams struct { Query string `json:"q"` TimestampStart uint64 `json:"timestampStart"` TimestampEnd uint64 `json:"timestampEnd"` - IdStart string `json:"idStart"` - IdEnd string `json:"idEnd"` + IdGt string `json:"idGt"` + IdLT string `json:"idLt"` } type LogsAggregateParams struct {