GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-06-04 11:25:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: use materialized view for usage explorer API (#1466)

Browse Source 
* feat: use materialized view for usage explorer API

Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
This commit is contained in:
Vishal Sharma 2022-08-04 12:55:21 +05:30 committed by GitHub
parent 3968f11b3d
commit a742c9aee1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/query-service/app/clickhouseReader/options.go
View File

@ -24,6 +24,7 @@ const (
defaultIndexTable string = "signoz_index_v2" defaultIndexTable string = "signoz_index_v2"
defaultErrorTable string = "signoz_error_index_v2" defaultErrorTable string = "signoz_error_index_v2"
defaultDurationTable string = "durationSortMV" defaultDurationTable string = "durationSortMV"
defaultUsageExplorerTable string = "usage_explorer"
defaultSpansTable string = "signoz_spans" defaultSpansTable string = "signoz_spans"
defaultDependencyGraphTable string = "dependency_graph_minutes" defaultDependencyGraphTable string = "dependency_graph_minutes"
defaultTopLevelOperationsTable string = "top_level_operations" defaultTopLevelOperationsTable string = "top_level_operations"
@ -52,6 +53,7 @@ type namespaceConfig struct {
OperationsTable string OperationsTable string
IndexTable string IndexTable string
DurationTable string DurationTable string
UsageExplorerTable string
SpansTable string SpansTable string
ErrorTable string ErrorTable string
DependencyGraphTable string DependencyGraphTable string
@ -114,6 +116,7 @@ func NewOptions(datasource string, primaryNamespace string, otherNamespaces ...s
IndexTable: defaultIndexTable, IndexTable: defaultIndexTable,
ErrorTable: defaultErrorTable, ErrorTable: defaultErrorTable,
DurationTable: defaultDurationTable, DurationTable: defaultDurationTable,
UsageExplorerTable: defaultUsageExplorerTable,
SpansTable: defaultSpansTable, SpansTable: defaultSpansTable,
DependencyGraphTable: defaultDependencyGraphTable, DependencyGraphTable: defaultDependencyGraphTable,
TopLevelOperationsTable: defaultTopLevelOperationsTable, TopLevelOperationsTable: defaultTopLevelOperationsTable,

40
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -47,16 +47,17 @@ import (
) )
const ( const (
primaryNamespace = "clickhouse" primaryNamespace = "clickhouse"
archiveNamespace = "clickhouse-archive" archiveNamespace = "clickhouse-archive"
signozTraceDBName = "signoz_traces" signozTraceDBName = "signoz_traces"
signozDurationMVTable = "durationSort" signozDurationMVTable = "durationSort"
signozSpansTable = "signoz_spans" signozUsageExplorerTable = "usage_explorer"
signozErrorIndexTable = "signoz_error_index_v2" signozSpansTable = "signoz_spans"
signozTraceTableName = "signoz_index_v2" signozErrorIndexTable = "signoz_error_index_v2"
signozMetricDBName = "signoz_metrics" signozTraceTableName = "signoz_index_v2"
signozSampleTableName = "samples_v2" signozMetricDBName = "signoz_metrics"
signozTSTableName = "time_series_v2" signozSampleTableName = "samples_v2"
signozTSTableName = "time_series_v2"
minTimespanForProgressiveSearch = time.Hour minTimespanForProgressiveSearch = time.Hour
minTimespanForProgressiveSearchMargin = time.Minute minTimespanForProgressiveSearchMargin = time.Minute
@ -80,6 +81,7 @@ type ClickHouseReader struct {
traceDB string traceDB string
operationsTable string operationsTable string
durationTable string durationTable string
usageExplorerTable string
indexTable string indexTable string
errorTable string errorTable string
spansTable string spansTable string
@ -120,6 +122,7 @@ func NewReader(localDB *sqlx.DB, configFile string) *ClickHouseReader {
operationsTable: options.primary.OperationsTable, operationsTable: options.primary.OperationsTable,
indexTable: options.primary.IndexTable, indexTable: options.primary.IndexTable,
errorTable: options.primary.ErrorTable, errorTable: options.primary.ErrorTable,
usageExplorerTable: options.primary.UsageExplorerTable,
durationTable: options.primary.DurationTable, durationTable: options.primary.DurationTable,
spansTable: options.primary.SpansTable, spansTable: options.primary.SpansTable,
dependencyGraphTable: options.primary.DependencyGraphTable, dependencyGraphTable: options.primary.DependencyGraphTable,
@ -1632,15 +1635,20 @@ func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *mo
func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetUsageParams) (*[]model.UsageItem, error) { func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetUsageParams) (*[]model.UsageItem, error) {
var usageItems []model.UsageItem var usageItems []model.UsageItem
namedArgs := []interface{}{
clickhouse.Named("interval", queryParams.StepHour),
clickhouse.Named("start", strconv.FormatInt(queryParams.Start.UnixNano(), 10)),
clickhouse.Named("end", strconv.FormatInt(queryParams.End.UnixNano(), 10)),
}
var query string var query string
if len(queryParams.ServiceName) != 0 { if len(queryParams.ServiceName) != 0 {
query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d HOUR) as time, count(1) as count FROM %s.%s WHERE serviceName='%s' AND timestamp>='%s' AND timestamp<='%s' GROUP BY time ORDER BY time ASC", queryParams.StepHour, r.traceDB, r.indexTable, queryParams.ServiceName, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10)) namedArgs = append(namedArgs, clickhouse.Named("serviceName", queryParams.ServiceName))
query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL @interval HOUR) as time, sum(count) as count FROM %s.%s WHERE service_name=@serviceName AND timestamp>=@start AND timestamp<=@end GROUP BY time ORDER BY time ASC", r.traceDB, r.usageExplorerTable)
} else { } else {
query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d HOUR) as time, count(1) as count FROM %s.%s WHERE timestamp>='%s' AND timestamp<='%s' GROUP BY time ORDER BY time ASC", queryParams.StepHour, r.traceDB, r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10)) query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL @interval HOUR) as time, sum(count) as count FROM %s.%s WHERE timestamp>=@start AND timestamp<=@end GROUP BY time ORDER BY time ASC", r.traceDB, r.usageExplorerTable)
} }
err := r.db.Select(ctx, &usageItems, query) err := r.db.Select(ctx, &usageItems, query, namedArgs...)
zap.S().Info(query) zap.S().Info(query)
@ -1983,7 +1991,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context,
switch params.Type { switch params.Type {
case constants.TraceTTL: case constants.TraceTTL:
tableNameArray := []string{signozTraceDBName + "." + signozTraceTableName, signozTraceDBName + "." + signozDurationMVTable, signozTraceDBName + "." + signozSpansTable, signozTraceDBName + "." + signozErrorIndexTable, signozTraceDBName + "." + defaultDependencyGraphTable} tableNameArray := []string{signozTraceDBName + "." + signozTraceTableName, signozTraceDBName + "." + signozDurationMVTable, signozTraceDBName + "." + signozSpansTable, signozTraceDBName + "." + signozErrorIndexTable, signozTraceDBName + "." + signozUsageExplorerTable, signozTraceDBName + "." + defaultDependencyGraphTable}
for _, tableName = range tableNameArray { for _, tableName = range tableNameArray {
statusItem, err := r.checkTTLStatusItem(ctx, tableName) statusItem, err := r.checkTTLStatusItem(ctx, tableName)
if err != nil { if err != nil {
@ -2258,7 +2266,7 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa
switch ttlParams.Type { switch ttlParams.Type {
case constants.TraceTTL: case constants.TraceTTL:
tableNameArray := []string{signozTraceDBName + "." + signozTraceTableName, signozTraceDBName + "." + signozDurationMVTable, signozTraceDBName + "." + signozSpansTable, signozTraceDBName + "." + signozErrorIndexTable} tableNameArray := []string{signozTraceDBName + "." + signozTraceTableName, signozTraceDBName + "." + signozDurationMVTable, signozTraceDBName + "." + signozSpansTable, signozTraceDBName + "." + signozErrorIndexTable, signozTraceDBName + "." + signozUsageExplorerTable}
status, err := r.setTTLQueryStatus(ctx, tableNameArray) status, err := r.setTTLQueryStatus(ctx, tableNameArray)
if err != nil { if err != nil {
return nil, err return nil, err