GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-04 18:00:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6625 from SigNoz/release/v0.62.x

Browse Source 
Release/v0.62.x
This commit is contained in:
Prashant Shahi 2024-12-12 21:02:17 +05:30 committed by GitHub
commit 2b5a0ec496
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 2985 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
View File

@ -146,7 +146,7 @@ services:
condition: on-failure
query-service:
image: signoz/query-service:0.61.0
image: signoz/query-service:0.62.0
command:
[
"-config=/root/config/prometheus.yml",
@ -186,7 +186,7 @@ services:
<<: *db-depend
frontend:
image: signoz/frontend:0.61.0
image: signoz/frontend:0.62.0
deploy:
restart_policy:
condition: on-failure
@ -199,7 +199,7 @@ services:
- ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
otel-collector:
image: signoz/signoz-otel-collector:0.111.14
image: signoz/signoz-otel-collector:0.111.15
command:
[
"--config=/etc/otel-collector-config.yaml",
@ -237,7 +237,7 @@ services:
- query-service
otel-collector-migrator:
image: signoz/signoz-schema-migrator:0.111.14
image: signoz/signoz-schema-migrator:0.111.15
deploy:
restart_policy:
condition: on-failure

6
deploy/docker/clickhouse-setup/docker-compose-core.yaml
View File

@ -69,10 +69,12 @@ services:
- --storage.path=/data
otel-collector-migrator:
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
container_name: otel-migrator
command:
- "sync"
- "--dsn=tcp://clickhouse:9000"
- "--up="
depends_on:
clickhouse:
condition: service_healthy
@ -84,7 +86,7 @@ services:
# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
otel-collector:
container_name: signoz-otel-collector
image: signoz/signoz-otel-collector:0.111.14
image: signoz/signoz-otel-collector:0.111.15
command:
[
"--config=/etc/otel-collector-config.yaml",

10
deploy/docker/clickhouse-setup/docker-compose-minimal.yaml
View File

@ -162,7 +162,7 @@ services:
# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
query-service:
image: signoz/query-service:${DOCKER_TAG:-0.61.0}
image: signoz/query-service:${DOCKER_TAG:-0.62.0}
container_name: signoz-query-service
command:
[
@ -201,7 +201,7 @@ services:
<<: *db-depend
frontend:
image: signoz/frontend:${DOCKER_TAG:-0.61.0}
image: signoz/frontend:${DOCKER_TAG:-0.62.0}
container_name: signoz-frontend
restart: on-failure
depends_on:
@ -213,7 +213,7 @@ services:
- ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
otel-collector-migrator-sync:
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
container_name: otel-migrator-sync
command:
- "sync"
@ -228,7 +228,7 @@ services:
# condition: service_healthy
otel-collector-migrator-async:
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
container_name: otel-migrator-async
command:
- "async"
@ -245,7 +245,7 @@ services:
# condition: service_healthy
otel-collector:
image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.15}
container_name: signoz-otel-collector
command:
[

8
deploy/docker/clickhouse-setup/docker-compose.testing.yaml
View File

@ -167,7 +167,7 @@ services:
# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
query-service:
image: signoz/query-service:${DOCKER_TAG:-0.61.0}
image: signoz/query-service:${DOCKER_TAG:-0.62.0}
container_name: signoz-query-service
command:
[
@ -208,7 +208,7 @@ services:
<<: *db-depend
frontend:
image: signoz/frontend:${DOCKER_TAG:-0.61.0}
image: signoz/frontend:${DOCKER_TAG:-0.62.0}
container_name: signoz-frontend
restart: on-failure
depends_on:
@ -220,7 +220,7 @@ services:
- ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
otel-collector-migrator:
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.15}
container_name: otel-migrator
command:
- "--dsn=tcp://clickhouse:9000"
@ -234,7 +234,7 @@ services:
otel-collector:
image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.15}
container_name: signoz-otel-collector
command:
[

41
ee/query-service/app/api/traces.go
View File

@ -2,31 +2,32 @@ package api
import (
"net/http"
"go.signoz.io/signoz/ee/query-service/app/db"
"go.signoz.io/signoz/ee/query-service/model"
baseapp "go.signoz.io/signoz/pkg/query-service/app"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)
func (ah *APIHandler) searchTraces(w http.ResponseWriter, r *http.Request) {
ah.APIHandler.SearchTraces(w, r)
return
if !ah.CheckFeature(basemodel.SmartTraceDetail) {
zap.L().Info("SmartTraceDetail feature is not enabled in this plan")
ah.APIHandler.SearchTraces(w, r)
return
}
searchTracesParams, err := baseapp.ParseSearchTracesParams(r)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params")
return
}
// This is commented since this will be taken care by new trace API
result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm)
if ah.HandleError(w, err, http.StatusBadRequest) {
return
}
// if !ah.CheckFeature(basemodel.SmartTraceDetail) {
// zap.L().Info("SmartTraceDetail feature is not enabled in this plan")
// ah.APIHandler.SearchTraces(w, r)
// return
// }
// searchTracesParams, err := baseapp.ParseSearchTracesParams(r)
// if err != nil {
// RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params")
// return
// }
// result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm)
// if ah.HandleError(w, err, http.StatusBadRequest) {
// return
// }
// ah.WriteJSON(w, r, result)
ah.WriteJSON(w, r, result)
}

4
frontend/src/container/CreateAlertRule/defaults.ts
View File

@ -117,7 +117,7 @@ export const logAlertDefaults: AlertDef = {
chQueries: {
A: {
name: 'A',
query: `select \ntoStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 MINUTE) AS interval, \ntoFloat64(count()) as value \nFROM signoz_logs.distributed_logs_v2 \nWHERE timestamp BETWEEN {{.start_timestamp_nano}} AND {{.end_timestamp_nano}} \nGROUP BY interval;\n\n-- available variables:\n-- \t{{.start_timestamp_nano}}\n-- \t{{.end_timestamp_nano}}\n\n-- required columns (or alias):\n-- \tvalue\n-- \tinterval`,
query: `select \ntoStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 MINUTE) AS interval, \ntoFloat64(count()) as value \nFROM signoz_logs.distributed_logs_v2 \nWHERE timestamp BETWEEN {{.start_timestamp_nano}} AND {{.end_timestamp_nano}} \nAND ts_bucket_start BETWEEN {{.start_timestamp}} - 1800 AND {{.end_timestamp}} \nGROUP BY interval;\n\n-- Please check docs here https://signoz.io/docs/userguide/logs_clickhouse_queries/\n\n-- available variables:\n-- \t{{.start_timestamp_nano}}\n-- \t{{.end_timestamp_nano}}\n-- \t{{.start_timestamp}}\n-- \t{{.end_timestamp}}\n\n-- required columns (or alias):\n-- \tvalue\n-- \tinterval`,
legend: '',
disabled: false,
},
@ -149,7 +149,7 @@ export const traceAlertDefaults: AlertDef = {
chQueries: {
A: {
name: 'A',
query: `SELECT \n\ttoStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, \n\tstringTagMap['peer.service'] AS op_name, \n\ttoFloat64(avg(durationNano)) AS value \nFROM signoz_traces.distributed_signoz_index_v2 \nWHERE stringTagMap['peer.service']!='' \nAND timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} \nGROUP BY (op_name, interval);\n\n-- available variables:\n-- \t{{.start_datetime}}\n-- \t{{.end_datetime}}\n\n-- required column alias:\n-- \tvalue\n-- \tinterval`,
query: `SELECT \n\ttoStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, \n\tresource_string_service$$name AS \`service.name\`, \n\ttoFloat64(avg(duration_nano)) AS value \nFROM signoz_traces.distributed_signoz_index_v3 \nWHERE resource_string_service$$name !='' \nAND timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} \nAND ts_bucket_start BETWEEN {{.start_timestamp}} - 1800 AND {{.end_timestamp}} \nGROUP BY (\`service.name\`, interval);\n\n-- Please check docs here https://signoz.io/docs/userguide/writing-clickhouse-traces-query/\n\n-- available variables:\n-- \t{{.start_datetime}}\n-- \t{{.end_datetime}}\n-- \t{{.start_timestamp}}\n-- \t{{.end_timestamp}}\n\n-- required column alias:\n-- \tvalue\n-- \tinterval`,
legend: '',
disabled: false,
},

72
frontend/src/container/MetricsApplication/Tabs/Overview.tsx
View File

@ -145,41 +145,49 @@ function Application(): JSX.Element {
[servicename, topLevelOperations],
);
const operationPerSecWidget = getWidgetQueryBuilder({
query: {
queryType: EQueryType.QUERY_BUILDER,
promql: [],
builder: operationPerSec({
servicename,
tagFilterItems,
topLevelOperations: topLevelOperationsRoute,
const operationPerSecWidget = useMemo(
() =>
getWidgetQueryBuilder({
query: {
queryType: EQueryType.QUERY_BUILDER,
promql: [],
builder: operationPerSec({
servicename,
tagFilterItems,
topLevelOperations: topLevelOperationsRoute,
}),
clickhouse_sql: [],
id: uuid(),
},
title: GraphTitle.RATE_PER_OPS,
panelTypes: PANEL_TYPES.TIME_SERIES,
yAxisUnit: 'ops',
id: SERVICE_CHART_ID.rps,
}),
clickhouse_sql: [],
id: uuid(),
},
title: GraphTitle.RATE_PER_OPS,
panelTypes: PANEL_TYPES.TIME_SERIES,
yAxisUnit: 'ops',
id: SERVICE_CHART_ID.rps,
});
[servicename, tagFilterItems, topLevelOperationsRoute],
);
const errorPercentageWidget = getWidgetQueryBuilder({
query: {
queryType: EQueryType.QUERY_BUILDER,
promql: [],
builder: errorPercentage({
servicename,
tagFilterItems,
topLevelOperations: topLevelOperationsRoute,
const errorPercentageWidget = useMemo(
() =>
getWidgetQueryBuilder({
query: {
queryType: EQueryType.QUERY_BUILDER,
promql: [],
builder: errorPercentage({
servicename,
tagFilterItems,
topLevelOperations: topLevelOperationsRoute,
}),
clickhouse_sql: [],
id: uuid(),
},
title: GraphTitle.ERROR_PERCENTAGE,
panelTypes: PANEL_TYPES.TIME_SERIES,
yAxisUnit: '%',
id: SERVICE_CHART_ID.errorPercentage,
}),
clickhouse_sql: [],
id: uuid(),
},
title: GraphTitle.ERROR_PERCENTAGE,
panelTypes: PANEL_TYPES.TIME_SERIES,
yAxisUnit: '%',
id: SERVICE_CHART_ID.errorPercentage,
});
[servicename, tagFilterItems, topLevelOperationsRoute],
);
const stepInterval = useMemo(
() =>

38
frontend/src/container/MetricsApplication/Tabs/Overview/ServiceOverview.tsx
View File

@ -53,24 +53,28 @@ function ServiceOverview({
[isSpanMetricEnable, queries],
);
const latencyWidget = getWidgetQueryBuilder({
query: {
queryType: EQueryType.QUERY_BUILDER,
promql: [],
builder: latency({
servicename,
tagFilterItems,
isSpanMetricEnable,
topLevelOperationsRoute,
const latencyWidget = useMemo(
() =>
getWidgetQueryBuilder({
query: {
queryType: EQueryType.QUERY_BUILDER,
promql: [],
builder: latency({
servicename,
tagFilterItems,
isSpanMetricEnable,
topLevelOperationsRoute,
}),
clickhouse_sql: [],
id: uuid(),
},
title: GraphTitle.LATENCY,
panelTypes: PANEL_TYPES.TIME_SERIES,
yAxisUnit: 'ns',
id: SERVICE_CHART_ID.latency,
}),
clickhouse_sql: [],
id: uuid(),
},
title: GraphTitle.LATENCY,
panelTypes: PANEL_TYPES.TIME_SERIES,
yAxisUnit: 'ns',
id: SERVICE_CHART_ID.latency,
});
[isSpanMetricEnable, servicename, tagFilterItems, topLevelOperationsRoute],
);
const isQueryEnabled =
!topLevelOperationsIsLoading && topLevelOperationsRoute.length > 0;

6
frontend/src/container/SideNav/SideNav.tsx
View File

@ -16,7 +16,7 @@ import history from 'lib/history';
import {
AlertTriangle,
CheckSquare,
RocketIcon,
PackagePlus,
UserCircle,
} from 'lucide-react';
import { MouseEvent, useCallback, useEffect, useMemo, useState } from 'react';
@ -395,9 +395,9 @@ function SideNav({
onClickGetStarted(event);
}}
>
<RocketIcon size={16} />
<PackagePlus size={16} />
<div className="license tag nav-item-label"> Get Started </div>
<div className="license tag nav-item-label"> New source </div>
</Button>
</div>
)}

2
go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/ClickHouse/clickhouse-go/v2 v2.25.0
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
github.com/SigNoz/signoz-otel-collector v0.111.14
github.com/SigNoz/signoz-otel-collector v0.111.15
github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
github.com/antonmedv/expr v1.15.3

4
go.sum
View File

@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE=
github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA=
github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0=
github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
github.com/SigNoz/signoz-otel-collector v0.111.15 h1:X2aV/FpMZ9kdN10byKrc2ODF39fAcOo6D68rqUFhxSk=
github.com/SigNoz/signoz-otel-collector v0.111.15/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=

32
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -1203,7 +1203,9 @@ func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetU
return &usageItems, nil
}
func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams) (*[]model.SearchSpansResult, error) {
func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams,
smartTraceAlgorithm func(payload []model.SearchSpanResponseItem, targetSpanId string,
levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) {
searchSpansResult := []model.SearchSpansResult{
{
Columns: []string{"__time", "SpanId", "TraceId", "ServiceName", "Name", "Kind", "DurationNano", "TagsKeys", "TagsValues", "References", "Events", "HasError", "StatusMessage", "StatusCodeString", "SpanKind"},
@ -1318,9 +1320,29 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea
end = time.Now()
zap.L().Debug("getTraceSQLQuery unmarshal took: ", zap.Duration("duration", end.Sub(start)))
for i, item := range searchSpanResponses {
spanEvents := item.GetValues()
searchSpansResult[0].Events[i] = spanEvents
err = r.featureFlags.CheckFeature(model.SmartTraceDetail)
smartAlgoEnabled := err == nil
if len(searchScanResponses) > params.SpansRenderLimit && smartAlgoEnabled {
start = time.Now()
searchSpansResult, err = smartTraceAlgorithm(searchSpanResponses, params.SpanID, params.LevelUp, params.LevelDown, params.SpansRenderLimit)
if err != nil {
return nil, err
}
end = time.Now()
zap.L().Debug("smartTraceAlgo took: ", zap.Duration("duration", end.Sub(start)))
userEmail, err := auth.GetEmailFromJwt(ctx)
if err == nil {
data := map[string]interface{}{
"traceSize": len(searchScanResponses),
"spansRenderLimit": params.SpansRenderLimit,
}
telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LARGE_TRACE_OPENED, data, userEmail, true, false)
}
} else {
for i, item := range searchSpanResponses {
spanEvents := item.GetValues()
searchSpansResult[0].Events[i] = spanEvents
}
}
searchSpansResult[0].StartTimestampMillis = startTime - (durationNano / 1000000)
@ -1334,7 +1356,7 @@ func (r *ClickHouseReader) SearchTraces(ctx context.Context, params *model.Searc
levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) {
if r.useTraceNewSchema {
return r.SearchTracesV2(ctx, params)
return r.SearchTracesV2(ctx, params, smartTraceAlgorithm)
}
var countSpans uint64

4
pkg/query-service/app/dashboards/model.go
View File

@ -289,6 +289,10 @@ func GetDashboard(ctx context.Context, uuid string) (*Dashboard, *model.ApiError
return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
}
if dashboard.Data["title"] == "Ingestion" && dashboard.Data["description"] != nil {
dashboard.Data["description"] = "This dashboard is deprecated. Please use the new Ingestion V2 dashboard. " + dashboard.Data["description"].(string)
}
return &dashboard, nil
}

46
pkg/query-service/app/http_handler.go
View File

@ -3141,14 +3141,14 @@ func (aH *APIHandler) getProducerThroughputOverview(
Hash: make(map[string]struct{}),
}
queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache)
producerQueryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache)
if err != nil {
zap.L().Error(err.Error())
RespondError(w, apiErr, nil)
return
}
if err := validateQueryRangeParamsV3(queryRangeParams); err != nil {
if err := validateQueryRangeParamsV3(producerQueryRangeParams); err != nil {
zap.L().Error(err.Error())
RespondError(w, apiErr, nil)
return
@ -3157,7 +3157,7 @@ func (aH *APIHandler) getProducerThroughputOverview(
var result []*v3.Result
var errQuriesByName map[string]error
result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), queryRangeParams)
result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), producerQueryRangeParams)
if err != nil {
apiErrObj := &model.ApiError{Typ: model.ErrorBadData, Err: err}
RespondError(w, apiErrObj, errQuriesByName)
@ -3165,21 +3165,21 @@ func (aH *APIHandler) getProducerThroughputOverview(
}
for _, res := range result {
for _, list := range res.List {
serviceName, serviceNameOk := list.Data["service_name"].(*string)
topicName, topicNameOk := list.Data["topic"].(*string)
params := []string{*serviceName, *topicName}
for _, series := range res.Series {
serviceName, serviceNameOk := series.Labels["service_name"]
topicName, topicNameOk := series.Labels["topic"]
params := []string{serviceName, topicName}
hashKey := uniqueIdentifier(params, "#")
_, ok := attributeCache.Hash[hashKey]
if topicNameOk && serviceNameOk && !ok {
attributeCache.Hash[hashKey] = struct{}{}
attributeCache.TopicName = append(attributeCache.TopicName, *topicName)
attributeCache.ServiceName = append(attributeCache.ServiceName, *serviceName)
attributeCache.TopicName = append(attributeCache.TopicName, topicName)
attributeCache.ServiceName = append(attributeCache.ServiceName, serviceName)
}
}
}
queryRangeParams, err = mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-latency", attributeCache)
queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-byte-rate", attributeCache)
if err != nil {
zap.L().Error(err.Error())
RespondError(w, apiErr, nil)
@ -3198,26 +3198,32 @@ func (aH *APIHandler) getProducerThroughputOverview(
return
}
latencyColumn := &v3.Result{QueryName: "latency"}
var latencySeries []*v3.Row
byteRateColumn := &v3.Result{QueryName: "byte_rate"}
var byteRateSeries []*v3.Series
for _, res := range resultFetchLatency {
for _, list := range res.List {
topic, topicOk := list.Data["topic"].(*string)
serviceName, serviceNameOk := list.Data["service_name"].(*string)
params := []string{*serviceName, *topic}
for _, series := range res.Series {
topic, topicOk := series.Labels["topic"]
serviceName, serviceNameOk := series.Labels["service_name"]
params := []string{serviceName, topic}
hashKey := uniqueIdentifier(params, "#")
_, ok := attributeCache.Hash[hashKey]
if topicOk && serviceNameOk && ok {
latencySeries = append(latencySeries, list)
byteRateSeries = append(byteRateSeries, series)
}
}
}
latencyColumn.List = latencySeries
result = append(result, latencyColumn)
byteRateColumn.Series = byteRateSeries
var latencyColumnResult []*v3.Result
latencyColumnResult = append(latencyColumnResult, byteRateColumn)
resultFetchLatency = postprocess.TransformToTableForBuilderQueries(latencyColumnResult, queryRangeParams)
result = postprocess.TransformToTableForClickHouseQueries(result)
result = append(result, resultFetchLatency[0])
resp := v3.QueryRangeResponse{
Result: resultFetchLatency,
Result: result,
}
aH.Respond(w, resp)
}

16
pkg/query-service/app/integrations/messagingQueues/kafka/translator.go
View File

@ -61,14 +61,17 @@ func buildClickHouseQueryNetwork(messagingQueue *MessagingQueue, queueType strin
func buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd int64, attributeCache *Clients) (map[string]*v3.BuilderQuery, error) {
bq := make(map[string]*v3.BuilderQuery)
queryName := fmt.Sprintf("latency")
queryName := fmt.Sprintf("byte_rate")
chq := &v3.BuilderQuery{
QueryName: queryName,
StepInterval: common.MinAllowedStepInterval(unixMilliStart, unixMilliEnd),
DataSource: v3.DataSourceMetrics,
AggregateAttribute: v3.AttributeKey{
Key: "kafka_producer_byte_rate",
Key: "kafka_producer_byte_rate",
DataType: v3.AttributeKeyDataTypeFloat64,
Type: v3.AttributeKeyType("Gauge"),
IsColumn: true,
},
AggregateOperator: v3.AggregateOperatorAvg,
Temporality: v3.Unspecified,
@ -276,7 +279,7 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string,
cq, err = buildCompositeQuery(&v3.ClickHouseQuery{
Query: query,
}, queryContext)
} else if queryContext == "producer-throughput-overview-latency" {
} else if queryContext == "producer-throughput-overview-byte-rate" {
bhq, err := buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd, attributeCache)
if err != nil {
return nil, err
@ -284,7 +287,8 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string,
cq = &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder,
BuilderQueries: bhq,
PanelType: v3.PanelTypeList,
PanelType: v3.PanelTypeTable,
FillGaps: false,
}
}
@ -315,7 +319,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer
if !ok {
return nil, fmt.Errorf("invalid type for Topic")
}
if queryContext != "consumer-throughput-details" {
if !(queryContext == "consumer-throughput-details" || queryContext == "producer-throughput-details") {
partition, ok = messagingQueue.Variables["partition"]
if !ok {
return nil, fmt.Errorf("invalid type for Partition")
@ -364,7 +368,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer
func buildCompositeQuery(chq *v3.ClickHouseQuery, queryContext string) (*v3.CompositeQuery, error) {
if queryContext == "producer-consumer-eval" || queryContext == "producer-throughput-overview" {
if queryContext == "producer-consumer-eval" {
return &v3.CompositeQuery{
QueryType: v3.QueryTypeClickHouseSQL,
ClickHouseQueries: map[string]*v3.ClickHouseQuery{queryContext: chq},

5
pkg/query-service/app/traces/v4/query_builder.go
View File

@ -355,8 +355,9 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.
filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery)
}
} else {
column := getColumnName(mq.AggregateAttribute)
filterSubQuery = fmt.Sprintf("%s AND has(%s, '%s')", filterSubQuery, column, mq.AggregateAttribute.Key)
cType := getClickHouseTracesColumnType(mq.AggregateAttribute.Type)
cDataType := getClickHouseTracesColumnDataType(mq.AggregateAttribute.DataType)
filterSubQuery = fmt.Sprintf("%s AND mapContains(%s_%s, '%s')", filterSubQuery, cType, cDataType, mq.AggregateAttribute.Key)
}
}
op := "toFloat64(count())"

56
pkg/query-service/app/traces/v4/query_builder_test.go
View File

@ -451,6 +451,62 @@ func Test_buildTracesQuery(t *testing.T) {
"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') " +
"group by `http.method` order by `http.method` ASC",
},
{
name: "Test buildTracesQuery - count with attr",
args: args{
panelType: v3.PanelTypeTable,
start: 1680066360726210000,
end: 1680066458000000000,
step: 1000,
mq: &v3.BuilderQuery{
AggregateOperator: v3.AggregateOperatorCount,
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Filters: &v3.FilterSet{
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Value: 100,
Operator: v3.FilterOperatorEqual,
},
},
},
GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{
{ColumnName: "http.method", Order: "ASC"}},
},
},
want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND mapContains(attributes_string, 'name') " +
"group by `http.method` order by `http.method` ASC",
},
{
name: "Test buildTracesQuery - count with mat attr",
args: args{
panelType: v3.PanelTypeTable,
start: 1680066360726210000,
end: 1680066458000000000,
step: 1000,
mq: &v3.BuilderQuery{
AggregateOperator: v3.AggregateOperatorCount,
AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
Filters: &v3.FilterSet{
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Value: 100,
Operator: v3.FilterOperatorEqual,
},
},
},
GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}},
OrderBy: []v3.OrderBy{
{ColumnName: "http.method", Order: "ASC"}},
},
},
want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND name != '' " +
"group by `http.method` order by `http.method` ASC",
},
{
name: "Test buildTracesQuery",
args: args{

2737
pkg/query-service/migrate/ingestion_dashboard.go Normal file
View File

File diff suppressed because it is too large Load Diff

12
pkg/query-service/migrate/migate.go
View File

@ -7,6 +7,7 @@ import (
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
)
type DataMigration struct {
@ -53,6 +54,17 @@ func Migrate(dsn string) error {
return err
}
if m, err := getMigrationVersion(conn, "0.62_ingestion_dashboard"); err == nil && m == nil {
if err := migrateIngestionDashboard(conn); err != nil {
zap.L().Error("failed to migrate 0.62_ingestion_dashboard", zap.Error(err))
} else {
_, err := conn.Exec("INSERT INTO data_migrations (version, succeeded) VALUES ('0.62_ingestion_dashboard', true)")
if err != nil {
return err
}
}
}
return nil
}

5
pkg/query-service/rules/threshold_rule.go
View File

@ -73,8 +73,9 @@ func NewThresholdRule(
}
t := ThresholdRule{
BaseRule: baseRule,
version: p.Version,
BaseRule: baseRule,
version: p.Version,
useTraceNewSchema: useTraceNewSchema,
}
querierOption := querier.QuerierOptions{