GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-12 06:19:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: query, response update

Browse Source
This commit is contained in:
shivanshu 2024-08-26 20:28:11 +05:30
parent 413caad0d8
commit f508ee7521
No known key found for this signature in database
GPG Key ID: 0F9ACBC3AA12DC71
3 changed files with 114 additions and 303 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pkg/query-service/app/http_handler.go
View File

@ -2534,17 +2534,16 @@ func (aH *APIHandler) getNetworkData(
RespondError(w, apiErrObj, errQuriesByName) RespondError(w, apiErrObj, errQuriesByName)
return return
} }
result = postprocess.TransformToTableForClickHouseQueries(result)
// iterate over the result and extract messaging.client_id
for _, res := range result { for _, res := range result {
table := res.Table for _, series := range res.Series {
for _, row := range table.Rows { clientID, clientIDOk := series.Labels["client_id"]
if row.Data["client_id"] != nil && row.Data["service_instance_id"] != nil && row.Data["service_name"] != nil { serviceInstanceID, serviceInstanceIDOk := series.Labels["service_instance_id"]
clientID := row.Data["client_id"].(string) serviceName, serviceNameOk := series.Labels["service_name"]
serviceInstanceId := row.Data["service_instance_id"].(string) if clientIDOk && serviceInstanceIDOk && serviceNameOk {
ServiceName := row.Data["service_name"].(string) attributeCache = append(attributeCache, mq.Clients{ClientID: clientID,
attributeCache = append(attributeCache, mq.Clients{ClientID: clientID, ServiceInstanceID: serviceInstanceId, ServiceName: ServiceName}) ServiceInstanceID: serviceInstanceID,
ServiceName: serviceName})
} }
} }
} }
@ -2567,12 +2566,21 @@ func (aH *APIHandler) getNetworkData(
RespondError(w, apiErrObj, errQuriesByNameFetchLatency) RespondError(w, apiErrObj, errQuriesByNameFetchLatency)
return return
} }
resultFetchLatency = postprocess.TransformToTableForClickHouseQueries(resultFetchLatency)
result = append(result, resultFetchLatency...) latencyColoumn := &v3.Result{QueryName: "latency"}
var latencySeries []*v3.Series
for _, res := range resultFetchLatency {
for _, series := range res.Series {
latencySeries = append(latencySeries, series)
}
}
latencyColoumn.Series = latencySeries
result = append(result, latencyColoumn)
resultFetchLatency = postprocess.TransformToTableForBuilderQueries(result, queryRangeParams)
resp := v3.QueryRangeResponse{ resp := v3.QueryRangeResponse{
Result: result, Result: resultFetchLatency,
} }
aH.Respond(w, resp) aH.Respond(w, resp)
} }

280
pkg/query-service/app/integrations/messagingQueues/kafka/consumerLag.md
View File

@ -1,11 +1,6 @@
## Consumer Lag feature break down ## Consumer Lag feature break down
### 1) Consumer Lag Graph ### 1) Consumer Group Details
---
### 2) Consumer Group Details
API endpoint: API endpoint:
@ -13,10 +8,11 @@ API endpoint:
POST /api/v1/messaging-queues/kafka/consumer-lag/consumer-details POST /api/v1/messaging-queues/kafka/consumer-lag/consumer-details
``` ```
Request-Body
```json ```json
{ {
"start": 1720685296000000000, "start": 1724429217000000000,
"end": 1721290096000000000, "end": 1724431017000000000,
"variables": { "variables": {
"partition": "0", "partition": "0",
"topic": "topic1", "topic": "topic1",
@ -24,8 +20,7 @@ POST /api/v1/messaging-queues/kafka/consumer-lag/consumer-details
} }
} }
``` ```
Response in query range `table` format
response in query range format `series`
```json ```json
{ {
"status": "success", "status": "success",
@ -64,11 +59,11 @@ response in query range format `series`
"rows": [ "rows": [
{ {
"data": { "data": {
"avg_msg_size": "0", "avg_msg_size": "15",
"error_rate": "0", "error_rate": "0",
"p99": "0.2942205100000016", "p99": "0.47993265000000035",
"service_name": "consumer-svc", "service_name": "consumer-svc",
"throughput": "0.00016534391534391533" "throughput": "39.86888888888889"
} }
} }
] ]
@ -79,9 +74,9 @@ response in query range format `series`
} }
``` ```
----
### 2) Producer Details
### 3) Producer Details
API endpoint: API endpoint:
@ -89,10 +84,11 @@ API endpoint:
POST /api/v1/messaging-queues/kafka/consumer-lag/producer-details POST /api/v1/messaging-queues/kafka/consumer-lag/producer-details
``` ```
Request-Body
```json ```json
{ {
"start": 1720685296000000000, "start": 1724429217000000000,
"end": 1721290096000000000, "end": 1724431017000000000,
"variables": { "variables": {
"partition": "0", "partition": "0",
"topic": "topic1" "topic": "topic1"
@ -100,7 +96,7 @@ POST /api/v1/messaging-queues/kafka/consumer-lag/producer-details
} }
``` ```
response in query range format `series` Response in query range `table` format
```json ```json
{ {
"status": "success", "status": "success",
@ -116,12 +112,12 @@ response in query range format `series`
"isValueColumn": false "isValueColumn": false
}, },
{ {
"name": "p99_query.p99", "name": "p99",
"queryName": "", "queryName": "",
"isValueColumn": false "isValueColumn": false
}, },
{ {
"name": "error_rate", "name": "error_percentage",
"queryName": "", "queryName": "",
"isValueColumn": false "isValueColumn": false
}, },
@ -134,56 +130,9 @@ response in query range format `series`
"rows": [ "rows": [
{ {
"data": { "data": {
"error_rate": "0", "error_percentage": "0",
"p99_query.p99": "150.08830908000002", "p99": "5.51359028",
"rps": "0.00016534391534391533", "rps": "39.86888888888889",
"service_name": "producer-svc"
}
}
]
}
}
]
}
}
```
response in query range format `table`
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99_query.p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "rps",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"error_rate": "0",
"p99_query.p99": "150.08830908000002",
"rps": "0.00016534391534391533",
"service_name": "producer-svc" "service_name": "producer-svc"
} }
} }
@ -195,8 +144,7 @@ response in query range format `table`
} }
``` ```
### 4) Network Fetch Latency: ### 3) Network Fetch Latency:
API endpoint: API endpoint:
@ -204,17 +152,18 @@ API endpoint:
POST /api/v1/messaging-queues/kafka/consumer-lag/network-latency POST /api/v1/messaging-queues/kafka/consumer-lag/network-latency
``` ```
Request-Body
```json ```json
{ {
"start": 1721174400000000000, "start": 1724673937000000000,
"end": 1722470400000000000, "end": 1724675737000000000,
"variables": { "variables": {
"consumer_group": "cg1" "consumer_group": "cg1"
} }
} }
``` ```
response in query range format `series` Response in query range `table` format
```json ```json
{ {
"status": "success", "status": "success",
@ -224,20 +173,25 @@ response in query range format `series`
{ {
"table": { "table": {
"columns": [ "columns": [
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{ {
"name": "client_id", "name": "client_id",
"queryName": "", "queryName": "",
"isValueColumn": false "isValueColumn": false
}, },
{ {
"name": "instance_id", "name": "service_instance_id",
"queryName": "", "queryName": "",
"isValueColumn": false "isValueColumn": false
}, },
{ {
"name": "serviceName", "name": "latency",
"queryName": "", "queryName": "latency",
"isValueColumn": false "isValueColumn": true
}, },
{ {
"name": "throughput", "name": "throughput",
@ -249,171 +203,19 @@ response in query range format `series`
{ {
"data": { "data": {
"client_id": "consumer-cg1-1", "client_id": "consumer-cg1-1",
"instance_id": "e33ffd7c-827a-427a-828e-547e00cb80d8", "latency": 25.21,
"serviceName": "consumer-svc", "service_instance_id": "ccf49550-2e8f-4c7b-be29-b9e0891ef93d",
"throughput": 0.00035 "service_name": "consumer-svc",
"throughput": 24.91
} }
}, },
{ {
"data": { "data": {
"client_id": "consumer-cg1-1", "client_id": "consumer-cg1-1",
"instance_id": "a96ff029-6f14-435a-a3d4-ab4742b4347f", "latency": 49.68,
"serviceName": "consumer-svc", "service_instance_id": "b0a851d7-1735-4e3f-8f5f-7c63a8a55a24",
"throughput": 0.00027 "service_name": "consumer-svc",
} "throughput": 14.97
},
{
"data": {
"client_id": "consumer-cg1-1",
"instance_id": "ac4833a8-fbe1-4592-a0ff-241f46a0851d",
"serviceName": "consumer-svc-2",
"throughput": 0.00019
}
},
{
"data": {
"client_id": "consumer-cg1-1",
"instance_id": "9e87227f-a564-4b55-bf7c-fb00365d9400",
"serviceName": "consumer-svc",
"throughput": 0.00008
}
}
]
}
},
{
"table": {
"columns": [
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "service_instance_id",
"queryName": "",
"isValueColumn": false
},
{
"name": "latency_0",
"queryName": "latency_0",
"isValueColumn": true
},
{
"name": "latency_1",
"queryName": "latency_1",
"isValueColumn": true
},
{
"name": "latency_2",
"queryName": "latency_2",
"isValueColumn": true
},
{
"name": "latency_3",
"queryName": "latency_3",
"isValueColumn": true
}
],
"rows": [
{
"data": {
"latency_0": 3230.1,
"latency_1": "n/a",
"latency_2": "n/a",
"latency_3": "n/a",
"service_instance_id": "a96ff029-6f14-435a-a3d4-ab4742b4347f",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": 503,
"latency_1": "n/a",
"latency_2": "n/a",
"latency_3": "n/a",
"service_instance_id": "e33ffd7c-827a-427a-828e-547e00cb80d8",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": 502.62,
"latency_1": "n/a",
"latency_2": "n/a",
"latency_3": "n/a",
"service_instance_id": "9e87227f-a564-4b55-bf7c-fb00365d9400",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": 3230.1,
"latency_2": "n/a",
"latency_3": "n/a",
"service_instance_id": "a96ff029-6f14-435a-a3d4-ab4742b4347f",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": 503,
"latency_2": "n/a",
"latency_3": "n/a",
"service_instance_id": "e33ffd7c-827a-427a-828e-547e00cb80d8",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": 502.62,
"latency_2": "n/a",
"latency_3": "n/a",
"service_instance_id": "9e87227f-a564-4b55-bf7c-fb00365d9400",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": "n/a",
"latency_2": 502.81,
"latency_3": "n/a",
"service_instance_id": "ac4833a8-fbe1-4592-a0ff-241f46a0851d",
"service_name": "consumer-svc-2"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": "n/a",
"latency_2": "n/a",
"latency_3": 3230.1,
"service_instance_id": "a96ff029-6f14-435a-a3d4-ab4742b4347f",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": "n/a",
"latency_2": "n/a",
"latency_3": 503,
"service_instance_id": "e33ffd7c-827a-427a-828e-547e00cb80d8",
"service_name": "consumer-svc"
}
},
{
"data": {
"latency_0": "n/a",
"latency_1": "n/a",
"latency_2": "n/a",
"latency_3": 502.62,
"service_instance_id": "9e87227f-a564-4b55-bf7c-fb00365d9400",
"service_name": "consumer-svc"
} }
} }
] ]

7
pkg/query-service/app/integrations/messagingQueues/kafka/sql.go
View File

@ -76,14 +76,15 @@ ORDER BY
} }
func generateNetworkLatencyThroughputSQL(start, end int64, consumerGroup, queueType string) string { func generateNetworkLatencyThroughputSQL(start, end int64, consumerGroup, queueType string) string {
timeRange := (end - start) / 1000000000
query := fmt.Sprintf(` query := fmt.Sprintf(`
--- Subquery for RPS calculation, desc sorted by rps --- Subquery for RPS calculation, desc sorted by rps
SELECT SELECT
stringTagMap['messaging.client_id'] AS client_id, stringTagMap['messaging.client_id'] AS client_id,
stringTagMap['service.instance.id'] AS service_instance_id, stringTagMap['service.instance.id'] AS service_instance_id,
serviceName AS service_name, serviceName AS service_name,
count(*) / ((%d - %d) / 1000000000) AS rps -- Convert nanoseconds to seconds count(*) / %d AS rps -- Convert nanoseconds to seconds
FROM signoz_traces.signoz_index_v2 FROM signoz_traces.distributed_signoz_index_v2
WHERE WHERE
timestamp >= '%d' timestamp >= '%d'
AND timestamp <= '%d' AND timestamp <= '%d'
@ -92,6 +93,6 @@ WHERE
AND stringTagMap['messaging.kafka.consumer.group'] = '%s' AND stringTagMap['messaging.kafka.consumer.group'] = '%s'
GROUP BY service_name, client_id, service_instance_id GROUP BY service_name, client_id, service_instance_id
ORDER BY rps DESC ORDER BY rps DESC
`, end, start, start, end, queueType, consumerGroup) `, timeRange, start, end, queueType, consumerGroup)
return query return query
} }