GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-07-31 15:31:58 +08:00
Code Issues Packages Projects Releases Wiki Activity
signoz/pkg/query-service/app/integrations/messagingQueues/kafka/consumerLag.md
Shivanshu Raj Shrivastava e51f4d986d
feat: kafka Scenario 1, 3, 4 all squashed (#6144) 
* feat: kafka partition level observerability features
2024-10-17 19:44:42 +05:30

981 lines
24 KiB
Markdown
Raw Permalink Blame History

## Consumer Lag feature break down
### 1) Consumer Group Details
API endpoint:
```
POST /api/v1/messaging-queues/kafka/consumer-lag/consumer-details
```
Request-Body
```json
{
"start": 1724429217000000000,
"end": 1724431017000000000,
"variables": {
"partition": "0",
"topic": "topic1",
"consumer_group": "cg1"
}
}
```
Response in query range `table` format
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "throughput",
"queryName": "",
"isValueColumn": false
},
{
"name": "avg_msg_size",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"avg_msg_size": "15",
"error_rate": "0",
"p99": "0.47993265000000035",
"service_name": "consumer-svc",
"throughput": "39.86888888888889"
}
}
]
}
}
]
}
}
```
----
### 2) Producer Details
API endpoint:
```
POST /api/v1/messaging-queues/kafka/consumer-lag/producer-details
```
Request-Body
```json
{
"start": 1724429217000000000,
"end": 1724431017000000000,
"variables": {
"partition": "0",
"topic": "topic1"
}
}
```
Response in query range `table` format
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_percentage",
"queryName": "",
"isValueColumn": false
},
{
"name": "throughput",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"error_percentage": "0",
"p99": "5.51359028",
"throughput": "39.86888888888889",
"service_name": "producer-svc"
}
}
]
}
}
]
}
}
```
### 3) Network Fetch Latency:
API endpoint:
```
POST /api/v1/messaging-queues/kafka/consumer-lag/network-latency
```
Request-Body
```json
{
"start": 1724673937000000000,
"end": 1724675737000000000,
"variables": {
"consumer_group": "cg1",
"partition": "0"
}
}
```
Response in query range `table` format
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "client_id",
"queryName": "",
"isValueColumn": false
},
{
"name": "service_instance_id",
"queryName": "",
"isValueColumn": false
},
{
"name": "latency",
"queryName": "latency",
"isValueColumn": true
},
{
"name": "throughput",
"queryName": "throughput",
"isValueColumn": true
}
],
"rows": [
{
"data": {
"client_id": "consumer-cg1-1",
"latency": 48.99,
"service_instance_id": "b0a851d7-1735-4e3f-8f5f-7c63a8a55a24",
"service_name": "consumer-svc",
"throughput": 14.97
}
},
{
"data": {
"client_id": "consumer-cg1-1",
"latency": 25.21,
"service_instance_id": "ccf49550-2e8f-4c7b-be29-b9e0891ef93d",
"service_name": "consumer-svc",
"throughput": 24.91
}
}
]
}
}
]
}
}
```
### Onboarding APIs
```
/api/v1/messaging-queues/kafka/onboarding/producers
```
```json
{
"start": 1727620544260611424,
"end": 1727620556401481428
}
```
```json
// everything is present
{
"status": "success",
"data": [
{
"attribute": "kind",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.name",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.partition.id",
"error_message": "messaging.destination.partition.id attribute is not present in your spans",
"status": "0"
},
{
"attribute": "messaging.system",
"error_message": "",
"status": "1"
},
{
"attribute": "telemetry ingestion",
"error_message": "",
"status": "1"
}
]
}
// partial attributes are present
{
"status": "success",
"data": [
{
"attribute": "kind",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.name",
"error_message": "messaging.destination.name attribute is not present in your spans",
"status": "0"
},
{
"attribute": "messaging.destination.partition.id",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.system",
"error_message": "",
"status": "1"
},
{
"attribute": "telemetry ingestion",
"error_message": "",
"status": "1"
}
]
}
// no data available
{
"status": "success",
"data": [
{
"attribute": "telemetry ingestion",
"error_message": "No data available in the given time range",
"status": "0"
}
]
}
```
```
/api/v1/messaging-queues/kafka/onboarding/consumers
```
```json
// everything is present
{
"status": "success",
"data": [
{
"attribute": "kind",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.client_id",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.name",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.partition.id",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.kafka.consumer.group",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.message.body.size",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.system",
"error_message": "",
"status": "1"
},
{
"attribute": "service.instance.id",
"error_message": "",
"status": "1"
},
{
"attribute": "service_name",
"error_message": "",
"status": "1"
},
{
"attribute": "telemetry ingestion",
"error_message": "",
"status": "1"
}
]
}
// partial attributes are present
{
"status": "success",
"data": [
{
"attribute": "kind",
"error_message": "check if your consumer spans has kind=5 as attribute",
"status": "0"
},
{
"attribute": "messaging.client_id",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.name",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.destination.partition.id",
"error_message": "",
"status": "1"
},
{
"attribute": "messaging.kafka.consumer.group",
"error_message": "messaging.kafka.consumer.group attribute is not present in your spans",
"status": "0"
},
{
"attribute": "messaging.message.body.size",
"error_message": "messaging.message.body.size attribute is not present in your spans",
"status": "0"
},
{
"attribute": "messaging.system",
"error_message": "",
"status": "1"
},
{
"attribute": "service.instance.id",
"error_message": "",
"status": "1"
},
{
"attribute": "service_name",
"error_message": "",
"status": "1"
},
{
"attribute": "telemetry ingestion",
"error_message": "",
"status": "1"
}
]
}
// no data available
{
"status": "success",
"data": [
{
"attribute": "telemetry ingestion",
"error_message": "No data available in the given time range",
"status": "0"
}
]
}
```
```
/api/v1/messaging-queues/kafka/onboarding/kafka
```
```json
{
"start": 1728485200000000000,
"end": 1728749800000000000
}
```
```json
// everything is present
{
"status": "success",
"data": [
{
"attribute": "telemetry ingestion",
"error_message": "",
"status": "1"
},
{
"attribute": "kafka_consumer_fetch_latency_avg",
"error_message": "Metric kafka_consumer_fetch_latency_avg is not present in the given time range.",
"status": "0"
},
{
"attribute": "kafka_consumer_group_lag",
"error_message": "Metric kafka_consumer_group_lag is not present in the given time range.",
"status": "0"
}
]
}
// partial attributes are present
{
"status": "success",
"data": [
{
"attribute": "telemetry ingestion",
"error_message": "",
"status": "1"
},
{
"attribute": "kafka_consumer_fetch_latency_avg",
"error_message": "",
"status": "1"
},
{
"attribute": "kafka_consumer_group_lag",
"error_message": "",
"status": "1"
}
]
}
// no data available
{
"status": "success",
"data": [
{
"attribute": "telemetry ingestion",
"error_message": "No data available in the given time range",
"status": "0"
}
]
}
```
### Partition Latency
```json
/api/v1/messaging-queues/kafka/partition-latency/overview
```
```json
{
"start": 1728287046000000000,
"end": 1728587046000000000
}
```
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "topic",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "partition_latency",
"queryName": "partition_latency",
"isValueColumn": true
}
],
"rows": [
{
"data": {
"p99": "2",
"partition_latency": 1.18,
"topic": "topic1"
}
},
{
"data": {
"p99": "2",
"partition_latency": 0.15,
"topic": "topic2"
}
},
{
"data": {
"p99": "2",
"partition_latency": 0.26,
"topic": "topic3"
}
}
]
}
}
]
}
}
```
---------
```json
/api/v1/messaging-queues/kafka/partition-latency/consumer
```
```json
{
"start": 1728287046000000000,
"end": 1728587046000000000,
"variables": {
"partition": "2",
"topic": "topic1"
}
}
```
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "consumer_group",
"queryName": "",
"isValueColumn": false
},
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "throughput",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"consumer_group": "cg1",
"error_rate": "0",
"p99": "0.11994228000000004",
"service_name": "consumer-svc",
"throughput": "1.18116"
}
}
]
}
}
]
}
}
```
---------
### Topic throughput
```json
/api/v1/messaging-queues/kafka/topic-throughput/producer
```
```json
{
"start": 1728287046000000000,
"end": 1728587046000000000
}
```
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "topic",
"queryName": "",
"isValueColumn": false
},
{
"name": "serviceName",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "throughput",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"error_rate": "0",
"p99": "8.662880220000002",
"serviceName": "producer-svc1",
"throughput": "0.41642666666666667",
"topic": "topic1"
}
},
{
"data": {
"error_rate": "0",
"p99": "9.786847500000016",
"serviceName": "producer-svc2",
"throughput": "0.76473",
"topic": "topic1"
}
},
{
"data": {
"error_rate": "0",
"p99": "14.432925500000021",
"serviceName": "producer-svc3",
"throughput": "0.08976",
"topic": "topic2"
}
},
{
"data": {
"error_rate": "0",
"p99": "14.32833297000002",
"serviceName": "producer-svc2",
"throughput": "0.06449333333333333",
"topic": "topic2"
}
},
{
"data": {
"error_rate": "0",
"p99": "13.416533810000036",
"serviceName": "producer-svc4",
"throughput": "0.14766",
"topic": "topic3"
}
},
{
"data": {
"error_rate": "0",
"p99": "13.366232000000034",
"serviceName": "producer-svc3",
"throughput": "0.11166666666666666",
"topic": "topic3"
}
}
]
}
}
]
}
}
```
---------
### Topic throughput
```json
/api/v1/messaging-queues/kafka/topic-throughput/producer-details
```
```json
{
"start": 1728287046000000000,
"end": 1728587046000000000,
"variables": {
"partition": "2",
"topic": "topic1",
"service_name": "producer-svc2"
}
}
```
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "partition",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "throughput",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"error_rate": "0",
"p99": "9.165558780000026",
"partition": "2",
"throughput": "0.76473"
}
}
]
}
}
]
}
}
```
---------
### Topic throughput
```json
/api/v1/messaging-queues/kafka/topic-throughput/consumer
```
```json
{
"start": 1728287046000000000,
"end": 1728587046000000000
}
```
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "topic",
"queryName": "",
"isValueColumn": false
},
{
"name": "service_name",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "ingestion_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "byte_rate",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"byte_rate": "17.7174",
"error_rate": "0",
"ingestion_rate": "1.18116",
"p99": "0.12260112000000009",
"service_name": "consumer-svc",
"topic": "topic1"
}
},
{
"data": {
"byte_rate": "2.1594533333333334",
"error_rate": "0",
"ingestion_rate": "0.15424666666666667",
"p99": "7.4079657800000005",
"service_name": "consumer-svc2",
"topic": "topic2"
}
},
{
"data": {
"byte_rate": "3.66446",
"error_rate": "0",
"ingestion_rate": "0.25933",
"p99": "6.135769970000011",
"service_name": "consumer-svc3",
"topic": "topic3"
}
}
]
}
}
]
}
}
```
---------
### Topic throughput
```json
/api/v1/messaging-queues/kafka/topic-throughput/consumer-details
```
```json
{
"start": 1728287046000000000,
"end": 1728587046000000000,
"variables": {
"topic": "topic1",
"service_name": "consumer-svc"
}
}
```
```json
{
"status": "success",
"data": {
"resultType": "",
"result": [
{
"table": {
"columns": [
{
"name": "partition",
"queryName": "",
"isValueColumn": false
},
{
"name": "p99",
"queryName": "",
"isValueColumn": false
},
{
"name": "error_rate",
"queryName": "",
"isValueColumn": false
},
{
"name": "throughput",
"queryName": "",
"isValueColumn": false
}
],
"rows": [
{
"data": {
"error_rate": "0",
"p99": "0.11789381000000003",
"partition": "2",
"throughput": "1.18116"
}
}
]
}
}
]
}
}
```
Reference in New Issue View Git Blame Copy Permalink