GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-07-08 10:21:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add byte rate to producer API (#6579)

Browse Source 
* fix: fix partition check for topic throughput
This commit is contained in:
Shivanshu Raj Shrivastava 2024-12-09 20:48:41 +05:30 committed by GitHub
parent 31a2926375
commit 51baf7f8d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 36 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
pkg/query-service/app/http_handler.go
View File

@ -3141,14 +3141,14 @@ func (aH *APIHandler) getProducerThroughputOverview(
Hash: make(map[string]struct{}), Hash: make(map[string]struct{}),
} }
queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache) producerQueryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview", attributeCache)
if err != nil { if err != nil {
zap.L().Error(err.Error()) zap.L().Error(err.Error())
RespondError(w, apiErr, nil) RespondError(w, apiErr, nil)
return return
} }
if err := validateQueryRangeParamsV3(queryRangeParams); err != nil { if err := validateQueryRangeParamsV3(producerQueryRangeParams); err != nil {
zap.L().Error(err.Error()) zap.L().Error(err.Error())
RespondError(w, apiErr, nil) RespondError(w, apiErr, nil)
return return
@ -3157,7 +3157,7 @@ func (aH *APIHandler) getProducerThroughputOverview(
var result []*v3.Result var result []*v3.Result
var errQuriesByName map[string]error var errQuriesByName map[string]error
result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), queryRangeParams) result, errQuriesByName, err = aH.querierV2.QueryRange(r.Context(), producerQueryRangeParams)
if err != nil { if err != nil {
apiErrObj := &model.ApiError{Typ: model.ErrorBadData, Err: err} apiErrObj := &model.ApiError{Typ: model.ErrorBadData, Err: err}
RespondError(w, apiErrObj, errQuriesByName) RespondError(w, apiErrObj, errQuriesByName)
@ -3165,21 +3165,21 @@ func (aH *APIHandler) getProducerThroughputOverview(
} }
for _, res := range result { for _, res := range result {
for _, list := range res.List { for _, series := range res.Series {
serviceName, serviceNameOk := list.Data["service_name"].(*string) serviceName, serviceNameOk := series.Labels["service_name"]
topicName, topicNameOk := list.Data["topic"].(*string) topicName, topicNameOk := series.Labels["topic"]
params := []string{*serviceName, *topicName} params := []string{serviceName, topicName}
hashKey := uniqueIdentifier(params, "#") hashKey := uniqueIdentifier(params, "#")
_, ok := attributeCache.Hash[hashKey] _, ok := attributeCache.Hash[hashKey]
if topicNameOk && serviceNameOk && !ok { if topicNameOk && serviceNameOk && !ok {
attributeCache.Hash[hashKey] = struct{}{} attributeCache.Hash[hashKey] = struct{}{}
attributeCache.TopicName = append(attributeCache.TopicName, *topicName) attributeCache.TopicName = append(attributeCache.TopicName, topicName)
attributeCache.ServiceName = append(attributeCache.ServiceName, *serviceName) attributeCache.ServiceName = append(attributeCache.ServiceName, serviceName)
} }
} }
} }
queryRangeParams, err = mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-latency", attributeCache) queryRangeParams, err := mq.BuildQRParamsWithCache(messagingQueue, "producer-throughput-overview-byte-rate", attributeCache)
if err != nil { if err != nil {
zap.L().Error(err.Error()) zap.L().Error(err.Error())
RespondError(w, apiErr, nil) RespondError(w, apiErr, nil)
@ -3198,26 +3198,32 @@ func (aH *APIHandler) getProducerThroughputOverview(
return return
} }
latencyColumn := &v3.Result{QueryName: "latency"} byteRateColumn := &v3.Result{QueryName: "byte_rate"}
var latencySeries []*v3.Row var byteRateSeries []*v3.Series
for _, res := range resultFetchLatency { for _, res := range resultFetchLatency {
for _, list := range res.List { for _, series := range res.Series {
topic, topicOk := list.Data["topic"].(*string) topic, topicOk := series.Labels["topic"]
serviceName, serviceNameOk := list.Data["service_name"].(*string) serviceName, serviceNameOk := series.Labels["service_name"]
params := []string{*serviceName, *topic} params := []string{serviceName, topic}
hashKey := uniqueIdentifier(params, "#") hashKey := uniqueIdentifier(params, "#")
_, ok := attributeCache.Hash[hashKey] _, ok := attributeCache.Hash[hashKey]
if topicOk && serviceNameOk && ok { if topicOk && serviceNameOk && ok {
latencySeries = append(latencySeries, list) byteRateSeries = append(byteRateSeries, series)
} }
} }
} }
latencyColumn.List = latencySeries byteRateColumn.Series = byteRateSeries
result = append(result, latencyColumn) var latencyColumnResult []*v3.Result
latencyColumnResult = append(latencyColumnResult, byteRateColumn)
resultFetchLatency = postprocess.TransformToTableForBuilderQueries(latencyColumnResult, queryRangeParams)
result = postprocess.TransformToTableForClickHouseQueries(result)
result = append(result, resultFetchLatency[0])
resp := v3.QueryRangeResponse{ resp := v3.QueryRangeResponse{
Result: resultFetchLatency, Result: result,
} }
aH.Respond(w, resp) aH.Respond(w, resp)
} }

16
pkg/query-service/app/integrations/messagingQueues/kafka/translator.go
View File

@ -61,14 +61,17 @@ func buildClickHouseQueryNetwork(messagingQueue *MessagingQueue, queueType strin
func buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd int64, attributeCache *Clients) (map[string]*v3.BuilderQuery, error) { func buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd int64, attributeCache *Clients) (map[string]*v3.BuilderQuery, error) {
bq := make(map[string]*v3.BuilderQuery) bq := make(map[string]*v3.BuilderQuery)
queryName := fmt.Sprintf("latency") queryName := fmt.Sprintf("byte_rate")
chq := &v3.BuilderQuery{ chq := &v3.BuilderQuery{
QueryName: queryName, QueryName: queryName,
StepInterval: common.MinAllowedStepInterval(unixMilliStart, unixMilliEnd), StepInterval: common.MinAllowedStepInterval(unixMilliStart, unixMilliEnd),
DataSource: v3.DataSourceMetrics, DataSource: v3.DataSourceMetrics,
AggregateAttribute: v3.AttributeKey{ AggregateAttribute: v3.AttributeKey{
Key: "kafka_producer_byte_rate", Key: "kafka_producer_byte_rate",
DataType: v3.AttributeKeyDataTypeFloat64,
Type: v3.AttributeKeyType("Gauge"),
IsColumn: true,
}, },
AggregateOperator: v3.AggregateOperatorAvg, AggregateOperator: v3.AggregateOperatorAvg,
Temporality: v3.Unspecified, Temporality: v3.Unspecified,
@ -276,7 +279,7 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string,
cq, err = buildCompositeQuery(&v3.ClickHouseQuery{ cq, err = buildCompositeQuery(&v3.ClickHouseQuery{
Query: query, Query: query,
}, queryContext) }, queryContext)
} else if queryContext == "producer-throughput-overview-latency" { } else if queryContext == "producer-throughput-overview-byte-rate" {
bhq, err := buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd, attributeCache) bhq, err := buildBuilderQueriesProducerBytes(unixMilliStart, unixMilliEnd, attributeCache)
if err != nil { if err != nil {
return nil, err return nil, err
@ -284,7 +287,8 @@ func BuildQRParamsWithCache(messagingQueue *MessagingQueue, queryContext string,
cq = &v3.CompositeQuery{ cq = &v3.CompositeQuery{
QueryType: v3.QueryTypeBuilder, QueryType: v3.QueryTypeBuilder,
BuilderQueries: bhq, BuilderQueries: bhq,
PanelType: v3.PanelTypeList, PanelType: v3.PanelTypeTable,
FillGaps: false,
} }
} }
@ -315,7 +319,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer
if !ok { if !ok {
return nil, fmt.Errorf("invalid type for Topic") return nil, fmt.Errorf("invalid type for Topic")
} }
if queryContext != "consumer-throughput-details" { if !(queryContext == "consumer-throughput-details" || queryContext == "producer-throughput-details") {
partition, ok = messagingQueue.Variables["partition"] partition, ok = messagingQueue.Variables["partition"]
if !ok { if !ok {
return nil, fmt.Errorf("invalid type for Partition") return nil, fmt.Errorf("invalid type for Partition")
@ -364,7 +368,7 @@ func BuildClickHouseQuery(messagingQueue *MessagingQueue, queueType string, quer
func buildCompositeQuery(chq *v3.ClickHouseQuery, queryContext string) (*v3.CompositeQuery, error) { func buildCompositeQuery(chq *v3.ClickHouseQuery, queryContext string) (*v3.CompositeQuery, error) {
if queryContext == "producer-consumer-eval" || queryContext == "producer-throughput-overview" { if queryContext == "producer-consumer-eval" {
return &v3.CompositeQuery{ return &v3.CompositeQuery{
QueryType: v3.QueryTypeClickHouseSQL, QueryType: v3.QueryTypeClickHouseSQL,
ClickHouseQueries: map[string]*v3.ClickHouseQuery{queryContext: chq}, ClickHouseQueries: map[string]*v3.ClickHouseQuery{queryContext: chq},