From b628765ae133765aeb53e8c64cfc631f54be80a5 Mon Sep 17 00:00:00 2001 From: Shivanshu Raj Shrivastava Date: Mon, 2 Jun 2025 13:00:19 +0530 Subject: [PATCH] feat: trace funnel queries Signed-off-by: Shivanshu Raj Shrivastava --- pkg/modules/tracefunnel/clickhouse_queries.go | 789 ++++++++++++++++++ pkg/modules/tracefunnel/query.go | 471 +++++++++++ pkg/query-service/app/http_handler.go | 247 +++++- .../app/traces/v4/query_builder.go | 56 ++ 4 files changed, 1559 insertions(+), 4 deletions(-) create mode 100644 pkg/modules/tracefunnel/clickhouse_queries.go create mode 100644 pkg/modules/tracefunnel/query.go diff --git a/pkg/modules/tracefunnel/clickhouse_queries.go b/pkg/modules/tracefunnel/clickhouse_queries.go new file mode 100644 index 0000000000..17123ce5a5 --- /dev/null +++ b/pkg/modules/tracefunnel/clickhouse_queries.go @@ -0,0 +1,789 @@ +package tracefunnel + +import ( + "fmt" +) + +func BuildTwoStepFunnelValidationQuery( + containsErrorT1 int, + containsErrorT2 int, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + clauseStep1 string, + clauseStep2 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + toDateTime64(%[3]d/1e9, 9) AS start_ts, + toDateTime64(%[4]d/1e9, 9) AS end_ts, + + ('%[5]s','%[6]s') AS step1, + ('%[7]s','%[8]s') AS step2 + +SELECT + trace_id +FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[9]s) + OR + (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[10]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time +) +ORDER BY t1_time +LIMIT 5;` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + clauseStep1, + clauseStep2, + ) +} + +func BuildThreeStepFunnelValidationQuery( + containsErrorT1 int, + containsErrorT2 int, + containsErrorT3 int, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + serviceNameT3 string, + spanNameT3 string, + clauseStep1 string, + clauseStep2 string, + clauseStep3 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + %[3]d AS contains_error_t3, + toDateTime64(%[4]d/1e9, 9) AS start_ts, + toDateTime64(%[5]d/1e9, 9) AS end_ts, + + ('%[6]s','%[7]s') AS step1, + ('%[8]s','%[9]s') AS step2, + ('%[10]s','%[11]s') AS step3 + +SELECT + trace_id +FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + minIf(timestamp, serviceName = step3.1 AND name = step3.2) AS t3_time + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[12]s) + OR (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[13]s) + OR (serviceName = step3.1 AND name = step3.2 AND (contains_error_t3 = 0 OR has_error = true) %[14]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time AND t3_time > t2_time +) +ORDER BY t1_time +LIMIT 5;` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + containsErrorT3, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + serviceNameT3, + spanNameT3, + clauseStep1, + clauseStep2, + clauseStep3, + ) +} + +func BuildTwoStepFunnelOverviewQuery( + containsErrorT1 int, + containsErrorT2 int, + latencyPointerT1 string, + latencyPointerT2 string, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + clauseStep1 string, + clauseStep2 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + '%[3]s' AS latency_pointer_t1, + '%[4]s' AS latency_pointer_t2, + toDateTime64(%[5]d/1e9, 9) AS start_ts, + toDateTime64(%[6]d/1e9, 9) AS end_ts, + (%[6]d - %[5]d)/1e9 AS time_window_sec, + + ('%[7]s','%[8]s') AS step1, + ('%[9]s','%[10]s') AS step2 + +, funnel AS ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS s1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS s2_error + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[11]s) + OR + (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[12]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time +) + +, totals AS ( + SELECT + count() AS total_s1_spans, + countIf(t2_time > t1_time) AS total_s2_spans, + sum(s1_error) AS sum_s1_error, + sum(s2_error) AS sum_s2_error, + avg((toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6) AS avg_duration, + quantile(0.99)((toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6) AS latency + FROM funnel +) + +SELECT + round(if(total_s1_spans > 0, total_s2_spans * 100.0 / total_s1_spans, 0), 2) AS conversion_rate, + total_s2_spans / time_window_sec AS avg_rate, + greatest(sum_s1_error, sum_s2_error) AS errors, + avg_duration, + latency +FROM totals; +` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + latencyPointerT1, + latencyPointerT2, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + clauseStep1, + clauseStep2, + ) +} + +func BuildThreeStepFunnelOverviewQuery( + containsErrorT1 int, + containsErrorT2 int, + containsErrorT3 int, + latencyPointerT1 string, + latencyPointerT2 string, + latencyPointerT3 string, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + serviceNameT3 string, + spanNameT3 string, + clauseStep1 string, + clauseStep2 string, + clauseStep3 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + %[3]d AS contains_error_t3, + '%[4]s' AS latency_pointer_t1, + '%[5]s' AS latency_pointer_t2, + '%[6]s' AS latency_pointer_t3, + toDateTime64(%[7]d/1e9, 9) AS start_ts, + toDateTime64(%[8]d/1e9, 9) AS end_ts, + (%[8]d - %[7]d)/1e9 AS time_window_sec, + + ('%[9]s','%[10]s') AS step1, + ('%[11]s','%[12]s') AS step2, + ('%[13]s','%[14]s') AS step3 + +, funnel AS ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + minIf(timestamp, serviceName = step3.1 AND name = step3.2) AS t3_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS s1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS s2_error, + toUInt8(anyIf(has_error, serviceName = step3.1 AND name = step3.2)) AS s3_error + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[15]s) + OR (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[16]s) + OR (serviceName = step3.1 AND name = step3.2 AND (contains_error_t3 = 0 OR has_error = true) %[17]s) + ) + GROUP BY trace_id +) + +, totals AS ( + SELECT + countIf(t1_time > 0) AS total_s1_spans, + countIf(t1_time > 0 AND t2_time > t1_time) AS total_s2_spans, + countIf(t2_time > 0 AND t3_time > t2_time) AS total_s3_spans, + + sum(s1_error) AS sum_s1_error, + sum(s2_error) AS sum_s2_error, + sum(s3_error) AS sum_s3_error, + + avgIf((toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time))/1e6, t1_time > 0 AND t2_time > t1_time) AS avg_duration_12, + quantileIf(0.99)((toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time))/1e6, t1_time > 0 AND t2_time > t1_time) AS latency_12, + + avgIf((toUnixTimestamp64Nano(t3_time) - toUnixTimestamp64Nano(t2_time))/1e6, t2_time > 0 AND t3_time > t2_time) AS avg_duration_23, + quantileIf(0.99)((toUnixTimestamp64Nano(t3_time) - toUnixTimestamp64Nano(t2_time))/1e6, t2_time > 0 AND t3_time > t2_time) AS latency_23 + FROM funnel +) + +SELECT + round(if(total_s1_spans > 0, total_s3_spans * 100.0 / total_s1_spans, 0), 2) AS conversion_rate, + total_s3_spans / nullIf(time_window_sec, 0) AS avg_rate, + greatest(sum_s1_error, sum_s2_error, sum_s3_error) AS errors, + avg_duration_23 AS avg_duration, + latency_23 AS latency +FROM totals; +` + return fmt.Sprintf( + queryTemplate, + containsErrorT1, + containsErrorT2, + containsErrorT3, + latencyPointerT1, + latencyPointerT2, + latencyPointerT3, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + serviceNameT3, + spanNameT3, + clauseStep1, + clauseStep2, + clauseStep3, + ) +} + +func BuildTwoStepFunnelCountQuery( + containsErrorT1 int, + containsErrorT2 int, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + clauseStep1 string, + clauseStep2 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + toDateTime64(%[3]d/1e9,9) AS start_ts, + toDateTime64(%[4]d/1e9,9) AS end_ts, + + ('%[5]s','%[6]s') AS step1, + ('%[7]s','%[8]s') AS step2 + +SELECT + count() AS total_s1_spans, + countIf(t1_error = 1) AS total_s1_errored_spans, + countIf(t2_time > t1_time) AS total_s2_spans, + countIf(t2_time > t1_time AND t2_error = 1) AS total_s2_errored_spans +FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS t1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS t2_error + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[9]s) + OR + (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[10]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time +) AS funnel; +` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + clauseStep1, + clauseStep2, + ) +} + +func BuildThreeStepFunnelCountQuery( + containsErrorT1 int, + containsErrorT2 int, + containsErrorT3 int, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + serviceNameT3 string, + spanNameT3 string, + clauseStep1 string, + clauseStep2 string, + clauseStep3 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + %[3]d AS contains_error_t3, + toDateTime64(%[4]d/1e9,9) AS start_ts, + toDateTime64(%[5]d/1e9,9) AS end_ts, + + ('%[6]s','%[7]s') AS step1, + ('%[8]s','%[9]s') AS step2, + ('%[10]s','%[11]s') AS step3 + +SELECT + count() AS total_s1_spans, + countIf(t1_error = 1) AS total_s1_errored_spans, + countIf(t2_time > t1_time) AS total_s2_spans, + countIf(t2_time > t1_time AND t2_error = 1) AS total_s2_errored_spans, + countIf(t2_time > t1_time AND t3_time > t2_time) AS total_s3_spans, + countIf(t2_time > t1_time AND t3_time > t2_time AND t3_error = 1) AS total_s3_errored_spans +FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + minIf(timestamp, serviceName = step3.1 AND name = step3.2) AS t3_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS t1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS t2_error, + toUInt8(anyIf(has_error, serviceName = step3.1 AND name = step3.2)) AS t3_error + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[12]s) + OR (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[13]s) + OR (serviceName = step3.1 AND name = step3.2 AND (contains_error_t3 = 0 OR has_error = true) %[14]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time AND t3_time > t2_time +) AS funnel; +` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + containsErrorT3, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + serviceNameT3, + spanNameT3, + clauseStep1, + clauseStep2, + clauseStep3, + ) +} + +func BuildTwoStepFunnelTopSlowTracesQuery( + containsErrorT1 int, + containsErrorT2 int, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + clauseStep1 string, + clauseStep2 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + toDateTime64(%[3]d/1e9, 9) AS start_ts, + toDateTime64(%[4]d/1e9, 9) AS end_ts, + + ('%[5]s','%[6]s') AS step1, + ('%[7]s','%[8]s') AS step2 + +SELECT + trace_id, + (toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6 AS duration_ms, + span_count +FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + count() AS span_count + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[9]s) + OR + (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[10]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time +) AS funnel +ORDER BY duration_ms DESC +LIMIT 5; +` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + clauseStep1, + clauseStep2, + ) +} + +func BuildTwoStepFunnelTopSlowErrorTracesQuery( + containsErrorT1 int, + containsErrorT2 int, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + clauseStep1 string, + clauseStep2 string, +) string { + queryTemplate := ` +WITH + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + toDateTime64(%[3]d/1e9, 9) AS start_ts, + toDateTime64(%[4]d/1e9, 9) AS end_ts, + + ('%[5]s','%[6]s') AS step1, + ('%[7]s','%[8]s') AS step2 + +SELECT + trace_id, + (toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6 AS duration_ms, + span_count +FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS t1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS t2_error, + count() AS span_count + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND ( + (serviceName = step1.1 AND name = step1.2 AND (contains_error_t1 = 0 OR has_error = true) %[9]s) + OR + (serviceName = step2.1 AND name = step2.2 AND (contains_error_t2 = 0 OR has_error = true) %[10]s) + ) + GROUP BY trace_id + HAVING t1_time > 0 AND t2_time > t1_time +) AS funnel +WHERE + (t1_error = 1 OR t2_error = 1) +ORDER BY duration_ms DESC +LIMIT 5; +` + return fmt.Sprintf(queryTemplate, + containsErrorT1, + containsErrorT2, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + clauseStep1, + clauseStep2, + ) +} + +func BuildTwoStepFunnelStepOverviewQuery( + containsErrorT1 int, + containsErrorT2 int, + latencyPointerT1 string, + latencyPointerT2 string, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + clauseStep1 string, + clauseStep2 string, + latencyTypeT2 string, +) string { + const tpl = ` +WITH + toDateTime64(%[5]d / 1e9, 9) AS start_ts, + toDateTime64(%[6]d / 1e9, 9) AS end_ts, + (%[6]d - %[5]d) / 1e9 AS time_window_sec, + + ('%[7]s', '%[8]s') AS step1, + ('%[9]s', '%[10]s') AS step2, + + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2 + +SELECT + round(total_s2_spans * 100.0 / total_s1_spans, 2) AS conversion_rate, + total_s2_spans / time_window_sec AS avg_rate, + greatest(sum_s1_error, sum_s2_error) AS errors, + avg_duration, + latency +FROM ( + SELECT + countIf(t1_time > 0) AS total_s1_spans, + countIf(t1_time > 0 AND t2_time > t1_time) AS total_s2_spans, + sum(s1_error) AS sum_s1_error, + sum(s2_error) AS sum_s2_error, + + avgIf( + (toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6, + t1_time > 0 AND t2_time > t1_time + ) AS avg_duration, + + quantileIf(%[13]s)( + (toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6, + t1_time > 0 AND t2_time > t1_time + ) AS latency + FROM ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS s1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS s2_error + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND serviceName IN (step1.1, step2.1) + AND name IN (step1.2, step2.2) + AND ((contains_error_t1 = 0) OR (has_error AND serviceName = step1.1 AND name = step1.2)) %[11]s + AND ((contains_error_t2 = 0) OR (has_error AND serviceName = step2.1 AND name = step2.2)) %[12]s + GROUP BY trace_id + ) AS funnel +) AS totals; +` + + return fmt.Sprintf(tpl, + containsErrorT1, + containsErrorT2, + latencyPointerT1, + latencyPointerT2, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + clauseStep1, + clauseStep2, + latencyTypeT2, + ) +} + +func BuildThreeStepFunnelStepOverviewQuery( + containsErrorT1 int, + containsErrorT2 int, + containsErrorT3 int, + latencyPointerT1 string, + latencyPointerT2 string, + latencyPointerT3 string, + startTs int64, + endTs int64, + serviceNameT1 string, + spanNameT1 string, + serviceNameT2 string, + spanNameT2 string, + serviceNameT3 string, + spanNameT3 string, + clauseStep1 string, + clauseStep2 string, + clauseStep3 string, + stepStart int64, + stepEnd int64, + latencyTypeT2 string, + latencyTypeT3 string, +) string { + const baseWithAndFunnel = ` +WITH + toDateTime64(%[7]d/1e9, 9) AS start_ts, + toDateTime64(%[8]d/1e9, 9) AS end_ts, + (%[8]d - %[7]d) / 1e9 AS time_window_sec, + + ('%[9]s','%[10]s') AS step1, + ('%[11]s','%[12]s') AS step2, + ('%[13]s','%[14]s') AS step3, + + %[1]d AS contains_error_t1, + %[2]d AS contains_error_t2, + %[3]d AS contains_error_t3, + + funnel AS ( + SELECT + trace_id, + minIf(timestamp, serviceName = step1.1 AND name = step1.2) AS t1_time, + minIf(timestamp, serviceName = step2.1 AND name = step2.2) AS t2_time, + minIf(timestamp, serviceName = step3.1 AND name = step3.2) AS t3_time, + toUInt8(anyIf(has_error, serviceName = step1.1 AND name = step1.2)) AS s1_error, + toUInt8(anyIf(has_error, serviceName = step2.1 AND name = step2.2)) AS s2_error, + toUInt8(anyIf(has_error, serviceName = step3.1 AND name = step3.2)) AS s3_error + FROM signoz_traces.signoz_index_v3 + WHERE + timestamp BETWEEN start_ts AND end_ts + AND serviceName IN (step1.1, step2.1, step3.1) + AND name IN (step1.2, step2.2, step3.2) + AND ((contains_error_t1 = 0) OR (has_error AND serviceName = step1.1 AND name = step1.2)) %[15]s + AND ((contains_error_t2 = 0) OR (has_error AND serviceName = step2.1 AND name = step2.2)) %[16]s + AND ((contains_error_t3 = 0) OR (has_error AND serviceName = step3.1 AND name = step3.2)) %[17]s + GROUP BY trace_id + ) +` + + const totals12 = ` +SELECT + round(if(total_s1_spans > 0, total_s2_spans * 100.0 / total_s1_spans, 0), 2) AS conversion_rate, + total_s2_spans / time_window_sec AS avg_rate, + greatest(sum_s1_error, sum_s2_error) AS errors, + avg_duration_12 AS avg_duration, + latency_12 AS latency +FROM ( + SELECT + countIf(t1_time > 0 AND t2_time > t1_time) AS total_s2_spans, + countIf(t1_time > 0) AS total_s1_spans, -- eligible only + sum(s1_error) AS sum_s1_error, + sum(s2_error) AS sum_s2_error, + avgIf((toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6, t1_time > 0 AND t2_time > t1_time) AS avg_duration_12, + quantileIf(%[18]s)((toUnixTimestamp64Nano(t2_time) - toUnixTimestamp64Nano(t1_time)) / 1e6, t1_time > 0 AND t2_time > t1_time) AS latency_12 + FROM funnel +) AS totals; +` + + const totals23 = ` +SELECT + round(if(total_s2_spans > 0, total_s3_spans * 100.0 / total_s2_spans, 0), 2) AS conversion_rate, + total_s3_spans / time_window_sec AS avg_rate, + greatest(sum_s2_error, sum_s3_error) AS errors, + avg_duration_23 AS avg_duration, + latency_23 AS latency +FROM ( + SELECT + countIf(t2_time > 0 AND t3_time > t2_time) AS total_s3_spans, + countIf(t2_time > 0) AS total_s2_spans, -- eligible only + sum(s2_error) AS sum_s2_error, + sum(s3_error) AS sum_s3_error, + avgIf((toUnixTimestamp64Nano(t3_time) - toUnixTimestamp64Nano(t2_time)) / 1e6, t2_time > 0 AND t3_time > t2_time) AS avg_duration_23, + quantileIf(%[19]s)((toUnixTimestamp64Nano(t3_time) - toUnixTimestamp64Nano(t2_time)) / 1e6, t2_time > 0 AND t3_time > t2_time) AS latency_23 + FROM funnel +) AS totals; +` + + const fallback = ` +SELECT 0 AS conversion_rate, 0 AS avg_rate, 0 AS errors, 0 AS avg_duration, 0 AS latency; +` + + var totalsTpl string + switch { + case stepStart == 1 && stepEnd == 2: + totalsTpl = totals12 + case stepStart == 2 && stepEnd == 3: + totalsTpl = totals23 + default: + totalsTpl = fallback + } + + return fmt.Sprintf( + baseWithAndFunnel+totalsTpl, + containsErrorT1, + containsErrorT2, + containsErrorT3, + latencyPointerT1, + latencyPointerT2, + latencyPointerT3, + startTs, + endTs, + serviceNameT1, + spanNameT1, + serviceNameT2, + spanNameT2, + serviceNameT3, + spanNameT3, + clauseStep1, + clauseStep2, + clauseStep3, + latencyTypeT2, + latencyTypeT3, + ) +} diff --git a/pkg/modules/tracefunnel/query.go b/pkg/modules/tracefunnel/query.go new file mode 100644 index 0000000000..0a083e9c7b --- /dev/null +++ b/pkg/modules/tracefunnel/query.go @@ -0,0 +1,471 @@ +package tracefunnel + +import ( + "fmt" + "strings" + + tracev4 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v4" + v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" + "github.com/SigNoz/signoz/pkg/types/tracefunneltypes" +) + +// sanitizeClause adds AND prefix to non-empty clauses +func sanitizeClause(clause string) string { + if clause == "" { + return "" + } + return "AND " + clause +} + +func ValidateTraces(funnel *tracefunneltypes.StorableFunnel, timeRange tracefunneltypes.TimeRange) (*v3.ClickHouseQuery, error) { + var query string + var err error + + funnelSteps := funnel.Steps + containsErrorT1 := 0 + containsErrorT2 := 0 + containsErrorT3 := 0 + + if funnelSteps[0].HasErrors { + containsErrorT1 = 1 + } + if funnelSteps[1].HasErrors { + containsErrorT2 = 1 + } + if len(funnel.Steps) > 2 && funnelSteps[2].HasErrors { + containsErrorT3 = 1 + } + + // Build filter clauses for each step + clauseStep1, err := tracev4.BuildTracesFilter(funnelSteps[0].Filters) + if err != nil { + return nil, err + } + clauseStep2, err := tracev4.BuildTracesFilter(funnelSteps[1].Filters) + if err != nil { + return nil, err + } + clauseStep3 := "" + if len(funnel.Steps) > 2 { + clauseStep3, err = tracev4.BuildTracesFilter(funnelSteps[2].Filters) + if err != nil { + return nil, err + } + } + + // Sanitize clauses + clauseStep1 = sanitizeClause(clauseStep1) + clauseStep2 = sanitizeClause(clauseStep2) + clauseStep3 = sanitizeClause(clauseStep3) + + if len(funnel.Steps) > 2 { + query = BuildThreeStepFunnelValidationQuery( + containsErrorT1, + containsErrorT2, + containsErrorT3, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + funnelSteps[2].ServiceName, + funnelSteps[2].SpanName, + clauseStep1, + clauseStep2, + clauseStep3, + ) + } else { + query = BuildTwoStepFunnelValidationQuery( + containsErrorT1, + containsErrorT2, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + clauseStep1, + clauseStep2, + ) + } + + return &v3.ClickHouseQuery{ + Query: query, + }, nil +} + +func GetFunnelAnalytics(funnel *tracefunneltypes.StorableFunnel, timeRange tracefunneltypes.TimeRange) (*v3.ClickHouseQuery, error) { + var query string + var err error + + funnelSteps := funnel.Steps + containsErrorT1 := 0 + containsErrorT2 := 0 + containsErrorT3 := 0 + latencyPointerT1 := funnelSteps[0].LatencyPointer + latencyPointerT2 := funnelSteps[1].LatencyPointer + latencyPointerT3 := "start" + if len(funnel.Steps) > 2 { + latencyPointerT3 = funnelSteps[2].LatencyPointer + } + + if funnelSteps[0].HasErrors { + containsErrorT1 = 1 + } + if funnelSteps[1].HasErrors { + containsErrorT2 = 1 + } + if len(funnel.Steps) > 2 && funnelSteps[2].HasErrors { + containsErrorT3 = 1 + } + + // Build filter clauses for each step + clauseStep1, err := tracev4.BuildTracesFilter(funnelSteps[0].Filters) + if err != nil { + return nil, err + } + clauseStep2, err := tracev4.BuildTracesFilter(funnelSteps[1].Filters) + if err != nil { + return nil, err + } + clauseStep3 := "" + if len(funnel.Steps) > 2 { + clauseStep3, err = tracev4.BuildTracesFilter(funnelSteps[2].Filters) + if err != nil { + return nil, err + } + } + + // Sanitize clauses + clauseStep1 = sanitizeClause(clauseStep1) + clauseStep2 = sanitizeClause(clauseStep2) + clauseStep3 = sanitizeClause(clauseStep3) + + if len(funnel.Steps) > 2 { + query = BuildThreeStepFunnelOverviewQuery( + containsErrorT1, + containsErrorT2, + containsErrorT3, + latencyPointerT1, + latencyPointerT2, + latencyPointerT3, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + funnelSteps[2].ServiceName, + funnelSteps[2].SpanName, + clauseStep1, + clauseStep2, + clauseStep3, + ) + } else { + query = BuildTwoStepFunnelOverviewQuery( + containsErrorT1, + containsErrorT2, + latencyPointerT1, + latencyPointerT2, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + clauseStep1, + clauseStep2, + ) + } + return &v3.ClickHouseQuery{Query: query}, nil +} + +func GetFunnelStepAnalytics(funnel *tracefunneltypes.StorableFunnel, timeRange tracefunneltypes.TimeRange, stepStart, stepEnd int64) (*v3.ClickHouseQuery, error) { + var query string + var err error + + funnelSteps := funnel.Steps + containsErrorT1 := 0 + containsErrorT2 := 0 + containsErrorT3 := 0 + latencyPointerT1 := funnelSteps[0].LatencyPointer + latencyPointerT2 := funnelSteps[1].LatencyPointer + latencyPointerT3 := "start" + if len(funnel.Steps) > 2 { + latencyPointerT3 = funnelSteps[2].LatencyPointer + } + latencyTypeT2 := "0.99" + latencyTypeT3 := "0.99" + + if stepStart == stepEnd { + return nil, fmt.Errorf("step start and end cannot be the same for /step/overview") + } + + if funnelSteps[0].HasErrors { + containsErrorT1 = 1 + } + if funnelSteps[1].HasErrors { + containsErrorT2 = 1 + } + if len(funnel.Steps) > 2 && funnelSteps[2].HasErrors { + containsErrorT3 = 1 + } + + if funnelSteps[1].LatencyType != "" { + latency := strings.ToLower(funnelSteps[1].LatencyType) + if latency == "p90" { + latencyTypeT2 = "0.90" + } else if latency == "p95" { + latencyTypeT2 = "0.95" + } else { + latencyTypeT2 = "0.99" + } + } + if len(funnel.Steps) > 2 && funnelSteps[2].LatencyType != "" { + latency := strings.ToLower(funnelSteps[2].LatencyType) + if latency == "p90" { + latencyTypeT3 = "0.90" + } else if latency == "p95" { + latencyTypeT3 = "0.95" + } else { + latencyTypeT3 = "0.99" + } + } + + // Build filter clauses for each step + clauseStep1, err := tracev4.BuildTracesFilter(funnelSteps[0].Filters) + if err != nil { + return nil, err + } + clauseStep2, err := tracev4.BuildTracesFilter(funnelSteps[1].Filters) + if err != nil { + return nil, err + } + clauseStep3 := "" + if len(funnel.Steps) > 2 { + clauseStep3, err = tracev4.BuildTracesFilter(funnelSteps[2].Filters) + if err != nil { + return nil, err + } + } + + // Sanitize clauses + clauseStep1 = sanitizeClause(clauseStep1) + clauseStep2 = sanitizeClause(clauseStep2) + clauseStep3 = sanitizeClause(clauseStep3) + + if len(funnel.Steps) > 2 { + query = BuildThreeStepFunnelStepOverviewQuery( + containsErrorT1, + containsErrorT2, + containsErrorT3, + latencyPointerT1, + latencyPointerT2, + latencyPointerT3, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + funnelSteps[2].ServiceName, + funnelSteps[2].SpanName, + clauseStep1, + clauseStep2, + clauseStep3, + stepStart, + stepEnd, + latencyTypeT2, + latencyTypeT3, + ) + } else { + query = BuildTwoStepFunnelStepOverviewQuery( + containsErrorT1, + containsErrorT2, + latencyPointerT1, + latencyPointerT2, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + clauseStep1, + clauseStep2, + latencyTypeT2, + ) + } + return &v3.ClickHouseQuery{Query: query}, nil +} + +func GetStepAnalytics(funnel *tracefunneltypes.StorableFunnel, timeRange tracefunneltypes.TimeRange) (*v3.ClickHouseQuery, error) { + var query string + + funnelSteps := funnel.Steps + containsErrorT1 := 0 + containsErrorT2 := 0 + containsErrorT3 := 0 + + if funnelSteps[0].HasErrors { + containsErrorT1 = 1 + } + if funnelSteps[1].HasErrors { + containsErrorT2 = 1 + } + if len(funnel.Steps) > 2 && funnelSteps[2].HasErrors { + containsErrorT3 = 1 + } + + // Build filter clauses for each step + clauseStep1, err := tracev4.BuildTracesFilter(funnelSteps[0].Filters) + if err != nil { + return nil, err + } + clauseStep2, err := tracev4.BuildTracesFilter(funnelSteps[1].Filters) + if err != nil { + return nil, err + } + clauseStep3 := "" + if len(funnel.Steps) > 2 { + clauseStep3, err = tracev4.BuildTracesFilter(funnelSteps[2].Filters) + if err != nil { + return nil, err + } + } + + // Sanitize clauses + clauseStep1 = sanitizeClause(clauseStep1) + clauseStep2 = sanitizeClause(clauseStep2) + clauseStep3 = sanitizeClause(clauseStep3) + + if len(funnel.Steps) > 2 { + query = BuildThreeStepFunnelCountQuery( + containsErrorT1, + containsErrorT2, + containsErrorT3, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + funnelSteps[2].ServiceName, + funnelSteps[2].SpanName, + clauseStep1, + clauseStep2, + clauseStep3, + ) + } else { + query = BuildTwoStepFunnelCountQuery( + containsErrorT1, + containsErrorT2, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[0].ServiceName, + funnelSteps[0].SpanName, + funnelSteps[1].ServiceName, + funnelSteps[1].SpanName, + clauseStep1, + clauseStep2, + ) + } + + return &v3.ClickHouseQuery{ + Query: query, + }, nil +} + +func GetSlowestTraces(funnel *tracefunneltypes.StorableFunnel, timeRange tracefunneltypes.TimeRange, stepStart, stepEnd int64) (*v3.ClickHouseQuery, error) { + funnelSteps := funnel.Steps + containsErrorT1 := 0 + containsErrorT2 := 0 + stepStartOrder := 0 + stepEndOrder := 1 + + if stepStart != stepEnd { + stepStartOrder = int(stepStart) - 1 + stepEndOrder = int(stepEnd) - 1 + if funnelSteps[stepStartOrder].HasErrors { + containsErrorT1 = 1 + } + if funnelSteps[stepEndOrder].HasErrors { + containsErrorT2 = 1 + } + } + + // Build filter clauses for the steps + clauseStep1, err := tracev4.BuildTracesFilter(funnelSteps[stepStartOrder].Filters) + if err != nil { + return nil, err + } + clauseStep2, err := tracev4.BuildTracesFilter(funnelSteps[stepEndOrder].Filters) + if err != nil { + return nil, err + } + + // Sanitize clauses + clauseStep1 = sanitizeClause(clauseStep1) + clauseStep2 = sanitizeClause(clauseStep2) + + query := BuildTwoStepFunnelTopSlowTracesQuery( + containsErrorT1, + containsErrorT2, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[stepStartOrder].ServiceName, + funnelSteps[stepStartOrder].SpanName, + funnelSteps[stepEndOrder].ServiceName, + funnelSteps[stepEndOrder].SpanName, + clauseStep1, + clauseStep2, + ) + return &v3.ClickHouseQuery{Query: query}, nil +} + +func GetErroredTraces(funnel *tracefunneltypes.StorableFunnel, timeRange tracefunneltypes.TimeRange, stepStart, stepEnd int64) (*v3.ClickHouseQuery, error) { + funnelSteps := funnel.Steps + containsErrorT1 := 0 + containsErrorT2 := 0 + stepStartOrder := 0 + stepEndOrder := 1 + + if stepStart != stepEnd { + stepStartOrder = int(stepStart) - 1 + stepEndOrder = int(stepEnd) - 1 + if funnelSteps[stepStartOrder].HasErrors { + containsErrorT1 = 1 + } + if funnelSteps[stepEndOrder].HasErrors { + containsErrorT2 = 1 + } + } + + // Build filter clauses for the steps + clauseStep1, err := tracev4.BuildTracesFilter(funnelSteps[stepStartOrder].Filters) + if err != nil { + return nil, err + } + clauseStep2, err := tracev4.BuildTracesFilter(funnelSteps[stepEndOrder].Filters) + if err != nil { + return nil, err + } + + // Sanitize clauses + clauseStep1 = sanitizeClause(clauseStep1) + clauseStep2 = sanitizeClause(clauseStep2) + + query := BuildTwoStepFunnelTopSlowErrorTracesQuery( + containsErrorT1, + containsErrorT2, + timeRange.StartTime, + timeRange.EndTime, + funnelSteps[stepStartOrder].ServiceName, + funnelSteps[stepStartOrder].SpanName, + funnelSteps[stepEndOrder].ServiceName, + funnelSteps[stepEndOrder].SpanName, + clauseStep1, + clauseStep2, + ) + return &v3.ClickHouseQuery{Query: query}, nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index c3e6c7fb32..488d11569d 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -41,6 +41,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/SigNoz/signoz/pkg/cache" + traceFunnelsModule "github.com/SigNoz/signoz/pkg/modules/tracefunnel" "github.com/SigNoz/signoz/pkg/query-service/agentConf" "github.com/SigNoz/signoz/pkg/query-service/app/cloudintegrations" "github.com/SigNoz/signoz/pkg/query-service/app/inframetrics" @@ -56,6 +57,7 @@ import ( "github.com/SigNoz/signoz/pkg/query-service/app/queryBuilder" tracesV3 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v3" tracesV4 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v4" + "github.com/SigNoz/signoz/pkg/query-service/constants" "github.com/SigNoz/signoz/pkg/query-service/contextlinks" v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" "github.com/SigNoz/signoz/pkg/query-service/postprocess" @@ -65,6 +67,7 @@ import ( "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/pipelinetypes" ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes" + traceFunnels "github.com/SigNoz/signoz/pkg/types/tracefunneltypes" "go.uber.org/zap" @@ -5161,22 +5164,258 @@ func (aH *APIHandler) RegisterTraceFunnelsRoutes(router *mux.Router, am *middlew // API endpoints traceFunnelsRouter.HandleFunc("/new", - am.EditAccess(aH.Signoz.Handlers.TraceFunnel.New)). + am.ViewAccess(aH.Signoz.Handlers.TraceFunnel.New)). Methods(http.MethodPost) traceFunnelsRouter.HandleFunc("/list", am.ViewAccess(aH.Signoz.Handlers.TraceFunnel.List)). Methods(http.MethodGet) traceFunnelsRouter.HandleFunc("/steps/update", - am.EditAccess(aH.Signoz.Handlers.TraceFunnel.UpdateSteps)). + am.ViewAccess(aH.Signoz.Handlers.TraceFunnel.UpdateSteps)). Methods(http.MethodPut) traceFunnelsRouter.HandleFunc("/{funnel_id}", am.ViewAccess(aH.Signoz.Handlers.TraceFunnel.Get)). Methods(http.MethodGet) traceFunnelsRouter.HandleFunc("/{funnel_id}", - am.EditAccess(aH.Signoz.Handlers.TraceFunnel.Delete)). + am.ViewAccess(aH.Signoz.Handlers.TraceFunnel.Delete)). Methods(http.MethodDelete) traceFunnelsRouter.HandleFunc("/{funnel_id}", - am.EditAccess(aH.Signoz.Handlers.TraceFunnel.UpdateFunnel)). + am.ViewAccess(aH.Signoz.Handlers.TraceFunnel.UpdateFunnel)). Methods(http.MethodPut) + + // Analytics endpoints + traceFunnelsRouter.HandleFunc("/{funnel_id}/analytics/validate", aH.handleValidateTraces).Methods("POST") + traceFunnelsRouter.HandleFunc("/{funnel_id}/analytics/overview", aH.handleFunnelAnalytics).Methods("POST") + traceFunnelsRouter.HandleFunc("/{funnel_id}/analytics/steps", aH.handleStepAnalytics).Methods("POST") + traceFunnelsRouter.HandleFunc("/{funnel_id}/analytics/steps/overview", aH.handleFunnelStepAnalytics).Methods("POST") + traceFunnelsRouter.HandleFunc("/{funnel_id}/analytics/slow-traces", aH.handleFunnelSlowTraces).Methods("POST") + traceFunnelsRouter.HandleFunc("/{funnel_id}/analytics/error-traces", aH.handleFunnelErrorTraces).Methods("POST") + +} + +func (aH *APIHandler) handleValidateTraces(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funnelID := vars["funnel_id"] + + claims, err := authtypes.ClaimsFromContext(r.Context()) + + if err != nil { + render.Error(w, err) + return + } + + funnel, err := aH.Signoz.Modules.TraceFunnel.Get(r.Context(), valuer.MustNewUUID(funnelID), valuer.MustNewUUID(claims.OrgID)) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("funnel not found: %v", err)}, nil) + return + } + + var timeRange traceFunnels.TimeRange + if err := json.NewDecoder(r.Body).Decode(&timeRange); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("error decoding time range: %v", err)}, nil) + return + } + + if len(funnel.Steps) < 2 { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("funnel must have at least 2 steps")}, nil) + return + } + + chq, err := traceFunnelsModule.ValidateTraces(funnel, timeRange) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error building clickhouse query: %v", err)}, nil) + return + } + + results, err := aH.reader.GetListResultV3(r.Context(), chq.Query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error converting clickhouse results to list: %v", err)}, nil) + return + } + aH.Respond(w, results) +} + +func (aH *APIHandler) handleFunnelAnalytics(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funnelID := vars["funnel_id"] + + claims, err := authtypes.ClaimsFromContext(r.Context()) + + if err != nil { + render.Error(w, err) + return + } + + funnel, err := aH.Signoz.Modules.TraceFunnel.Get(r.Context(), valuer.MustNewUUID(funnelID), valuer.MustNewUUID(claims.OrgID)) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("funnel not found: %v", err)}, nil) + return + } + + var stepTransition traceFunnels.StepTransitionRequest + if err := json.NewDecoder(r.Body).Decode(&stepTransition); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("error decoding time range: %v", err)}, nil) + return + } + + chq, err := traceFunnelsModule.GetFunnelAnalytics(funnel, stepTransition.TimeRange) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error building clickhouse query: %v", err)}, nil) + return + } + + results, err := aH.reader.GetListResultV3(r.Context(), chq.Query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error converting clickhouse results to list: %v", err)}, nil) + return + } + aH.Respond(w, results) +} + +func (aH *APIHandler) handleFunnelStepAnalytics(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funnelID := vars["funnel_id"] + + claims, err := authtypes.ClaimsFromContext(r.Context()) + + if err != nil { + render.Error(w, err) + return + } + + funnel, err := aH.Signoz.Modules.TraceFunnel.Get(r.Context(), valuer.MustNewUUID(funnelID), valuer.MustNewUUID(claims.OrgID)) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("funnel not found: %v", err)}, nil) + return + } + + var stepTransition traceFunnels.StepTransitionRequest + if err := json.NewDecoder(r.Body).Decode(&stepTransition); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("error decoding time range: %v", err)}, nil) + return + } + + chq, err := traceFunnelsModule.GetFunnelStepAnalytics(funnel, stepTransition.TimeRange, stepTransition.StepStart, stepTransition.StepEnd) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error building clickhouse query: %v", err)}, nil) + return + } + + results, err := aH.reader.GetListResultV3(r.Context(), chq.Query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error converting clickhouse results to list: %v", err)}, nil) + return + } + aH.Respond(w, results) +} + +func (aH *APIHandler) handleStepAnalytics(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funnelID := vars["funnel_id"] + + claims, err := authtypes.ClaimsFromContext(r.Context()) + + if err != nil { + render.Error(w, err) + return + } + + funnel, err := aH.Signoz.Modules.TraceFunnel.Get(r.Context(), valuer.MustNewUUID(funnelID), valuer.MustNewUUID(claims.OrgID)) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("funnel not found: %v", err)}, nil) + return + } + + var timeRange traceFunnels.TimeRange + if err := json.NewDecoder(r.Body).Decode(&timeRange); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("error decoding time range: %v", err)}, nil) + return + } + + chq, err := traceFunnelsModule.GetStepAnalytics(funnel, timeRange) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error building clickhouse query: %v", err)}, nil) + return + } + + results, err := aH.reader.GetListResultV3(r.Context(), chq.Query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error converting clickhouse results to list: %v", err)}, nil) + return + } + aH.Respond(w, results) +} + +func (aH *APIHandler) handleFunnelSlowTraces(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funnelID := vars["funnel_id"] + + claims, err := authtypes.ClaimsFromContext(r.Context()) + + if err != nil { + render.Error(w, err) + return + } + + funnel, err := aH.Signoz.Modules.TraceFunnel.Get(r.Context(), valuer.MustNewUUID(funnelID), valuer.MustNewUUID(claims.OrgID)) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("funnel not found: %v", err)}, nil) + return + } + + var req traceFunnels.StepTransitionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("invalid request body: %v", err)}, nil) + return + } + + chq, err := traceFunnelsModule.GetSlowestTraces(funnel, req.TimeRange, req.StepStart, req.StepEnd) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error building clickhouse query: %v", err)}, nil) + return + } + + results, err := aH.reader.GetListResultV3(r.Context(), chq.Query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error converting clickhouse results to list: %v", err)}, nil) + return + } + aH.Respond(w, results) +} + +func (aH *APIHandler) handleFunnelErrorTraces(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + funnelID := vars["funnel_id"] + + claims, err := authtypes.ClaimsFromContext(r.Context()) + + if err != nil { + render.Error(w, err) + return + } + + funnel, err := aH.Signoz.Modules.TraceFunnel.Get(r.Context(), valuer.MustNewUUID(funnelID), valuer.MustNewUUID(claims.OrgID)) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("funnel not found: %v", err)}, nil) + return + } + + var req traceFunnels.StepTransitionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("invalid request body: %v", err)}, nil) + return + } + + chq, err := traceFunnelsModule.GetErroredTraces(funnel, req.TimeRange, req.StepStart, req.StepEnd) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error building clickhouse query: %v", err)}, nil) + return + } + + results, err := aH.reader.GetListResultV3(r.Context(), chq.Query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: fmt.Errorf("error converting clickhouse results to list: %v", err)}, nil) + return + } + aH.Respond(w, results) } diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 69a76de4d1..f67390b071 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -148,6 +148,62 @@ func BuildTracesFilterQuery(fs *v3.FilterSet) (string, error) { return queryString, nil } +func BuildTracesFilter(fs *v3.FilterSet) (string, error) { + var conditions []string + + if fs != nil && len(fs.Items) != 0 { + for _, item := range fs.Items { + + val := item.Value + // generate the key + columnName := getColumnName(item.Key) + var fmtVal string + item.Operator = v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) + if item.Operator != v3.FilterOperatorExists && item.Operator != v3.FilterOperatorNotExists { + var err error + val, err = utils.ValidateAndCastValue(val, item.Key.DataType) + if err != nil { + return "", fmt.Errorf("invalid value for key %s: %v", item.Key.Key, err) + } + } + if val != nil { + fmtVal = utils.ClickHouseFormattedValue(val) + } + if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { + switch item.Operator { + case v3.FilterOperatorContains, v3.FilterOperatorNotContains: + // we also want to treat %, _ as literals for contains + val := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", item.Value), false) + conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, val)) + case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: + conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal)) + case v3.FilterOperatorExists, v3.FilterOperatorNotExists: + if item.Key.IsColumn { + subQuery, err := existsSubQueryForFixedColumn(item.Key, item.Operator) + if err != nil { + return "", err + } + conditions = append(conditions, subQuery) + } else { + cType := getClickHouseTracesColumnType(item.Key.Type) + cDataType := getClickHouseTracesColumnDataType(item.Key.DataType) + col := fmt.Sprintf("%s_%s", cType, cDataType) + conditions = append(conditions, fmt.Sprintf(operator, col, item.Key.Key)) + } + + default: + conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) + } + } else { + return "", fmt.Errorf("unsupported operator %s", item.Operator) + } + } + } + queryString := strings.Join(conditions, " AND ") + + return queryString, nil +} + func handleEmptyValuesInGroupBy(groupBy []v3.AttributeKey) (string, error) { // TODO(nitya): in future when we support user based mat column handle them // skipping now as we don't support creating them