From f8d3fa0fdb2bdb7f4f14f994adfb70c7515e47a7 Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Tue, 17 Oct 2023 11:49:50 +0530 Subject: [PATCH] chore: update query service telemetry (#3735) * chore: update query service telemetry * chore: address review comments * chore: add group call and update metrics condition * chore: update company_domain --- ee/query-service/app/server.go | 50 +++++++----- ee/query-service/constants/constants.go | 2 +- pkg/query-service/app/server.go | 49 +++++++----- pkg/query-service/dao/sqlite/connection.go | 1 + pkg/query-service/telemetry/ignored.go | 1 + pkg/query-service/telemetry/telemetry.go | 88 +++++++++++++++++----- 6 files changed, 130 insertions(+), 61 deletions(-) diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 5a1dcd9bd5..4d457776a1 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" _ "net/http/pprof" // http profiler @@ -20,9 +20,11 @@ import ( "github.com/soheilhy/cmux" "go.signoz.io/signoz/ee/query-service/app/api" "go.signoz.io/signoz/ee/query-service/app/db" + "go.signoz.io/signoz/ee/query-service/constants" "go.signoz.io/signoz/ee/query-service/dao" "go.signoz.io/signoz/ee/query-service/interfaces" baseInterface "go.signoz.io/signoz/pkg/query-service/interfaces" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" licensepkg "go.signoz.io/signoz/ee/query-service/license" "go.signoz.io/signoz/ee/query-service/usage" @@ -198,6 +200,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } telemetry.GetInstance().SetReader(reader) + telemetry.GetInstance().SetSaasOperator(constants.SaasSegmentKey) var c cache.Cache if serverOptions.CacheConfigPath != "" { @@ -385,20 +388,20 @@ func (lrw *loggingResponseWriter) Flush() { lrw.ResponseWriter.(http.Flusher).Flush() } -func extractDashboardMetaData(path string, r *http.Request) (map[string]interface{}, bool) { - pathToExtractBodyFrom := "/api/v2/metrics/query_range" +func extractQueryRangeV3Data(path string, r *http.Request) (map[string]interface{}, bool) { + pathToExtractBodyFrom := "/api/v3/query_range" data := map[string]interface{}{} - var postData *basemodel.QueryRangeParamsV2 + var postData *v3.QueryRangeParamsV3 if path == pathToExtractBodyFrom && (r.Method == "POST") { if r.Body != nil { - bodyBytes, err := ioutil.ReadAll(r.Body) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { return nil, false } r.Body.Close() // must close - r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) json.Unmarshal(bodyBytes, &postData) } else { @@ -409,24 +412,31 @@ func extractDashboardMetaData(path string, r *http.Request) (map[string]interfac return nil, false } - signozMetricNotFound := false - + signozMetricsUsed := false + signozLogsUsed := false + dataSources := []string{} if postData != nil { - signozMetricNotFound = telemetry.GetInstance().CheckSigNozMetricsV2(postData.CompositeMetricQuery) - if postData.CompositeMetricQuery != nil { - data["queryType"] = postData.CompositeMetricQuery.QueryType - data["panelType"] = postData.CompositeMetricQuery.PanelType + if postData.CompositeQuery != nil { + data["queryType"] = postData.CompositeQuery.QueryType + data["panelType"] = postData.CompositeQuery.PanelType + + signozLogsUsed, signozMetricsUsed = telemetry.GetInstance().CheckSigNozSignals(postData) } - - data["datasource"] = postData.DataSource } - if signozMetricNotFound { - telemetry.GetInstance().AddActiveMetricsUser() - telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_DASHBOARDS_METADATA, data, true) + if signozMetricsUsed || signozLogsUsed { + if signozMetricsUsed { + dataSources = append(dataSources, "metrics") + telemetry.GetInstance().AddActiveMetricsUser() + } + if signozLogsUsed { + dataSources = append(dataSources, "logs") + telemetry.GetInstance().AddActiveLogsUser() + } + data["dataSources"] = dataSources + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_QUERY_RANGE_V3, data, true) } - return data, true } @@ -449,7 +459,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { route := mux.CurrentRoute(r) path, _ := route.GetPathTemplate() - dashboardMetadata, metadataExists := extractDashboardMetaData(path, r) + queryRangeV3data, metadataExists := extractQueryRangeV3Data(path, r) getActiveLogs(path, r) lrw := NewLoggingResponseWriter(w) @@ -457,7 +467,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { data := map[string]interface{}{"path": path, "statusCode": lrw.statusCode} if metadataExists { - for key, value := range dashboardMetadata { + for key, value := range queryRangeV3data { data[key] = value } } diff --git a/ee/query-service/constants/constants.go b/ee/query-service/constants/constants.go index 4953f4d3eb..aeeea03cf2 100644 --- a/ee/query-service/constants/constants.go +++ b/ee/query-service/constants/constants.go @@ -10,7 +10,7 @@ const ( var LicenseSignozIo = "https://license.signoz.io/api/v1" var LicenseAPIKey = GetOrDefaultEnv("SIGNOZ_LICENSE_API_KEY", "") - +var SaasSegmentKey = GetOrDefaultEnv("SIGNOZ_SAAS_SEGMENT_KEY", "") var SpanLimitStr = GetOrDefaultEnv("SPAN_LIMIT", "5000") func GetOrDefaultEnv(key string, fallback string) string { diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 08fb4e7850..b71b9bde3e 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" _ "net/http/pprof" // http profiler @@ -24,6 +24,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/logparsingpipeline" "go.signoz.io/signoz/pkg/query-service/app/opamp" opAmpModel "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/app/explorer" "go.signoz.io/signoz/pkg/query-service/auth" @@ -320,20 +321,21 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) { func (lrw *loggingResponseWriter) Flush() { lrw.ResponseWriter.(http.Flusher).Flush() } -func extractDashboardMetaData(path string, r *http.Request) (map[string]interface{}, bool) { - pathToExtractBodyFrom := "/api/v2/metrics/query_range" + +func extractQueryRangeV3Data(path string, r *http.Request) (map[string]interface{}, bool) { + pathToExtractBodyFrom := "/api/v3/query_range" data := map[string]interface{}{} - var postData *model.QueryRangeParamsV2 + var postData *v3.QueryRangeParamsV3 if path == pathToExtractBodyFrom && (r.Method == "POST") { if r.Body != nil { - bodyBytes, err := ioutil.ReadAll(r.Body) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { return nil, false } r.Body.Close() // must close - r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) json.Unmarshal(bodyBytes, &postData) } else { @@ -344,24 +346,31 @@ func extractDashboardMetaData(path string, r *http.Request) (map[string]interfac return nil, false } - signozMetricNotFound := false - + signozMetricsUsed := false + signozLogsUsed := false + dataSources := []string{} if postData != nil { - signozMetricNotFound = telemetry.GetInstance().CheckSigNozMetricsV2(postData.CompositeMetricQuery) - if postData.CompositeMetricQuery != nil { - data["queryType"] = postData.CompositeMetricQuery.QueryType - data["panelType"] = postData.CompositeMetricQuery.PanelType + if postData.CompositeQuery != nil { + data["queryType"] = postData.CompositeQuery.QueryType + data["panelType"] = postData.CompositeQuery.PanelType + + signozLogsUsed, signozMetricsUsed = telemetry.GetInstance().CheckSigNozSignals(postData) } - - data["datasource"] = postData.DataSource } - if signozMetricNotFound { - telemetry.GetInstance().AddActiveMetricsUser() - telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_DASHBOARDS_METADATA, data, true) + if signozMetricsUsed || signozLogsUsed { + if signozMetricsUsed { + dataSources = append(dataSources, "metrics") + telemetry.GetInstance().AddActiveMetricsUser() + } + if signozLogsUsed { + dataSources = append(dataSources, "logs") + telemetry.GetInstance().AddActiveLogsUser() + } + data["dataSources"] = dataSources + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_QUERY_RANGE_V3, data, true) } - return data, true } @@ -384,7 +393,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { route := mux.CurrentRoute(r) path, _ := route.GetPathTemplate() - dashboardMetadata, metadataExists := extractDashboardMetaData(path, r) + queryRangeV3data, metadataExists := extractQueryRangeV3Data(path, r) getActiveLogs(path, r) lrw := NewLoggingResponseWriter(w) @@ -392,7 +401,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { data := map[string]interface{}{"path": path, "statusCode": lrw.statusCode} if metadataExists { - for key, value := range dashboardMetadata { + for key, value := range queryRangeV3data { data[key] = value } } diff --git a/pkg/query-service/dao/sqlite/connection.go b/pkg/query-service/dao/sqlite/connection.go index f79d67a122..a7335d6426 100644 --- a/pkg/query-service/dao/sqlite/connection.go +++ b/pkg/query-service/dao/sqlite/connection.go @@ -143,6 +143,7 @@ func (mds *ModelDaoSqlite) initializeOrgPreferences(ctx context.Context) error { telemetry.GetInstance().SetCountUsers(int8(countUsers)) if countUsers > 0 { telemetry.GetInstance().SetCompanyDomain(users[countUsers-1].Email) + telemetry.GetInstance().SetUserEmail(users[countUsers-1].Email) } return nil diff --git a/pkg/query-service/telemetry/ignored.go b/pkg/query-service/telemetry/ignored.go index 6ffacba848..29c06fe1ac 100644 --- a/pkg/query-service/telemetry/ignored.go +++ b/pkg/query-service/telemetry/ignored.go @@ -7,6 +7,7 @@ func IgnoredPaths() map[string]struct{} { "/api/v1/query_range": {}, "/api/v2/metrics/query_range": {}, "/api/v1/health": {}, + "/api/v1/featureFlags": {}, } return ignoredPaths diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 4cb4f501a3..cab790d8a3 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -15,6 +15,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/interfaces" "go.signoz.io/signoz/pkg/query-service/model" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/version" "gopkg.in/segmentio/analytics-go.v3" ) @@ -35,7 +36,7 @@ const ( TELEMETRY_EVENT_LANGUAGE = "Language" TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" TELEMETRY_EVENT_DISTRIBUTED = "Distributed" - TELEMETRY_EVENT_DASHBOARDS_METADATA = "Dashboards Metadata" + TELEMETRY_EVENT_QUERY_RANGE_V3 = "Query Range V3 Metadata" TELEMETRY_EVENT_ACTIVE_USER = "Active User" TELEMETRY_EVENT_ACTIVE_USER_PH = "Active User V2" ) @@ -74,28 +75,34 @@ func (a *Telemetry) IsSampled() bool { } -func (telemetry *Telemetry) CheckSigNozMetricsV2(compositeQuery *model.CompositeMetricQuery) bool { +func (telemetry *Telemetry) CheckSigNozSignals(postData *v3.QueryRangeParamsV3) (bool, bool) { + signozLogsUsed := false + signozMetricsUsed := false - signozMetricsNotFound := false - - if compositeQuery.BuilderQueries != nil && len(compositeQuery.BuilderQueries) > 0 { - if !strings.Contains(compositeQuery.BuilderQueries["A"].MetricName, "signoz_") && len(compositeQuery.BuilderQueries["A"].MetricName) > 0 { - signozMetricsNotFound = true + if postData.CompositeQuery.QueryType == v3.QueryTypeBuilder { + for _, query := range postData.CompositeQuery.BuilderQueries { + if query.DataSource == v3.DataSourceLogs && len(query.Filters.Items) > 0 { + signozLogsUsed = true + } else if query.DataSource == v3.DataSourceMetrics && + !strings.Contains(query.AggregateAttribute.Key, "signoz_") && + len(query.AggregateAttribute.Key) > 0 { + signozMetricsUsed = true + } + } + } else if postData.CompositeQuery.QueryType == v3.QueryTypePromQL { + for _, query := range postData.CompositeQuery.PromQueries { + if !strings.Contains(query.Query, "signoz_") && len(query.Query) > 0 { + signozMetricsUsed = true + } + } + } else if postData.CompositeQuery.QueryType == v3.QueryTypeClickHouseSQL { + for _, query := range postData.CompositeQuery.ClickHouseQueries { + if strings.Contains(query.Query, "signoz_metrics") && len(query.Query) > 0 { + signozMetricsUsed = true + } } } - - if compositeQuery.PromQueries != nil && len(compositeQuery.PromQueries) > 0 { - if !strings.Contains(compositeQuery.PromQueries["A"].Query, "signoz_") && len(compositeQuery.PromQueries["A"].Query) > 0 { - signozMetricsNotFound = true - } - } - if compositeQuery.ClickHouseQueries != nil && len(compositeQuery.ClickHouseQueries) > 0 { - if !strings.Contains(compositeQuery.ClickHouseQueries["A"].Query, "signoz_") && len(compositeQuery.ClickHouseQueries["A"].Query) > 0 { - signozMetricsNotFound = true - } - } - - return signozMetricsNotFound + return signozLogsUsed, signozMetricsUsed } func (telemetry *Telemetry) AddActiveTracesUser() { @@ -116,8 +123,10 @@ func (telemetry *Telemetry) AddActiveLogsUser() { type Telemetry struct { operator analytics.Client + saasOperator analytics.Client phOperator ph.Client ipAddress string + userEmail string isEnabled bool isAnonymous bool distinctId string @@ -249,10 +258,25 @@ func getOutboundIP() string { } func (a *Telemetry) IdentifyUser(user *model.User) { + if user.Email == "admin@admin.com" || user.Email == "admin@signoz.cloud" { + return + } a.SetCompanyDomain(user.Email) + a.SetUserEmail(user.Email) if !a.isTelemetryEnabled() || a.isTelemetryAnonymous() { return } + if a.saasOperator != nil { + a.saasOperator.Enqueue(analytics.Identify{ + UserId: a.userEmail, + Traits: analytics.NewTraits().SetName(user.Name).SetEmail(user.Email), + }) + a.saasOperator.Enqueue(analytics.Group{ + UserId: a.userEmail, + GroupId: a.getCompanyDomain(), + Traits: analytics.NewTraits().Set("company_domain", a.getCompanyDomain()), + }) + } a.operator.Enqueue(analytics.Identify{ UserId: a.ipAddress, @@ -272,6 +296,21 @@ func (a *Telemetry) SetCountUsers(countUsers int8) { a.countUsers = countUsers } +func (a *Telemetry) SetUserEmail(email string) { + a.userEmail = email +} + +func (a *Telemetry) GetUserEmail() string { + return a.userEmail +} + +func (a *Telemetry) SetSaasOperator(saasOperatorKey string) { + if saasOperatorKey == "" { + return + } + a.saasOperator = analytics.New(saasOperatorKey) +} + func (a *Telemetry) SetCompanyDomain(email string) { email_split := strings.Split(email, "@") @@ -342,6 +381,15 @@ func (a *Telemetry) SendEvent(event string, data map[string]interface{}, opts .. userId = a.GetDistinctId() } + if a.saasOperator != nil && a.GetUserEmail() != "" && + (event == TELEMETRY_EVENT_NUMBER_OF_SERVICES || event == TELEMETRY_EVENT_ACTIVE_USER) { + a.saasOperator.Enqueue(analytics.Track{ + Event: event, + UserId: a.GetUserEmail(), + Properties: properties, + }) + } + a.operator.Enqueue(analytics.Track{ Event: event, UserId: userId,