chore: move analytics related methods from CH reader to their own mod… (#5935)

pkg/query-service/app/clickhouseReader/reader.go

@@ -42,14 +42,11 @@ import (
 	"go.uber.org/zap"
 
 	queryprogress "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader/query_progress"
-	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
-	"go.signoz.io/signoz/pkg/query-service/app/explorer"
 	"go.signoz.io/signoz/pkg/query-service/app/logs"
 	"go.signoz.io/signoz/pkg/query-service/app/services"
 	"go.signoz.io/signoz/pkg/query-service/auth"
 	"go.signoz.io/signoz/pkg/query-service/common"
 	"go.signoz.io/signoz/pkg/query-service/constants"
-	"go.signoz.io/signoz/pkg/query-service/dao"
 	chErrors "go.signoz.io/signoz/pkg/query-service/errors"
 	am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
 	"go.signoz.io/signoz/pkg/query-service/interfaces"
@@ -3243,171 +3240,6 @@ func removeUnderscoreDuplicateFields(fields []model.LogField) []model.LogField {
 	return updatedFields
 }
 
-// GetDashboardsInfo returns analytics data for dashboards
-func (r *ClickHouseReader) GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) {
-	dashboardsInfo := model.DashboardsInfo{}
-	// fetch dashboards from dashboard db
-	query := "SELECT data FROM dashboards"
-	var dashboardsData []dashboards.Dashboard
-	err := r.localDB.Select(&dashboardsData, query)
-	if err != nil {
-		zap.L().Error("Error in processing sql query", zap.Error(err))
-		return &dashboardsInfo, err
-	}
-	totalDashboardsWithPanelAndName := 0
-	var dashboardNames []string
-	count := 0
-	logChQueriesCount := 0
-	for _, dashboard := range dashboardsData {
-		if isDashboardWithPanelAndName(dashboard.Data) {
-			totalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName + 1
-		}
-		dashboardName := extractDashboardName(dashboard.Data)
-		if dashboardName != "" {
-			dashboardNames = append(dashboardNames, dashboardName)
-		}
-		dashboardInfo := countPanelsInDashboard(dashboard.Data)
-		dashboardsInfo.LogsBasedPanels += dashboardInfo.LogsBasedPanels
-		dashboardsInfo.TracesBasedPanels += dashboardInfo.TracesBasedPanels
-		dashboardsInfo.MetricBasedPanels += dashboardsInfo.MetricBasedPanels
-		if isDashboardWithTSV2(dashboard.Data) {
-			count = count + 1
-		}
-		if isDashboardWithLogsClickhouseQuery(dashboard.Data) {
-			logChQueriesCount = logChQueriesCount + 1
-		}
-	}
-
-	dashboardsInfo.DashboardNames = dashboardNames
-	dashboardsInfo.TotalDashboards = len(dashboardsData)
-	dashboardsInfo.TotalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName
-	dashboardsInfo.QueriesWithTSV2 = count
-	dashboardsInfo.DashboardsWithLogsChQuery = logChQueriesCount
-	return &dashboardsInfo, nil
-}
-
-func isDashboardWithTSV2(data map[string]interface{}) bool {
-	jsonData, err := json.Marshal(data)
-	if err != nil {
-		return false
-	}
-	return strings.Contains(string(jsonData), "time_series_v2")
-}
-
-func isDashboardWithLogsClickhouseQuery(data map[string]interface{}) bool {
-	jsonData, err := json.Marshal(data)
-	if err != nil {
-		return false
-	}
-	result := strings.Contains(string(jsonData), "signoz_logs.distributed_logs") ||
-		strings.Contains(string(jsonData), "signoz_logs.logs")
-	return result
-}
-
-func isDashboardWithPanelAndName(data map[string]interface{}) bool {
-	isDashboardName := false
-	isDashboardWithPanelAndName := false
-	if data != nil && data["title"] != nil && data["widgets"] != nil {
-		title, ok := data["title"].(string)
-		if ok && title != "Sample Title" {
-			isDashboardName = true
-		}
-		widgets, ok := data["widgets"]
-		if ok && isDashboardName {
-			data, ok := widgets.([]interface{})
-			if ok && len(data) > 0 {
-				isDashboardWithPanelAndName = true
-			}
-		}
-	}
-
-	return isDashboardWithPanelAndName
-}
-
-func extractDashboardName(data map[string]interface{}) string {
-
-	if data != nil && data["title"] != nil {
-		title, ok := data["title"].(string)
-		if ok {
-			return title
-		}
-	}
-
-	return ""
-}
-
-func countPanelsInDashboard(data map[string]interface{}) model.DashboardsInfo {
-	var logsPanelCount, tracesPanelCount, metricsPanelCount int
-	// totalPanels := 0
-	if data != nil && data["widgets"] != nil {
-		widgets, ok := data["widgets"]
-		if ok {
-			data, ok := widgets.([]interface{})
-			if ok {
-				for _, widget := range data {
-					sData, ok := widget.(map[string]interface{})
-					if ok && sData["query"] != nil {
-						// totalPanels++
-						query, ok := sData["query"].(map[string]interface{})
-						if ok && query["queryType"] == "builder" && query["builder"] != nil {
-							builderData, ok := query["builder"].(map[string]interface{})
-							if ok && builderData["queryData"] != nil {
-								builderQueryData, ok := builderData["queryData"].([]interface{})
-								if ok {
-									for _, queryData := range builderQueryData {
-										data, ok := queryData.(map[string]interface{})
-										if ok {
-											if data["dataSource"] == "traces" {
-												tracesPanelCount++
-											} else if data["dataSource"] == "metrics" {
-												metricsPanelCount++
-											} else if data["dataSource"] == "logs" {
-												logsPanelCount++
-											}
-										}
-									}
-								}
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-	return model.DashboardsInfo{
-		LogsBasedPanels:   logsPanelCount,
-		TracesBasedPanels: tracesPanelCount,
-		MetricBasedPanels: metricsPanelCount,
-	}
-}
-
-func (r *ClickHouseReader) GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error) {
-	savedViewsInfo := model.SavedViewsInfo{}
-	savedViews, err := explorer.GetViews()
-	if err != nil {
-		zap.S().Debug("Error in fetching saved views info: ", err)
-		return &savedViewsInfo, err
-	}
-	savedViewsInfo.TotalSavedViews = len(savedViews)
-	for _, view := range savedViews {
-		if view.SourcePage == "traces" {
-			savedViewsInfo.TracesSavedViews += 1
-		} else if view.SourcePage == "logs" {
-			savedViewsInfo.LogsSavedViews += 1
-		}
-	}
-	return &savedViewsInfo, nil
-}
-
-func (r *ClickHouseReader) GetUsers(ctx context.Context) ([]model.UserPayload, error) {
-
-	users, apiErr := dao.DB().GetUsers(ctx)
-	if apiErr != nil {
-		return nil, apiErr.Err
-	}
-	return users, nil
-}
-
 func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) {
 	// response will contain top level fields from the otel log model
 	response := model.GetFieldsResponse{

pkg/query-service/app/dashboards/model.go

@@ -15,6 +15,8 @@ import (
 	"go.signoz.io/signoz/pkg/query-service/common"
 	"go.signoz.io/signoz/pkg/query-service/interfaces"
 	"go.signoz.io/signoz/pkg/query-service/model"
+
+	"go.signoz.io/signoz/pkg/query-service/telemetry"
 	"go.uber.org/zap"
 )
 
@@ -149,6 +151,8 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) {
 		return nil, fmt.Errorf("error in adding column locked to dashboards table: %s", err.Error())
 	}
 
+	telemetry.GetInstance().SetDashboardsInfoCallback(GetDashboardsInfo)
+
 	return db, nil
 }
 
@@ -434,3 +438,126 @@ func getIdDifference(existingIds []string, newIds []string) []string {
 
 	return difference
 }
+
+// GetDashboardsInfo returns analytics data for dashboards
+func GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) {
+	dashboardsInfo := model.DashboardsInfo{}
+	// fetch dashboards from dashboard db
+	query := "SELECT data FROM dashboards"
+	var dashboardsData []Dashboard
+	err := db.Select(&dashboardsData, query)
+	if err != nil {
+		zap.L().Error("Error in processing sql query", zap.Error(err))
+		return &dashboardsInfo, err
+	}
+	totalDashboardsWithPanelAndName := 0
+	var dashboardNames []string
+	count := 0
+	for _, dashboard := range dashboardsData {
+		if isDashboardWithPanelAndName(dashboard.Data) {
+			totalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName + 1
+		}
+		dashboardName := extractDashboardName(dashboard.Data)
+		if dashboardName != "" {
+			dashboardNames = append(dashboardNames, dashboardName)
+		}
+		dashboardInfo := countPanelsInDashboard(dashboard.Data)
+		dashboardsInfo.LogsBasedPanels += dashboardInfo.LogsBasedPanels
+		dashboardsInfo.TracesBasedPanels += dashboardInfo.TracesBasedPanels
+		dashboardsInfo.MetricBasedPanels += dashboardsInfo.MetricBasedPanels
+		if isDashboardWithTSV2(dashboard.Data) {
+			count = count + 1
+		}
+	}
+
+	dashboardsInfo.DashboardNames = dashboardNames
+	dashboardsInfo.TotalDashboards = len(dashboardsData)
+	dashboardsInfo.TotalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName
+	dashboardsInfo.QueriesWithTSV2 = count
+	return &dashboardsInfo, nil
+}
+
+func isDashboardWithTSV2(data map[string]interface{}) bool {
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return false
+	}
+	return strings.Contains(string(jsonData), "time_series_v2")
+}
+
+func isDashboardWithPanelAndName(data map[string]interface{}) bool {
+	isDashboardName := false
+	isDashboardWithPanelAndName := false
+	if data != nil && data["title"] != nil && data["widgets"] != nil {
+		title, ok := data["title"].(string)
+		if ok && title != "Sample Title" {
+			isDashboardName = true
+		}
+		widgets, ok := data["widgets"]
+		if ok && isDashboardName {
+			data, ok := widgets.([]interface{})
+			if ok && len(data) > 0 {
+				isDashboardWithPanelAndName = true
+			}
+		}
+	}
+
+	return isDashboardWithPanelAndName
+}
+
+func extractDashboardName(data map[string]interface{}) string {
+
+	if data != nil && data["title"] != nil {
+		title, ok := data["title"].(string)
+		if ok {
+			return title
+		}
+	}
+
+	return ""
+}
+
+func countPanelsInDashboard(data map[string]interface{}) model.DashboardsInfo {
+	var logsPanelCount, tracesPanelCount, metricsPanelCount int
+	// totalPanels := 0
+	if data != nil && data["widgets"] != nil {
+		widgets, ok := data["widgets"]
+		if ok {
+			data, ok := widgets.([]interface{})
+			if ok {
+				for _, widget := range data {
+					sData, ok := widget.(map[string]interface{})
+					if ok && sData["query"] != nil {
+						// totalPanels++
+						query, ok := sData["query"].(map[string]interface{})
+						if ok && query["queryType"] == "builder" && query["builder"] != nil {
+							builderData, ok := query["builder"].(map[string]interface{})
+							if ok && builderData["queryData"] != nil {
+								builderQueryData, ok := builderData["queryData"].([]interface{})
+								if ok {
+									for _, queryData := range builderQueryData {
+										data, ok := queryData.(map[string]interface{})
+										if ok {
+											if data["dataSource"] == "traces" {
+												tracesPanelCount++
+											} else if data["dataSource"] == "metrics" {
+												metricsPanelCount++
+											} else if data["dataSource"] == "logs" {
+												logsPanelCount++
+											}
+										}
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+	return model.DashboardsInfo{
+		LogsBasedPanels:   logsPanelCount,
+		TracesBasedPanels: tracesPanelCount,
+		MetricBasedPanels: metricsPanelCount,
+	}
+}

pkg/query-service/app/explorer/db.go

@@ -10,7 +10,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
 	"go.signoz.io/signoz/pkg/query-service/auth"
+	"go.signoz.io/signoz/pkg/query-service/model"
 	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+	"go.signoz.io/signoz/pkg/query-service/telemetry"
+	"go.uber.org/zap"
 )
 
 var db *sqlx.DB
@@ -57,6 +60,8 @@ func InitWithDSN(dataSourceName string) (*sqlx.DB, error) {
 		return nil, fmt.Errorf("error in creating saved views table: %s", err.Error())
 	}
 
+	telemetry.GetInstance().SetSavedViewsInfoCallback(GetSavedViewsInfo)
+
 	return db, nil
 }
 
@@ -228,3 +233,21 @@ func DeleteView(uuid_ string) error {
 	}
 	return nil
 }
+
+func GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error) {
+	savedViewsInfo := model.SavedViewsInfo{}
+	savedViews, err := GetViews()
+	if err != nil {
+		zap.S().Debug("Error in fetching saved views info: ", err)
+		return &savedViewsInfo, err
+	}
+	savedViewsInfo.TotalSavedViews = len(savedViews)
+	for _, view := range savedViews {
+		if view.SourcePage == "traces" {
+			savedViewsInfo.TracesSavedViews += 1
+		} else if view.SourcePage == "logs" {
+			savedViewsInfo.LogsSavedViews += 1
+		}
+	}
+	return &savedViewsInfo, nil
+}

pkg/query-service/dao/sqlite/connection.go

@@ -105,6 +105,7 @@ func InitDB(dataSourceName string) (*ModelDaoSqlite, error) {
 
 	telemetry.GetInstance().SetUserCountCallback(mds.GetUserCount)
 	telemetry.GetInstance().SetUserRoleCallback(mds.GetUserRole)
+	telemetry.GetInstance().SetGetUsersCallback(mds.GetUsers)
 
 	return mds, nil
 }

pkg/query-service/interfaces/interface.go

@@ -71,8 +71,6 @@ type Reader interface {
 	LiveTailLogsV3(ctx context.Context, query string, timestampStart uint64, idStart string, client *model.LogsLiveTailClient)
 	LiveTailLogsV4(ctx context.Context, query string, timestampStart uint64, idStart string, client *model.LogsLiveTailClientV2)
 
-	GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error)
-	GetSavedViewsInfo(ctx context.Context) (*model.SavedViewsInfo, error)
 	GetTotalSpans(ctx context.Context) (uint64, error)
 	GetTotalLogs(ctx context.Context) (uint64, error)
 	GetTotalSamples(ctx context.Context) (uint64, error)
@@ -91,7 +89,6 @@ type Reader interface {
 	GetLogAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error)
 	GetLogAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error)
 	GetLogAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error)
-	GetUsers(ctx context.Context) ([]model.UserPayload, error)
 	GetQBFilterSuggestionsForLogs(
 		ctx context.Context,
 		req *v3.QBFilterSuggestionsRequest,

pkg/query-service/telemetry/telemetry.go

@@ -178,9 +178,12 @@ type Telemetry struct {
 	patTokenUser  bool
 	mutex         sync.RWMutex
 
-	alertsInfoCallback func(ctx context.Context) (*model.AlertsInfo, error)
-	userCountCallback  func(ctx context.Context) (int, error)
-	userRoleCallback   func(ctx context.Context, groupId string) (string, error)
+	alertsInfoCallback     func(ctx context.Context) (*model.AlertsInfo, error)
+	userCountCallback      func(ctx context.Context) (int, error)
+	userRoleCallback       func(ctx context.Context, groupId string) (string, error)
+	getUsersCallback       func(ctx context.Context) ([]model.UserPayload, *model.ApiError)
+	dashboardsInfoCallback func(ctx context.Context) (*model.DashboardsInfo, error)
+	savedViewsInfoCallback func(ctx context.Context) (*model.SavedViewsInfo, error)
 }
 
 func (a *Telemetry) SetAlertsInfoCallback(callback func(ctx context.Context) (*model.AlertsInfo, error)) {
@@ -195,6 +198,18 @@ func (a *Telemetry) SetUserRoleCallback(callback func(ctx context.Context, group
 	a.userRoleCallback = callback
 }
 
+func (a *Telemetry) SetGetUsersCallback(callback func(ctx context.Context) ([]model.UserPayload, *model.ApiError)) {
+	a.getUsersCallback = callback
+}
+
+func (a *Telemetry) SetSavedViewsInfoCallback(callback func(ctx context.Context) (*model.SavedViewsInfo, error)) {
+	a.savedViewsInfoCallback = callback
+}
+
+func (a *Telemetry) SetDashboardsInfoCallback(callback func(ctx context.Context) (*model.DashboardsInfo, error)) {
+	a.dashboardsInfoCallback = callback
+}
+
 func createTelemetry() {
 	// Do not do anything in CI (not even resolving the outbound IP address)
 	if testing.Testing() {
@@ -296,7 +311,7 @@ func createTelemetry() {
 			data[key] = value
 		}
 
-		users, apiErr := telemetry.reader.GetUsers(ctx)
+		users, apiErr := telemetry.getUsersCallback(ctx)
 		if apiErr == nil {
 			for _, user := range users {
 				if user.Email == DEFAULT_CLOUD_EMAIL {
@@ -308,7 +323,7 @@ func createTelemetry() {
 
 		alertsInfo, err := telemetry.alertsInfoCallback(ctx)
 		if err == nil {
-			dashboardsInfo, err := telemetry.reader.GetDashboardsInfo(ctx)
+			dashboardsInfo, err := telemetry.dashboardsInfoCallback(ctx)
 			if err == nil {
 				channels, err := telemetry.reader.GetChannels()
 				if err == nil {
@@ -328,7 +343,7 @@ func createTelemetry() {
 							alertsInfo.MSTeamsChannels++
 						}
 					}
-					savedViewsInfo, err := telemetry.reader.GetSavedViewsInfo(ctx)
+					savedViewsInfo, err := telemetry.savedViewsInfoCallback(ctx)
 					if err == nil {
 						dashboardsAlertsData := map[string]interface{}{
 							"totalDashboards":                 dashboardsInfo.TotalDashboards,