signoz/pkg/query-service/app/dashboards/model.go

package dashboards

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"regexp"
	"slices"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/gosimple/slug"
	"github.com/jmoiron/sqlx"
	"go.signoz.io/signoz/pkg/query-service/common"
	"go.signoz.io/signoz/pkg/query-service/interfaces"
	"go.signoz.io/signoz/pkg/query-service/model"

	"go.signoz.io/signoz/pkg/query-service/telemetry"
	"go.uber.org/zap"
)

// This time the global variable is unexported.
var db *sqlx.DB

// User for mapping job,instance from grafana
var (
	instanceEQRE = regexp.MustCompile("instance(?s)=(?s)\\\"{{.instance}}\\\"")
	nodeEQRE     = regexp.MustCompile("instance(?s)=(?s)\\\"{{.node}}\\\"")
	jobEQRE      = regexp.MustCompile("job(?s)=(?s)\\\"{{.job}}\\\"")
	instanceRERE = regexp.MustCompile("instance(?s)=~(?s)\\\"{{.instance}}\\\"")
	nodeRERE     = regexp.MustCompile("instance(?s)=~(?s)\\\"{{.node}}\\\"")
	jobRERE      = regexp.MustCompile("job(?s)=~(?s)\\\"{{.job}}\\\"")
)

// InitDB sets up setting up the connection pool global variable.
func InitDB(dataSourceName string) (*sqlx.DB, error) {
	var err error

	db, err = sqlx.Open("sqlite3", dataSourceName)
	if err != nil {
		return nil, err
	}

	table_schema := `CREATE TABLE IF NOT EXISTS dashboards (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		uuid TEXT NOT NULL UNIQUE,
		created_at datetime NOT NULL,
		updated_at datetime NOT NULL,
		data TEXT NOT NULL
	);`

	_, err = db.Exec(table_schema)
	if err != nil {
		return nil, fmt.Errorf("error in creating dashboard table: %s", err.Error())
	}

	table_schema = `CREATE TABLE IF NOT EXISTS rules (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		updated_at datetime NOT NULL,
		deleted INTEGER DEFAULT 0,
		data TEXT NOT NULL
	);`

	_, err = db.Exec(table_schema)
	if err != nil {
		return nil, fmt.Errorf("error in creating rules table: %s", err.Error())
	}

	table_schema = `CREATE TABLE IF NOT EXISTS notification_channels (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		created_at datetime NOT NULL,
		updated_at datetime NOT NULL,
		name TEXT NOT NULL UNIQUE,
		type TEXT NOT NULL,
		deleted INTEGER DEFAULT 0,
		data TEXT NOT NULL
	);`

	_, err = db.Exec(table_schema)
	if err != nil {
		return nil, fmt.Errorf("error in creating notification_channles table: %s", err.Error())
	}

	tableSchema := `CREATE TABLE IF NOT EXISTS planned_maintenance (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		name TEXT NOT NULL,
		description TEXT,
		alert_ids TEXT,
		schedule TEXT NOT NULL,
		created_at datetime NOT NULL,
		created_by TEXT NOT NULL,
		updated_at datetime NOT NULL,
		updated_by TEXT NOT NULL
	);`
	_, err = db.Exec(tableSchema)
	if err != nil {
		return nil, fmt.Errorf("error in creating planned_maintenance table: %s", err.Error())
	}

	table_schema = `CREATE TABLE IF NOT EXISTS ttl_status (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		transaction_id TEXT NOT NULL,
		created_at datetime NOT NULL,
		updated_at datetime NOT NULL,
		table_name TEXT NOT NULL,
		ttl INTEGER DEFAULT 0,
		cold_storage_ttl INTEGER DEFAULT 0,
		status TEXT NOT NULL
	);`

	_, err = db.Exec(table_schema)
	if err != nil {
		return nil, fmt.Errorf("error in creating ttl_status table: %s", err.Error())
	}

	// sqlite does not support "IF NOT EXISTS"
	createdAt := `ALTER TABLE rules ADD COLUMN created_at datetime;`
	_, err = db.Exec(createdAt)
	if err != nil && !strings.Contains(err.Error(), "duplicate column name") {
		return nil, fmt.Errorf("error in adding column created_at to rules table: %s", err.Error())
	}

	createdBy := `ALTER TABLE rules ADD COLUMN created_by TEXT;`
	_, err = db.Exec(createdBy)
	if err != nil && !strings.Contains(err.Error(), "duplicate column name") {
		return nil, fmt.Errorf("error in adding column created_by to rules table: %s", err.Error())
	}

	updatedBy := `ALTER TABLE rules ADD COLUMN updated_by TEXT;`
	_, err = db.Exec(updatedBy)
	if err != nil && !strings.Contains(err.Error(), "duplicate column name") {
		return nil, fmt.Errorf("error in adding column updated_by to rules table: %s", err.Error())
	}

	createdBy = `ALTER TABLE dashboards ADD COLUMN created_by TEXT;`
	_, err = db.Exec(createdBy)
	if err != nil && !strings.Contains(err.Error(), "duplicate column name") {
		return nil, fmt.Errorf("error in adding column created_by to dashboards table: %s", err.Error())
	}

	updatedBy = `ALTER TABLE dashboards ADD COLUMN updated_by TEXT;`
	_, err = db.Exec(updatedBy)
	if err != nil && !strings.Contains(err.Error(), "duplicate column name") {
		return nil, fmt.Errorf("error in adding column updated_by to dashboards table: %s", err.Error())
	}

	locked := `ALTER TABLE dashboards ADD COLUMN locked INTEGER DEFAULT 0;`
	_, err = db.Exec(locked)
	if err != nil && !strings.Contains(err.Error(), "duplicate column name") {
		return nil, fmt.Errorf("error in adding column locked to dashboards table: %s", err.Error())
	}

	telemetry.GetInstance().SetDashboardsInfoCallback(GetDashboardsInfo)

	return db, nil
}

type Dashboard struct {
	Id        int       `json:"id" db:"id"`
	Uuid      string    `json:"uuid" db:"uuid"`
	Slug      string    `json:"-" db:"-"`
	CreatedAt time.Time `json:"created_at" db:"created_at"`
	CreateBy  *string   `json:"created_by" db:"created_by"`
	UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
	UpdateBy  *string   `json:"updated_by" db:"updated_by"`
	Title     string    `json:"-" db:"-"`
	Data      Data      `json:"data" db:"data"`
	Locked    *int      `json:"isLocked" db:"locked"`
}

type Data map[string]interface{}

// func (c *Data) Value() (driver.Value, error) {
// 	if c != nil {
// 		b, err := json.Marshal(c)
// 		if err != nil {
// 			return nil, err
// 		}
// 		return string(b), nil
// 	}
// 	return nil, nil
// }

func (c *Data) Scan(src interface{}) error {
	var data []byte
	if b, ok := src.([]byte); ok {
		data = b
	} else if s, ok := src.(string); ok {
		data = []byte(s)
	}
	return json.Unmarshal(data, c)
}

// CreateDashboard creates a new dashboard
func CreateDashboard(ctx context.Context, data map[string]interface{}, fm interfaces.FeatureLookup) (*Dashboard, *model.ApiError) {
	dash := &Dashboard{
		Data: data,
	}
	var userEmail string
	if user := common.GetUserFromContext(ctx); user != nil {
		userEmail = user.Email
	}
	dash.CreatedAt = time.Now()
	dash.CreateBy = &userEmail
	dash.UpdatedAt = time.Now()
	dash.UpdateBy = &userEmail
	dash.UpdateSlug()
	dash.Uuid = uuid.New().String()
	if data["uuid"] != nil {
		dash.Uuid = data["uuid"].(string)
	}

	mapData, err := json.Marshal(dash.Data)
	if err != nil {
		zap.L().Error("Error in marshalling data field in dashboard: ", zap.Any("dashboard", dash), zap.Error(err))
		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
	}

	result, err := db.Exec("INSERT INTO dashboards (uuid, created_at, created_by, updated_at, updated_by, data) VALUES ($1, $2, $3, $4, $5, $6)",
		dash.Uuid, dash.CreatedAt, userEmail, dash.UpdatedAt, userEmail, mapData)

	if err != nil {
		zap.L().Error("Error in inserting dashboard data: ", zap.Any("dashboard", dash), zap.Error(err))
		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
	}
	lastInsertId, err := result.LastInsertId()
	if err != nil {
		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
	}
	dash.Id = int(lastInsertId)

	return dash, nil
}

func GetDashboards(ctx context.Context) ([]Dashboard, *model.ApiError) {

	dashboards := []Dashboard{}
	query := `SELECT * FROM dashboards`

	err := db.Select(&dashboards, query)
	if err != nil {
		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
	}

	return dashboards, nil
}

func DeleteDashboard(ctx context.Context, uuid string, fm interfaces.FeatureLookup) *model.ApiError {

	dashboard, dErr := GetDashboard(ctx, uuid)
	if dErr != nil {
		zap.L().Error("Error in getting dashboard: ", zap.String("uuid", uuid), zap.Any("error", dErr))
		return dErr
	}

	if user := common.GetUserFromContext(ctx); user != nil {
		if dashboard.Locked != nil && *dashboard.Locked == 1 {
			return model.BadRequest(fmt.Errorf("dashboard is locked, please unlock the dashboard to be able to delete it"))
		}
	}

	query := `DELETE FROM dashboards WHERE uuid=?`

	result, err := db.Exec(query, uuid)
	if err != nil {
		return &model.ApiError{Typ: model.ErrorExec, Err: err}
	}

	affectedRows, err := result.RowsAffected()
	if err != nil {
		return &model.ApiError{Typ: model.ErrorExec, Err: err}
	}
	if affectedRows == 0 {
		return &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
	}

	return nil
}

func GetDashboard(ctx context.Context, uuid string) (*Dashboard, *model.ApiError) {

	dashboard := Dashboard{}
	query := `SELECT * FROM dashboards WHERE uuid=?`

	err := db.Get(&dashboard, query, uuid)
	if err != nil {
		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
	}

	if dashboard.Data["title"] == "Ingestion" && dashboard.Data["description"] != nil {
		dashboard.Data["description"] = "This dashboard is deprecated. Please use the new Ingestion V2 dashboard. " + dashboard.Data["description"].(string)
	}

	return &dashboard, nil
}

func UpdateDashboard(ctx context.Context, uuid string, data map[string]interface{}, fm interfaces.FeatureLookup) (*Dashboard, *model.ApiError) {

	mapData, err := json.Marshal(data)
	if err != nil {
		zap.L().Error("Error in marshalling data field in dashboard: ", zap.Any("data", data), zap.Error(err))
		return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
	}

	dashboard, apiErr := GetDashboard(ctx, uuid)
	if apiErr != nil {
		return nil, apiErr
	}

	var userEmail string
	if user := common.GetUserFromContext(ctx); user != nil {
		userEmail = user.Email
		if dashboard.Locked != nil && *dashboard.Locked == 1 {
			return nil, model.BadRequest(fmt.Errorf("dashboard is locked, please unlock the dashboard to be able to edit it"))
		}
	}

	// if the total count of panels has reduced by more than 1,
	// return error
	existingIds := getWidgetIds(dashboard.Data)
	newIds := getWidgetIds(data)

	differenceIds := getIdDifference(existingIds, newIds)

	if len(differenceIds) > 1 {
		return nil, model.BadRequest(fmt.Errorf("deleting more than one panel is not supported"))
	}

	dashboard.UpdatedAt = time.Now()
	dashboard.UpdateBy = &userEmail
	dashboard.Data = data

	_, err = db.Exec("UPDATE dashboards SET updated_at=$1, updated_by=$2, data=$3 WHERE uuid=$4;",
		dashboard.UpdatedAt, userEmail, mapData, dashboard.Uuid)

	if err != nil {
		zap.L().Error("Error in inserting dashboard data", zap.Any("data", data), zap.Error(err))
		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
	}
	return dashboard, nil
}

func LockUnlockDashboard(ctx context.Context, uuid string, lock bool) *model.ApiError {
	var query string
	if lock {
		query = `UPDATE dashboards SET locked=1 WHERE uuid=?;`
	} else {
		query = `UPDATE dashboards SET locked=0 WHERE uuid=?;`
	}

	_, err := db.Exec(query, uuid)

	if err != nil {
		zap.L().Error("Error in updating dashboard", zap.String("uuid", uuid), zap.Error(err))
		return &model.ApiError{Typ: model.ErrorExec, Err: err}
	}

	return nil
}

// UpdateSlug updates the slug
func (d *Dashboard) UpdateSlug() {
	var title string

	if val, ok := d.Data["title"]; ok {
		title = val.(string)
	}

	d.Slug = SlugifyTitle(title)
}

func IsPostDataSane(data *map[string]interface{}) error {
	val, ok := (*data)["title"]
	if !ok || val == nil {
		return fmt.Errorf("title not found in post data")
	}

	return nil
}

func SlugifyTitle(title string) string {
	s := slug.Make(strings.ToLower(title))
	if s == "" {
		// If the dashboard name is only characters outside of the
		// sluggable characters, the slug creation will return an
		// empty string which will mess up URLs. This failsafe picks
		// that up and creates the slug as a base64 identifier instead.
		s = base64.RawURLEncoding.EncodeToString([]byte(title))
		if slug.MaxLength != 0 && len(s) > slug.MaxLength {
			s = s[:slug.MaxLength]
		}
	}
	return s
}

func getWidgetIds(data map[string]interface{}) []string {
	widgetIds := []string{}
	if data != nil && data["widgets"] != nil {
		widgets, ok := data["widgets"]
		if ok {
			data, ok := widgets.([]interface{})
			if ok {
				for _, widget := range data {
					sData, ok := widget.(map[string]interface{})
					if ok && sData["query"] != nil && sData["id"] != nil {
						id, ok := sData["id"].(string)

						if ok {
							widgetIds = append(widgetIds, id)
						}

					}
				}
			}
		}
	}
	return widgetIds
}

func getIdDifference(existingIds []string, newIds []string) []string {
	// Convert newIds array to a map for faster lookups
	newIdsMap := make(map[string]bool)
	for _, id := range newIds {
		newIdsMap[id] = true
	}

	// Initialize a map to keep track of elements in the difference array
	differenceMap := make(map[string]bool)

	// Initialize the difference array
	difference := []string{}

	// Iterate through existingIds
	for _, id := range existingIds {
		// If the id is not found in newIds, and it's not already in the difference array
		if _, found := newIdsMap[id]; !found && !differenceMap[id] {
			difference = append(difference, id)
			differenceMap[id] = true // Mark the id as seen in the difference array
		}
	}

	return difference
}

// GetDashboardsInfo returns analytics data for dashboards
func GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) {
	dashboardsInfo := model.DashboardsInfo{}
	// fetch dashboards from dashboard db
	query := "SELECT data FROM dashboards"
	var dashboardsData []Dashboard
	err := db.Select(&dashboardsData, query)
	if err != nil {
		zap.L().Error("Error in processing sql query", zap.Error(err))
		return &dashboardsInfo, err
	}
	totalDashboardsWithPanelAndName := 0
	var dashboardNames []string
	count := 0
	queriesWithTagAttrs := 0
	for _, dashboard := range dashboardsData {
		if isDashboardWithPanelAndName(dashboard.Data) {
			totalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName + 1
		}
		dashboardName := extractDashboardName(dashboard.Data)
		if dashboardName != "" {
			dashboardNames = append(dashboardNames, dashboardName)
		}
		dashboardInfo := countPanelsInDashboard(dashboard.Data)
		dashboardsInfo.LogsBasedPanels += dashboardInfo.LogsBasedPanels
		dashboardsInfo.TracesBasedPanels += dashboardInfo.TracesBasedPanels
		dashboardsInfo.MetricBasedPanels += dashboardsInfo.MetricBasedPanels
		dashboardsInfo.LogsPanelsWithAttrContainsOp += dashboardInfo.LogsPanelsWithAttrContainsOp
		dashboardsInfo.DashboardsWithLogsChQuery += dashboardInfo.DashboardsWithLogsChQuery
		dashboardsInfo.DashboardsWithTraceChQuery += dashboardInfo.DashboardsWithTraceChQuery
		if isDashboardWithTSV2(dashboard.Data) {
			count = count + 1
		}

		if isDashboardWithTagAttrs(dashboard.Data) {
			queriesWithTagAttrs += 1
		}

		if dashboardInfo.DashboardsWithTraceChQuery > 0 {
			dashboardsInfo.DashboardNamesWithTraceChQuery = append(dashboardsInfo.DashboardNamesWithTraceChQuery, dashboardName)
		}

		// check if dashboard is a has a log operator with contains
	}

	dashboardsInfo.DashboardNames = dashboardNames
	dashboardsInfo.TotalDashboards = len(dashboardsData)
	dashboardsInfo.TotalDashboardsWithPanelAndName = totalDashboardsWithPanelAndName
	dashboardsInfo.QueriesWithTSV2 = count
	dashboardsInfo.QueriesWithTagAttrs = queriesWithTagAttrs
	return &dashboardsInfo, nil
}

func isDashboardWithTSV2(data map[string]interface{}) bool {
	jsonData, err := json.Marshal(data)
	if err != nil {
		return false
	}
	return strings.Contains(string(jsonData), "time_series_v2")
}

func isDashboardWithTagAttrs(data map[string]interface{}) bool {
	jsonData, err := json.Marshal(data)
	if err != nil {
		return false
	}
	return strings.Contains(string(jsonData), "span_attributes") ||
		strings.Contains(string(jsonData), "tag_attributes")
}

func isDashboardWithLogsClickhouseQuery(data map[string]interface{}) bool {
	jsonData, err := json.Marshal(data)
	if err != nil {
		return false
	}
	result := strings.Contains(string(jsonData), "signoz_logs.distributed_logs") ||
		strings.Contains(string(jsonData), "signoz_logs.logs")
	return result
}

func isDashboardWithTracesClickhouseQuery(data map[string]interface{}) bool {
	jsonData, err := json.Marshal(data)
	if err != nil {
		return false
	}

	// also check if the query is actually active
	str := string(jsonData)
	result := strings.Contains(str, "signoz_traces.distributed_signoz_index_v2") ||
		strings.Contains(str, "signoz_traces.distributed_signoz_spans") ||
		strings.Contains(str, "signoz_traces.distributed_signoz_error_index_v2")
	return result
}

func isDashboardWithPanelAndName(data map[string]interface{}) bool {
	isDashboardName := false
	isDashboardWithPanelAndName := false
	if data != nil && data["title"] != nil && data["widgets"] != nil {
		title, ok := data["title"].(string)
		if ok && title != "Sample Title" {
			isDashboardName = true
		}
		widgets, ok := data["widgets"]
		if ok && isDashboardName {
			data, ok := widgets.([]interface{})
			if ok && len(data) > 0 {
				isDashboardWithPanelAndName = true
			}
		}
	}

	return isDashboardWithPanelAndName
}

func extractDashboardName(data map[string]interface{}) string {

	if data != nil && data["title"] != nil {
		title, ok := data["title"].(string)
		if ok {
			return title
		}
	}

	return ""
}

func checkLogPanelAttrContains(data map[string]interface{}) int {
	var logsPanelsWithAttrContains int
	filters, ok := data["filters"].(map[string]interface{})
	if ok && filters["items"] != nil {
		items, ok := filters["items"].([]interface{})
		if ok {
			for _, item := range items {
				itemMap, ok := item.(map[string]interface{})
				if ok {
					opStr, ok := itemMap["op"].(string)
					if ok {
						if slices.Contains([]string{"contains", "ncontains", "like", "nlike"}, opStr) {
							// check if it's not body
							key, ok := itemMap["key"].(map[string]string)
							if ok && key["key"] != "body" {
								logsPanelsWithAttrContains++
							}
						}
					}
				}
			}
		}
	}
	return logsPanelsWithAttrContains
}

func countPanelsInDashboard(inputData map[string]interface{}) model.DashboardsInfo {
	var logsPanelCount, tracesPanelCount, metricsPanelCount, logsPanelsWithAttrContains int
	traceChQueryCount := 0
	logChQueryCount := 0

	// totalPanels := 0
	if inputData != nil && inputData["widgets"] != nil {
		widgets, ok := inputData["widgets"]
		if ok {
			data, ok := widgets.([]interface{})
			if ok {
				for _, widget := range data {
					sData, ok := widget.(map[string]interface{})
					if ok && sData["query"] != nil {
						// totalPanels++
						query, ok := sData["query"].(map[string]interface{})
						if ok && query["queryType"] == "builder" && query["builder"] != nil {
							builderData, ok := query["builder"].(map[string]interface{})
							if ok && builderData["queryData"] != nil {
								builderQueryData, ok := builderData["queryData"].([]interface{})
								if ok {
									for _, queryData := range builderQueryData {
										data, ok := queryData.(map[string]interface{})
										if ok {
											if data["dataSource"] == "traces" {
												tracesPanelCount++
											} else if data["dataSource"] == "metrics" {
												metricsPanelCount++
											} else if data["dataSource"] == "logs" {
												logsPanelCount++
												logsPanelsWithAttrContains += checkLogPanelAttrContains(data)
											}
										}
									}
								}
							}
						} else if ok && query["queryType"] == "clickhouse_sql" && query["clickhouse_sql"] != nil {
							if isDashboardWithLogsClickhouseQuery(inputData) {
								logChQueryCount = 1
							}
							if isDashboardWithTracesClickhouseQuery(inputData) {
								traceChQueryCount = 1
							}
						}
					}
				}
			}
		}
	}

	return model.DashboardsInfo{
		LogsBasedPanels:   logsPanelCount,
		TracesBasedPanels: tracesPanelCount,
		MetricBasedPanels: metricsPanelCount,

		DashboardsWithLogsChQuery:    logChQueryCount,
		DashboardsWithTraceChQuery:   traceChQueryCount,
		LogsPanelsWithAttrContainsOp: logsPanelsWithAttrContains,
	}
}