feat: ability to save and retrieve the explorer queries (#2284)

ee/query-service/app/server.go

@@ -27,7 +27,9 @@ import (
 
 	baseapp "go.signoz.io/signoz/pkg/query-service/app"
 	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
+	"go.signoz.io/signoz/pkg/query-service/app/explorer"
 	baseauth "go.signoz.io/signoz/pkg/query-service/auth"
+	"go.signoz.io/signoz/pkg/query-service/constants"
 	baseconst "go.signoz.io/signoz/pkg/query-service/constants"
 	"go.signoz.io/signoz/pkg/query-service/healthcheck"
 	basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
@@ -84,6 +86,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	}
 
 	localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)
+	explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH)
 
 	if err != nil {
 		return nil, err

pkg/query-service/app/explorer/db.go

@@ -0,0 +1,155 @@
+package explorer
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/jmoiron/sqlx"
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+)
+
+var db *sqlx.DB
+
+type ExplorerQuery struct {
+	UUID       string    `json:"uuid" db:"uuid"`
+	CreatedAt  time.Time `json:"created_at" db:"created_at"`
+	UpdatedAt  time.Time `json:"updated_at" db:"updated_at"`
+	SourcePage string    `json:"source_page" db:"source_page"`
+	// 0 - false, 1 - true
+	IsView    int8   `json:"is_view" db:"is_view"`
+	Data      string `json:"data" db:"data"`
+	ExtraData string `json:"extra_data" db:"extra_data"`
+}
+
+// InitWithDSN sets up setting up the connection pool global variable.
+func InitWithDSN(dataSourceName string) (*sqlx.DB, error) {
+	var err error
+
+	db, err = sqlx.Open("sqlite3", dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+
+	tableSchema := `CREATE TABLE IF NOT EXISTS explorer_queries (
+		uuid TEXT PRIMARY KEY,
+		created_at datetime NOT NULL,
+		updated_at datetime NOT NULL,
+		source_page TEXT NOT NULL,
+		is_view INTEGER NOT NULL,
+		data TEXT NOT NULL,
+		extra_data TEXT
+	);`
+
+	_, err = db.Exec(tableSchema)
+	if err != nil {
+		return nil, fmt.Errorf("Error in creating explorer queries table: %s", err.Error())
+	}
+
+	return db, nil
+}
+
+func InitWithDB(sqlDB *sqlx.DB) {
+	db = sqlDB
+}
+
+func GetQueries() ([]*v3.ExplorerQuery, error) {
+	var queries []ExplorerQuery
+	err := db.Select(&queries, "SELECT * FROM explorer_queries")
+	if err != nil {
+		return nil, fmt.Errorf("Error in getting explorer queries: %s", err.Error())
+	}
+
+	var explorerQueries []*v3.ExplorerQuery
+	for _, query := range queries {
+		var compositeQuery v3.CompositeQuery
+		err = json.Unmarshal([]byte(query.Data), &compositeQuery)
+		if err != nil {
+			return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error())
+		}
+		explorerQueries = append(explorerQueries, &v3.ExplorerQuery{
+			UUID:           query.UUID,
+			SourcePage:     query.SourcePage,
+			CompositeQuery: &compositeQuery,
+			IsView:         query.IsView,
+			ExtraData:      query.ExtraData,
+		})
+	}
+	return explorerQueries, nil
+}
+
+func CreateQuery(query v3.ExplorerQuery) (string, error) {
+	data, err := json.Marshal(query.CompositeQuery)
+	if err != nil {
+		return "", fmt.Errorf("Error in marshalling explorer query data: %s", err.Error())
+	}
+
+	uuid_ := query.UUID
+
+	if uuid_ == "" {
+		uuid_ = uuid.New().String()
+	}
+	createdAt := time.Now()
+	updatedAt := time.Now()
+
+	_, err = db.Exec(
+		"INSERT INTO explorer_queries (uuid, created_at, updated_at, source_page, is_view, data, extra_data) VALUES (?, ?, ?, ?, ?, ?, ?)",
+		uuid_,
+		createdAt,
+		updatedAt,
+		query.SourcePage,
+		query.IsView,
+		data,
+		query.ExtraData,
+	)
+	if err != nil {
+		return "", fmt.Errorf("Error in creating explorer query: %s", err.Error())
+	}
+	return uuid_, nil
+}
+
+func GetQuery(uuid_ string) (*v3.ExplorerQuery, error) {
+	var query ExplorerQuery
+	err := db.Get(&query, "SELECT * FROM explorer_queries WHERE uuid = ?", uuid_)
+	if err != nil {
+		return nil, fmt.Errorf("Error in getting explorer query: %s", err.Error())
+	}
+
+	var compositeQuery v3.CompositeQuery
+	err = json.Unmarshal([]byte(query.Data), &compositeQuery)
+	if err != nil {
+		return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error())
+	}
+	return &v3.ExplorerQuery{
+		UUID:           query.UUID,
+		SourcePage:     query.SourcePage,
+		CompositeQuery: &compositeQuery,
+		IsView:         query.IsView,
+		ExtraData:      query.ExtraData,
+	}, nil
+}
+
+func UpdateQuery(uuid_ string, query v3.ExplorerQuery) error {
+	data, err := json.Marshal(query.CompositeQuery)
+	if err != nil {
+		return fmt.Errorf("Error in marshalling explorer query data: %s", err.Error())
+	}
+
+	updatedAt := time.Now()
+
+	_, err = db.Exec("UPDATE explorer_queries SET updated_at = ?, source_page = ?, is_view = ?, data = ?, extra_data = ? WHERE uuid = ?",
+		updatedAt, query.SourcePage, query.IsView, data, query.ExtraData, uuid_)
+	if err != nil {
+		return fmt.Errorf("Error in updating explorer query: %s", err.Error())
+	}
+	return nil
+}
+
+func DeleteQuery(uuid_ string) error {
+	_, err := db.Exec("DELETE FROM explorer_queries WHERE uuid = ?", uuid_)
+	if err != nil {
+		return fmt.Errorf("Error in deleting explorer query: %s", err.Error())
+	}
+	return nil
+}

pkg/query-service/app/http_handler.go

@@ -19,6 +19,7 @@ import (
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/prometheus/prometheus/promql"
 	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
+	"go.signoz.io/signoz/pkg/query-service/app/explorer"
 	"go.signoz.io/signoz/pkg/query-service/app/logs"
 	"go.signoz.io/signoz/pkg/query-service/app/metrics"
 	"go.signoz.io/signoz/pkg/query-service/app/parser"
@@ -280,6 +281,12 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) {
 	router.HandleFunc("/api/v1/variables/query", am.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet)
 	router.HandleFunc("/api/v2/variables/query", am.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost)
 
+	router.HandleFunc("/api/v1/explorer/queries", am.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/explorer/queries", am.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut)
+	router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete)
+
 	router.HandleFunc("/api/v1/feedback", am.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost)
 	// router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet)
 	router.HandleFunc("/api/v1/services", am.ViewAccess(aH.getServices)).Methods(http.MethodPost)
@@ -2253,6 +2260,82 @@ func (aH *APIHandler) logAggregate(w http.ResponseWriter, r *http.Request) {
 	aH.WriteJSON(w, r, res)
 }
 
+func (aH *APIHandler) getExplorerQueries(w http.ResponseWriter, r *http.Request) {
+	queries, err := explorer.GetQueries()
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+	aH.Respond(w, queries)
+}
+
+func (aH *APIHandler) createExplorerQueries(w http.ResponseWriter, r *http.Request) {
+	var query v3.ExplorerQuery
+	err := json.NewDecoder(r.Body).Decode(&query)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+	// validate the query
+	if err := query.Validate(); err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+	uuid, err := explorer.CreateQuery(query)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	aH.Respond(w, uuid)
+}
+
+func (aH *APIHandler) getExplorerQuery(w http.ResponseWriter, r *http.Request) {
+	queryID := mux.Vars(r)["queryId"]
+	query, err := explorer.GetQuery(queryID)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	aH.Respond(w, query)
+}
+
+func (aH *APIHandler) updateExplorerQuery(w http.ResponseWriter, r *http.Request) {
+	queryID := mux.Vars(r)["queryId"]
+	var query v3.ExplorerQuery
+	err := json.NewDecoder(r.Body).Decode(&query)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+	// validate the query
+	if err := query.Validate(); err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
+		return
+	}
+
+	err = explorer.UpdateQuery(queryID, query)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	aH.Respond(w, query)
+}
+
+func (aH *APIHandler) deleteExplorerQuery(w http.ResponseWriter, r *http.Request) {
+
+	queryID := mux.Vars(r)["queryId"]
+	err := explorer.DeleteQuery(queryID)
+	if err != nil {
+		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
+		return
+	}
+
+	aH.Respond(w, nil)
+}
+
 func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r *http.Request) {
 	var response *v3.AggregateAttributeResponse
 	req, err := parseAggregateAttributeRequest(r)

pkg/query-service/app/server.go

@@ -20,6 +20,7 @@ import (
 	"github.com/soheilhy/cmux"
 	"go.signoz.io/signoz/pkg/query-service/app/clickhouseReader"
 	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
+	"go.signoz.io/signoz/pkg/query-service/app/explorer"
 	"go.signoz.io/signoz/pkg/query-service/auth"
 	"go.signoz.io/signoz/pkg/query-service/constants"
 	"go.signoz.io/signoz/pkg/query-service/dao"
@@ -77,6 +78,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	}
 
 	localDB, err := dashboards.InitDB(constants.RELATIONAL_DATASOURCE_PATH)
+	explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH)
 
 	if err != nil {
 		return nil, err

pkg/query-service/model/v3/v3.go

@@ -3,6 +3,8 @@ package v3
 import (
 	"fmt"
 	"time"
+
+	"github.com/google/uuid"
 )
 
 type DataSource string
@@ -95,6 +97,19 @@ func (a AggregateOperator) Validate() error {
 	}
 }
 
+// RequireAttribute returns true if the aggregate operator requires an attribute
+// to be specified.
+func (a AggregateOperator) RequireAttribute() bool {
+	switch a {
+	case AggregateOperatorNoOp,
+		AggregateOpeatorCount,
+		AggregateOperatorCountDistinct:
+		return false
+	default:
+		return true
+	}
+}
+
 type ReduceToOperator string
 
 const (
@@ -245,11 +260,35 @@ type PromQuery struct {
 	Disabled bool   `json:"disabled"`
 }
 
+func (p *PromQuery) Validate() error {
+	if p == nil {
+		return nil
+	}
+
+	if p.Query == "" {
+		return fmt.Errorf("query is empty")
+	}
+
+	return nil
+}
+
 type ClickHouseQuery struct {
 	Query    string `json:"query"`
 	Disabled bool   `json:"disabled"`
 }
 
+func (c *ClickHouseQuery) Validate() error {
+	if c == nil {
+		return nil
+	}
+
+	if c.Query == "" {
+		return fmt.Errorf("query is empty")
+	}
+
+	return nil
+}
+
 type CompositeQuery struct {
 	BuilderQueries    map[string]*BuilderQuery    `json:"builderQueries,omitempty"`
 	ClickHouseQueries map[string]*ClickHouseQuery `json:"chQueries,omitempty"`
@@ -258,6 +297,50 @@ type CompositeQuery struct {
 	QueryType         QueryType                   `json:"queryType"`
 }
 
+func (c *CompositeQuery) Validate() error {
+	if c == nil {
+		return nil
+	}
+
+	if c.BuilderQueries == nil && c.ClickHouseQueries == nil && c.PromQueries == nil {
+		return fmt.Errorf("composite query must contain at least one query")
+	}
+
+	if c.BuilderQueries != nil {
+		for name, query := range c.BuilderQueries {
+			if err := query.Validate(); err != nil {
+				return fmt.Errorf("builder query %s is invalid: %w", name, err)
+			}
+		}
+	}
+
+	if c.ClickHouseQueries != nil {
+		for name, query := range c.ClickHouseQueries {
+			if err := query.Validate(); err != nil {
+				return fmt.Errorf("clickhouse query %s is invalid: %w", name, err)
+			}
+		}
+	}
+
+	if c.PromQueries != nil {
+		for name, query := range c.PromQueries {
+			if err := query.Validate(); err != nil {
+				return fmt.Errorf("prom query %s is invalid: %w", name, err)
+			}
+		}
+	}
+
+	if err := c.PanelType.Validate(); err != nil {
+		return fmt.Errorf("panel type is invalid: %w", err)
+	}
+
+	if err := c.QueryType.Validate(); err != nil {
+		return fmt.Errorf("query type is invalid: %w", err)
+	}
+
+	return nil
+}
+
 type BuilderQuery struct {
 	QueryName          string            `json:"queryName"`
 	DataSource         DataSource        `json:"dataSource"`
@@ -276,11 +359,61 @@ type BuilderQuery struct {
 	SelectColumns      []string          `json:"selectColumns,omitempty"`
 }
 
+func (b *BuilderQuery) Validate() error {
+	if b == nil {
+		return nil
+	}
+	if b.QueryName == "" {
+		return fmt.Errorf("query name is required")
+	}
+
+	// if expression is same as query name, it's a simple builder query and not a formula
+	// formula involves more than one data source, aggregate operator, etc.
+	if b.QueryName == b.Expression {
+		if err := b.DataSource.Validate(); err != nil {
+			return fmt.Errorf("data source is invalid: %w", err)
+		}
+		if err := b.AggregateOperator.Validate(); err != nil {
+			return fmt.Errorf("aggregate operator is invalid: %w", err)
+		}
+		if b.AggregateAttribute == "" && b.AggregateOperator.RequireAttribute() {
+			return fmt.Errorf("aggregate attribute is required")
+		}
+	}
+
+	if b.Filters != nil {
+		if err := b.Filters.Validate(); err != nil {
+			return fmt.Errorf("filters are invalid: %w", err)
+		}
+	}
+	if b.GroupBy != nil {
+		for _, groupBy := range b.GroupBy {
+			if groupBy == "" {
+				return fmt.Errorf("group by cannot be empty")
+			}
+		}
+	}
+	if b.Expression == "" {
+		return fmt.Errorf("expression is required")
+	}
+	return nil
+}
+
 type FilterSet struct {
 	Operator string       `json:"op,omitempty"`
 	Items    []FilterItem `json:"items"`
 }
 
+func (f *FilterSet) Validate() error {
+	if f == nil {
+		return nil
+	}
+	if f.Operator != "" && f.Operator != "AND" && f.Operator != "OR" {
+		return fmt.Errorf("operator must be AND or OR")
+	}
+	return nil
+}
+
 type FilterItem struct {
 	Key      string      `json:"key"`
 	Value    interface{} `json:"value"`
@@ -323,3 +456,32 @@ type Point struct {
 	Timestamp int64   `json:"timestamp"`
 	Value     float64 `json:"value"`
 }
+
+// ExploreQuery is a query for the explore page
+// It is a composite query with a source page name
+// The source page name is used to identify the page that initiated the query
+// The source page could be "traces", "logs", "metrics" or "dashboards", "alerts" etc.
+type ExplorerQuery struct {
+	UUID           string          `json:"uuid,omitempty"`
+	SourcePage     string          `json:"sourcePage"`
+	CompositeQuery *CompositeQuery `json:"compositeQuery"`
+	// ExtraData is JSON encoded data used by frontend to store additional data
+	ExtraData string `json:"extraData"`
+	// 0 - false, 1 - true; this is int8 because sqlite doesn't support bool
+	IsView int8 `json:"isView"`
+}
+
+func (eq *ExplorerQuery) Validate() error {
+	if eq.IsView != 0 && eq.IsView != 1 {
+		return fmt.Errorf("isView must be 0 or 1")
+	}
+
+	if eq.CompositeQuery == nil {
+		return fmt.Errorf("composite query is required")
+	}
+
+	if eq.UUID == "" {
+		eq.UUID = uuid.New().String()
+	}
+	return eq.CompositeQuery.Validate()
+}