From c5d7d9d13455810b8c00b7ac7c6f8d1a17b62895 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 7 Mar 2023 00:26:25 +0530 Subject: [PATCH] feat: ability to save and retrieve the explorer queries (#2284) --- ee/query-service/app/server.go | 3 + pkg/query-service/app/explorer/db.go | 155 ++++++++++++++++++++++++ pkg/query-service/app/http_handler.go | 83 +++++++++++++ pkg/query-service/app/server.go | 2 + pkg/query-service/model/v3/v3.go | 162 ++++++++++++++++++++++++++ 5 files changed, 405 insertions(+) create mode 100644 pkg/query-service/app/explorer/db.go diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 899cc08880..af4a90f885 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -27,7 +27,9 @@ import ( baseapp "go.signoz.io/signoz/pkg/query-service/app" "go.signoz.io/signoz/pkg/query-service/app/dashboards" + "go.signoz.io/signoz/pkg/query-service/app/explorer" baseauth "go.signoz.io/signoz/pkg/query-service/auth" + "go.signoz.io/signoz/pkg/query-service/constants" baseconst "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/healthcheck" basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager" @@ -84,6 +86,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH) + explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH) if err != nil { return nil, err diff --git a/pkg/query-service/app/explorer/db.go b/pkg/query-service/app/explorer/db.go new file mode 100644 index 0000000000..d7b1193508 --- /dev/null +++ b/pkg/query-service/app/explorer/db.go @@ -0,0 +1,155 @@ +package explorer + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +var db *sqlx.DB + +type ExplorerQuery struct { + UUID string `json:"uuid" db:"uuid"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + SourcePage string `json:"source_page" db:"source_page"` + // 0 - false, 1 - true + IsView int8 `json:"is_view" db:"is_view"` + Data string `json:"data" db:"data"` + ExtraData string `json:"extra_data" db:"extra_data"` +} + +// InitWithDSN sets up setting up the connection pool global variable. +func InitWithDSN(dataSourceName string) (*sqlx.DB, error) { + var err error + + db, err = sqlx.Open("sqlite3", dataSourceName) + if err != nil { + return nil, err + } + + tableSchema := `CREATE TABLE IF NOT EXISTS explorer_queries ( + uuid TEXT PRIMARY KEY, + created_at datetime NOT NULL, + updated_at datetime NOT NULL, + source_page TEXT NOT NULL, + is_view INTEGER NOT NULL, + data TEXT NOT NULL, + extra_data TEXT + );` + + _, err = db.Exec(tableSchema) + if err != nil { + return nil, fmt.Errorf("Error in creating explorer queries table: %s", err.Error()) + } + + return db, nil +} + +func InitWithDB(sqlDB *sqlx.DB) { + db = sqlDB +} + +func GetQueries() ([]*v3.ExplorerQuery, error) { + var queries []ExplorerQuery + err := db.Select(&queries, "SELECT * FROM explorer_queries") + if err != nil { + return nil, fmt.Errorf("Error in getting explorer queries: %s", err.Error()) + } + + var explorerQueries []*v3.ExplorerQuery + for _, query := range queries { + var compositeQuery v3.CompositeQuery + err = json.Unmarshal([]byte(query.Data), &compositeQuery) + if err != nil { + return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error()) + } + explorerQueries = append(explorerQueries, &v3.ExplorerQuery{ + UUID: query.UUID, + SourcePage: query.SourcePage, + CompositeQuery: &compositeQuery, + IsView: query.IsView, + ExtraData: query.ExtraData, + }) + } + return explorerQueries, nil +} + +func CreateQuery(query v3.ExplorerQuery) (string, error) { + data, err := json.Marshal(query.CompositeQuery) + if err != nil { + return "", fmt.Errorf("Error in marshalling explorer query data: %s", err.Error()) + } + + uuid_ := query.UUID + + if uuid_ == "" { + uuid_ = uuid.New().String() + } + createdAt := time.Now() + updatedAt := time.Now() + + _, err = db.Exec( + "INSERT INTO explorer_queries (uuid, created_at, updated_at, source_page, is_view, data, extra_data) VALUES (?, ?, ?, ?, ?, ?, ?)", + uuid_, + createdAt, + updatedAt, + query.SourcePage, + query.IsView, + data, + query.ExtraData, + ) + if err != nil { + return "", fmt.Errorf("Error in creating explorer query: %s", err.Error()) + } + return uuid_, nil +} + +func GetQuery(uuid_ string) (*v3.ExplorerQuery, error) { + var query ExplorerQuery + err := db.Get(&query, "SELECT * FROM explorer_queries WHERE uuid = ?", uuid_) + if err != nil { + return nil, fmt.Errorf("Error in getting explorer query: %s", err.Error()) + } + + var compositeQuery v3.CompositeQuery + err = json.Unmarshal([]byte(query.Data), &compositeQuery) + if err != nil { + return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error()) + } + return &v3.ExplorerQuery{ + UUID: query.UUID, + SourcePage: query.SourcePage, + CompositeQuery: &compositeQuery, + IsView: query.IsView, + ExtraData: query.ExtraData, + }, nil +} + +func UpdateQuery(uuid_ string, query v3.ExplorerQuery) error { + data, err := json.Marshal(query.CompositeQuery) + if err != nil { + return fmt.Errorf("Error in marshalling explorer query data: %s", err.Error()) + } + + updatedAt := time.Now() + + _, err = db.Exec("UPDATE explorer_queries SET updated_at = ?, source_page = ?, is_view = ?, data = ?, extra_data = ? WHERE uuid = ?", + updatedAt, query.SourcePage, query.IsView, data, query.ExtraData, uuid_) + if err != nil { + return fmt.Errorf("Error in updating explorer query: %s", err.Error()) + } + return nil +} + +func DeleteQuery(uuid_ string) error { + _, err := db.Exec("DELETE FROM explorer_queries WHERE uuid = ?", uuid_) + if err != nil { + return fmt.Errorf("Error in deleting explorer query: %s", err.Error()) + } + return nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index d924c498a1..eb8409d239 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -19,6 +19,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/prometheus/prometheus/promql" "go.signoz.io/signoz/pkg/query-service/app/dashboards" + "go.signoz.io/signoz/pkg/query-service/app/explorer" "go.signoz.io/signoz/pkg/query-service/app/logs" "go.signoz.io/signoz/pkg/query-service/app/metrics" "go.signoz.io/signoz/pkg/query-service/app/parser" @@ -280,6 +281,12 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) { router.HandleFunc("/api/v1/variables/query", am.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet) router.HandleFunc("/api/v2/variables/query", am.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/explorer/queries", am.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/queries", am.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/feedback", am.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost) // router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet) router.HandleFunc("/api/v1/services", am.ViewAccess(aH.getServices)).Methods(http.MethodPost) @@ -2253,6 +2260,82 @@ func (aH *APIHandler) logAggregate(w http.ResponseWriter, r *http.Request) { aH.WriteJSON(w, r, res) } +func (aH *APIHandler) getExplorerQueries(w http.ResponseWriter, r *http.Request) { + queries, err := explorer.GetQueries() + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + aH.Respond(w, queries) +} + +func (aH *APIHandler) createExplorerQueries(w http.ResponseWriter, r *http.Request) { + var query v3.ExplorerQuery + err := json.NewDecoder(r.Body).Decode(&query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + // validate the query + if err := query.Validate(); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + uuid, err := explorer.CreateQuery(query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + + aH.Respond(w, uuid) +} + +func (aH *APIHandler) getExplorerQuery(w http.ResponseWriter, r *http.Request) { + queryID := mux.Vars(r)["queryId"] + query, err := explorer.GetQuery(queryID) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + + aH.Respond(w, query) +} + +func (aH *APIHandler) updateExplorerQuery(w http.ResponseWriter, r *http.Request) { + queryID := mux.Vars(r)["queryId"] + var query v3.ExplorerQuery + err := json.NewDecoder(r.Body).Decode(&query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + // validate the query + if err := query.Validate(); err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + err = explorer.UpdateQuery(queryID, query) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + + aH.Respond(w, query) +} + +func (aH *APIHandler) deleteExplorerQuery(w http.ResponseWriter, r *http.Request) { + + queryID := mux.Vars(r)["queryId"] + err := explorer.DeleteQuery(queryID) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + + aH.Respond(w, nil) +} + func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r *http.Request) { var response *v3.AggregateAttributeResponse req, err := parseAggregateAttributeRequest(r) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 7981cb1b05..8088c2294e 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -20,6 +20,7 @@ import ( "github.com/soheilhy/cmux" "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader" "go.signoz.io/signoz/pkg/query-service/app/dashboards" + "go.signoz.io/signoz/pkg/query-service/app/explorer" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/dao" @@ -77,6 +78,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } localDB, err := dashboards.InitDB(constants.RELATIONAL_DATASOURCE_PATH) + explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH) if err != nil { return nil, err diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 5bdf575a8e..e097fdd6f1 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -3,6 +3,8 @@ package v3 import ( "fmt" "time" + + "github.com/google/uuid" ) type DataSource string @@ -95,6 +97,19 @@ func (a AggregateOperator) Validate() error { } } +// RequireAttribute returns true if the aggregate operator requires an attribute +// to be specified. +func (a AggregateOperator) RequireAttribute() bool { + switch a { + case AggregateOperatorNoOp, + AggregateOpeatorCount, + AggregateOperatorCountDistinct: + return false + default: + return true + } +} + type ReduceToOperator string const ( @@ -245,11 +260,35 @@ type PromQuery struct { Disabled bool `json:"disabled"` } +func (p *PromQuery) Validate() error { + if p == nil { + return nil + } + + if p.Query == "" { + return fmt.Errorf("query is empty") + } + + return nil +} + type ClickHouseQuery struct { Query string `json:"query"` Disabled bool `json:"disabled"` } +func (c *ClickHouseQuery) Validate() error { + if c == nil { + return nil + } + + if c.Query == "" { + return fmt.Errorf("query is empty") + } + + return nil +} + type CompositeQuery struct { BuilderQueries map[string]*BuilderQuery `json:"builderQueries,omitempty"` ClickHouseQueries map[string]*ClickHouseQuery `json:"chQueries,omitempty"` @@ -258,6 +297,50 @@ type CompositeQuery struct { QueryType QueryType `json:"queryType"` } +func (c *CompositeQuery) Validate() error { + if c == nil { + return nil + } + + if c.BuilderQueries == nil && c.ClickHouseQueries == nil && c.PromQueries == nil { + return fmt.Errorf("composite query must contain at least one query") + } + + if c.BuilderQueries != nil { + for name, query := range c.BuilderQueries { + if err := query.Validate(); err != nil { + return fmt.Errorf("builder query %s is invalid: %w", name, err) + } + } + } + + if c.ClickHouseQueries != nil { + for name, query := range c.ClickHouseQueries { + if err := query.Validate(); err != nil { + return fmt.Errorf("clickhouse query %s is invalid: %w", name, err) + } + } + } + + if c.PromQueries != nil { + for name, query := range c.PromQueries { + if err := query.Validate(); err != nil { + return fmt.Errorf("prom query %s is invalid: %w", name, err) + } + } + } + + if err := c.PanelType.Validate(); err != nil { + return fmt.Errorf("panel type is invalid: %w", err) + } + + if err := c.QueryType.Validate(); err != nil { + return fmt.Errorf("query type is invalid: %w", err) + } + + return nil +} + type BuilderQuery struct { QueryName string `json:"queryName"` DataSource DataSource `json:"dataSource"` @@ -276,11 +359,61 @@ type BuilderQuery struct { SelectColumns []string `json:"selectColumns,omitempty"` } +func (b *BuilderQuery) Validate() error { + if b == nil { + return nil + } + if b.QueryName == "" { + return fmt.Errorf("query name is required") + } + + // if expression is same as query name, it's a simple builder query and not a formula + // formula involves more than one data source, aggregate operator, etc. + if b.QueryName == b.Expression { + if err := b.DataSource.Validate(); err != nil { + return fmt.Errorf("data source is invalid: %w", err) + } + if err := b.AggregateOperator.Validate(); err != nil { + return fmt.Errorf("aggregate operator is invalid: %w", err) + } + if b.AggregateAttribute == "" && b.AggregateOperator.RequireAttribute() { + return fmt.Errorf("aggregate attribute is required") + } + } + + if b.Filters != nil { + if err := b.Filters.Validate(); err != nil { + return fmt.Errorf("filters are invalid: %w", err) + } + } + if b.GroupBy != nil { + for _, groupBy := range b.GroupBy { + if groupBy == "" { + return fmt.Errorf("group by cannot be empty") + } + } + } + if b.Expression == "" { + return fmt.Errorf("expression is required") + } + return nil +} + type FilterSet struct { Operator string `json:"op,omitempty"` Items []FilterItem `json:"items"` } +func (f *FilterSet) Validate() error { + if f == nil { + return nil + } + if f.Operator != "" && f.Operator != "AND" && f.Operator != "OR" { + return fmt.Errorf("operator must be AND or OR") + } + return nil +} + type FilterItem struct { Key string `json:"key"` Value interface{} `json:"value"` @@ -323,3 +456,32 @@ type Point struct { Timestamp int64 `json:"timestamp"` Value float64 `json:"value"` } + +// ExploreQuery is a query for the explore page +// It is a composite query with a source page name +// The source page name is used to identify the page that initiated the query +// The source page could be "traces", "logs", "metrics" or "dashboards", "alerts" etc. +type ExplorerQuery struct { + UUID string `json:"uuid,omitempty"` + SourcePage string `json:"sourcePage"` + CompositeQuery *CompositeQuery `json:"compositeQuery"` + // ExtraData is JSON encoded data used by frontend to store additional data + ExtraData string `json:"extraData"` + // 0 - false, 1 - true; this is int8 because sqlite doesn't support bool + IsView int8 `json:"isView"` +} + +func (eq *ExplorerQuery) Validate() error { + if eq.IsView != 0 && eq.IsView != 1 { + return fmt.Errorf("isView must be 0 or 1") + } + + if eq.CompositeQuery == nil { + return fmt.Errorf("composite query is required") + } + + if eq.UUID == "" { + eq.UUID = uuid.New().String() + } + return eq.CompositeQuery.Validate() +}