GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-14 01:25:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: ability to save and retrieve the explorer queries (#2284)

Browse Source
This commit is contained in:
Srikanth Chekuri 2023-03-07 00:26:25 +05:30 committed by GitHub
parent 6defa0ac8b
commit c5d7d9d134
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 405 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ee/query-service/app/server.go
View File

@ -27,7 +27,9 @@ import (
baseapp "go.signoz.io/signoz/pkg/query-service/app"
"go.signoz.io/signoz/pkg/query-service/app/dashboards"
"go.signoz.io/signoz/pkg/query-service/app/explorer"
baseauth "go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
baseconst "go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/healthcheck"
basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
@ -84,6 +86,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
}
localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)
explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH)
if err != nil {
return nil, err

155
pkg/query-service/app/explorer/db.go Normal file
View File

@ -0,0 +1,155 @@
package explorer
import (
"encoding/json"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
var db *sqlx.DB
type ExplorerQuery struct {
UUID string `json:"uuid" db:"uuid"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
SourcePage string `json:"source_page" db:"source_page"`
// 0 - false, 1 - true
IsView int8 `json:"is_view" db:"is_view"`
Data string `json:"data" db:"data"`
ExtraData string `json:"extra_data" db:"extra_data"`
}
// InitWithDSN sets up setting up the connection pool global variable.
func InitWithDSN(dataSourceName string) (*sqlx.DB, error) {
var err error
db, err = sqlx.Open("sqlite3", dataSourceName)
if err != nil {
return nil, err
}
tableSchema := `CREATE TABLE IF NOT EXISTS explorer_queries (
uuid TEXT PRIMARY KEY,
created_at datetime NOT NULL,
updated_at datetime NOT NULL,
source_page TEXT NOT NULL,
is_view INTEGER NOT NULL,
data TEXT NOT NULL,
extra_data TEXT
);`
_, err = db.Exec(tableSchema)
if err != nil {
return nil, fmt.Errorf("Error in creating explorer queries table: %s", err.Error())
}
return db, nil
}
func InitWithDB(sqlDB *sqlx.DB) {
db = sqlDB
}
func GetQueries() ([]*v3.ExplorerQuery, error) {
var queries []ExplorerQuery
err := db.Select(&queries, "SELECT * FROM explorer_queries")
if err != nil {
return nil, fmt.Errorf("Error in getting explorer queries: %s", err.Error())
}
var explorerQueries []*v3.ExplorerQuery
for _, query := range queries {
var compositeQuery v3.CompositeQuery
err = json.Unmarshal([]byte(query.Data), &compositeQuery)
if err != nil {
return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error())
}
explorerQueries = append(explorerQueries, &v3.ExplorerQuery{
UUID: query.UUID,
SourcePage: query.SourcePage,
CompositeQuery: &compositeQuery,
IsView: query.IsView,
ExtraData: query.ExtraData,
})
}
return explorerQueries, nil
}
func CreateQuery(query v3.ExplorerQuery) (string, error) {
data, err := json.Marshal(query.CompositeQuery)
if err != nil {
return "", fmt.Errorf("Error in marshalling explorer query data: %s", err.Error())
}
uuid_ := query.UUID
if uuid_ == "" {
uuid_ = uuid.New().String()
}
createdAt := time.Now()
updatedAt := time.Now()
_, err = db.Exec(
"INSERT INTO explorer_queries (uuid, created_at, updated_at, source_page, is_view, data, extra_data) VALUES (?, ?, ?, ?, ?, ?, ?)",
uuid_,
createdAt,
updatedAt,
query.SourcePage,
query.IsView,
data,
query.ExtraData,
)
if err != nil {
return "", fmt.Errorf("Error in creating explorer query: %s", err.Error())
}
return uuid_, nil
}
func GetQuery(uuid_ string) (*v3.ExplorerQuery, error) {
var query ExplorerQuery
err := db.Get(&query, "SELECT * FROM explorer_queries WHERE uuid = ?", uuid_)
if err != nil {
return nil, fmt.Errorf("Error in getting explorer query: %s", err.Error())
}
var compositeQuery v3.CompositeQuery
err = json.Unmarshal([]byte(query.Data), &compositeQuery)
if err != nil {
return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error())
}
return &v3.ExplorerQuery{
UUID: query.UUID,
SourcePage: query.SourcePage,
CompositeQuery: &compositeQuery,
IsView: query.IsView,
ExtraData: query.ExtraData,
}, nil
}
func UpdateQuery(uuid_ string, query v3.ExplorerQuery) error {
data, err := json.Marshal(query.CompositeQuery)
if err != nil {
return fmt.Errorf("Error in marshalling explorer query data: %s", err.Error())
}
updatedAt := time.Now()
_, err = db.Exec("UPDATE explorer_queries SET updated_at = ?, source_page = ?, is_view = ?, data = ?, extra_data = ? WHERE uuid = ?",
updatedAt, query.SourcePage, query.IsView, data, query.ExtraData, uuid_)
if err != nil {
return fmt.Errorf("Error in updating explorer query: %s", err.Error())
}
return nil
}
func DeleteQuery(uuid_ string) error {
_, err := db.Exec("DELETE FROM explorer_queries WHERE uuid = ?", uuid_)
if err != nil {
return fmt.Errorf("Error in deleting explorer query: %s", err.Error())
}
return nil
}

83
pkg/query-service/app/http_handler.go
View File

@ -19,6 +19,7 @@ import (
_ "github.com/mattn/go-sqlite3"
"github.com/prometheus/prometheus/promql"
"go.signoz.io/signoz/pkg/query-service/app/dashboards"
"go.signoz.io/signoz/pkg/query-service/app/explorer"
"go.signoz.io/signoz/pkg/query-service/app/logs"
"go.signoz.io/signoz/pkg/query-service/app/metrics"
"go.signoz.io/signoz/pkg/query-service/app/parser"
@ -280,6 +281,12 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) {
router.HandleFunc("/api/v1/variables/query", am.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet)
router.HandleFunc("/api/v2/variables/query", am.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost)
router.HandleFunc("/api/v1/explorer/queries", am.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/explorer/queries", am.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost)
router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut)
router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete)
router.HandleFunc("/api/v1/feedback", am.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost)
// router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet)
router.HandleFunc("/api/v1/services", am.ViewAccess(aH.getServices)).Methods(http.MethodPost)
@ -2253,6 +2260,82 @@ func (aH *APIHandler) logAggregate(w http.ResponseWriter, r *http.Request) {
aH.WriteJSON(w, r, res)
}
func (aH *APIHandler) getExplorerQueries(w http.ResponseWriter, r *http.Request) {
queries, err := explorer.GetQueries()
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
return
}
aH.Respond(w, queries)
}
func (aH *APIHandler) createExplorerQueries(w http.ResponseWriter, r *http.Request) {
var query v3.ExplorerQuery
err := json.NewDecoder(r.Body).Decode(&query)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return
}
// validate the query
if err := query.Validate(); err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return
}
uuid, err := explorer.CreateQuery(query)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
return
}
aH.Respond(w, uuid)
}
func (aH *APIHandler) getExplorerQuery(w http.ResponseWriter, r *http.Request) {
queryID := mux.Vars(r)["queryId"]
query, err := explorer.GetQuery(queryID)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
return
}
aH.Respond(w, query)
}
func (aH *APIHandler) updateExplorerQuery(w http.ResponseWriter, r *http.Request) {
queryID := mux.Vars(r)["queryId"]
var query v3.ExplorerQuery
err := json.NewDecoder(r.Body).Decode(&query)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return
}
// validate the query
if err := query.Validate(); err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
return
}
err = explorer.UpdateQuery(queryID, query)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
return
}
aH.Respond(w, query)
}
func (aH *APIHandler) deleteExplorerQuery(w http.ResponseWriter, r *http.Request) {
queryID := mux.Vars(r)["queryId"]
err := explorer.DeleteQuery(queryID)
if err != nil {
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
return
}
aH.Respond(w, nil)
}
func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r *http.Request) {
var response *v3.AggregateAttributeResponse
req, err := parseAggregateAttributeRequest(r)

2
pkg/query-service/app/server.go
View File

@ -20,6 +20,7 @@ import (
"github.com/soheilhy/cmux"
"go.signoz.io/signoz/pkg/query-service/app/clickhouseReader"
"go.signoz.io/signoz/pkg/query-service/app/dashboards"
"go.signoz.io/signoz/pkg/query-service/app/explorer"
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/dao"
@ -77,6 +78,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
}
localDB, err := dashboards.InitDB(constants.RELATIONAL_DATASOURCE_PATH)
explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH)
if err != nil {
return nil, err

162
pkg/query-service/model/v3/v3.go
View File

@ -3,6 +3,8 @@ package v3
import (
"fmt"
"time"
"github.com/google/uuid"
)
type DataSource string
@ -95,6 +97,19 @@ func (a AggregateOperator) Validate() error {
}
}
// RequireAttribute returns true if the aggregate operator requires an attribute
// to be specified.
func (a AggregateOperator) RequireAttribute() bool {
switch a {
case AggregateOperatorNoOp,
AggregateOpeatorCount,
AggregateOperatorCountDistinct:
return false
default:
return true
}
}
type ReduceToOperator string
const (
@ -245,11 +260,35 @@ type PromQuery struct {
Disabled bool `json:"disabled"`
}
func (p *PromQuery) Validate() error {
if p == nil {
return nil
}
if p.Query == "" {
return fmt.Errorf("query is empty")
}
return nil
}
type ClickHouseQuery struct {
Query string `json:"query"`
Disabled bool `json:"disabled"`
}
func (c *ClickHouseQuery) Validate() error {
if c == nil {
return nil
}
if c.Query == "" {
return fmt.Errorf("query is empty")
}
return nil
}
type CompositeQuery struct {
BuilderQueries map[string]*BuilderQuery `json:"builderQueries,omitempty"`
ClickHouseQueries map[string]*ClickHouseQuery `json:"chQueries,omitempty"`
@ -258,6 +297,50 @@ type CompositeQuery struct {
QueryType QueryType `json:"queryType"`
}
func (c *CompositeQuery) Validate() error {
if c == nil {
return nil
}
if c.BuilderQueries == nil && c.ClickHouseQueries == nil && c.PromQueries == nil {
return fmt.Errorf("composite query must contain at least one query")
}
if c.BuilderQueries != nil {
for name, query := range c.BuilderQueries {
if err := query.Validate(); err != nil {
return fmt.Errorf("builder query %s is invalid: %w", name, err)
}
}
}
if c.ClickHouseQueries != nil {
for name, query := range c.ClickHouseQueries {
if err := query.Validate(); err != nil {
return fmt.Errorf("clickhouse query %s is invalid: %w", name, err)
}
}
}
if c.PromQueries != nil {
for name, query := range c.PromQueries {
if err := query.Validate(); err != nil {
return fmt.Errorf("prom query %s is invalid: %w", name, err)
}
}
}
if err := c.PanelType.Validate(); err != nil {
return fmt.Errorf("panel type is invalid: %w", err)
}
if err := c.QueryType.Validate(); err != nil {
return fmt.Errorf("query type is invalid: %w", err)
}
return nil
}
type BuilderQuery struct {
QueryName string `json:"queryName"`
DataSource DataSource `json:"dataSource"`
@ -276,11 +359,61 @@ type BuilderQuery struct {
SelectColumns []string `json:"selectColumns,omitempty"`
}
func (b *BuilderQuery) Validate() error {
if b == nil {
return nil
}
if b.QueryName == "" {
return fmt.Errorf("query name is required")
}
// if expression is same as query name, it's a simple builder query and not a formula
// formula involves more than one data source, aggregate operator, etc.
if b.QueryName == b.Expression {
if err := b.DataSource.Validate(); err != nil {
return fmt.Errorf("data source is invalid: %w", err)
}
if err := b.AggregateOperator.Validate(); err != nil {
return fmt.Errorf("aggregate operator is invalid: %w", err)
}
if b.AggregateAttribute == "" && b.AggregateOperator.RequireAttribute() {
return fmt.Errorf("aggregate attribute is required")
}
}
if b.Filters != nil {
if err := b.Filters.Validate(); err != nil {
return fmt.Errorf("filters are invalid: %w", err)
}
}
if b.GroupBy != nil {
for _, groupBy := range b.GroupBy {
if groupBy == "" {
return fmt.Errorf("group by cannot be empty")
}
}
}
if b.Expression == "" {
return fmt.Errorf("expression is required")
}
return nil
}
type FilterSet struct {
Operator string `json:"op,omitempty"`
Items []FilterItem `json:"items"`
}
func (f *FilterSet) Validate() error {
if f == nil {
return nil
}
if f.Operator != "" && f.Operator != "AND" && f.Operator != "OR" {
return fmt.Errorf("operator must be AND or OR")
}
return nil
}
type FilterItem struct {
Key string `json:"key"`
Value interface{} `json:"value"`
@ -323,3 +456,32 @@ type Point struct {
Timestamp int64 `json:"timestamp"`
Value float64 `json:"value"`
}
// ExploreQuery is a query for the explore page
// It is a composite query with a source page name
// The source page name is used to identify the page that initiated the query
// The source page could be "traces", "logs", "metrics" or "dashboards", "alerts" etc.
type ExplorerQuery struct {
UUID string `json:"uuid,omitempty"`
SourcePage string `json:"sourcePage"`
CompositeQuery *CompositeQuery `json:"compositeQuery"`
// ExtraData is JSON encoded data used by frontend to store additional data
ExtraData string `json:"extraData"`
// 0 - false, 1 - true; this is int8 because sqlite doesn't support bool
IsView int8 `json:"isView"`
}
func (eq *ExplorerQuery) Validate() error {
if eq.IsView != 0 && eq.IsView != 1 {
return fmt.Errorf("isView must be 0 or 1")
}
if eq.CompositeQuery == nil {
return fmt.Errorf("composite query is required")
}
if eq.UUID == "" {
eq.UUID = uuid.New().String()
}
return eq.CompositeQuery.Validate()
}