diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 1fff220f05..6ff840bac2 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -867,11 +867,15 @@ func (aH *APIHandler) listRules(w http.ResponseWriter, r *http.Request) { func (aH *APIHandler) getDashboards(w http.ResponseWriter, r *http.Request) { allDashboards, err := dashboards.GetDashboards(r.Context()) - if err != nil { RespondError(w, err, nil) return } + + ic := aH.IntegrationsController + installedIntegrationDashboards, err := ic.GetDashboardsForInstalledIntegrations(r.Context()) + allDashboards = append(allDashboards, installedIntegrationDashboards...) + tagsFromReq, ok := r.URL.Query()["tags"] if !ok || len(tagsFromReq) == 0 || tagsFromReq[0] == "" { aH.Respond(w, allDashboards) @@ -1040,8 +1044,19 @@ func (aH *APIHandler) getDashboard(w http.ResponseWriter, r *http.Request) { dashboard, apiError := dashboards.GetDashboard(r.Context(), uuid) if apiError != nil { - RespondError(w, apiError, nil) - return + if apiError.Type() != model.ErrorNotFound { + RespondError(w, apiError, nil) + return + } + + dashboard, apiError = aH.IntegrationsController.GetInstalledIntegrationDashboardById( + r.Context(), uuid, + ) + if apiError != nil { + RespondError(w, apiError, nil) + return + } + } aH.Respond(w, dashboard) diff --git a/pkg/query-service/app/integrations/builtin.go b/pkg/query-service/app/integrations/builtin.go index 905bc39c84..a612e45ed3 100644 --- a/pkg/query-service/app/integrations/builtin.go +++ b/pkg/query-service/app/integrations/builtin.go @@ -127,11 +127,39 @@ func readBuiltInIntegration(dirpath string) ( ) } + err = validateIntegration(integration) + if err != nil { + return nil, fmt.Errorf("invalid integration spec %s: %w", integration.Id, err) + } + integration.Id = "builtin-" + integration.Id return &integration, nil } +func validateIntegration(i IntegrationDetails) error { + // Validate dashboard data + seenDashboardIds := map[string]interface{}{} + for _, dd := range i.Assets.Dashboards { + did, exists := dd["id"] + if !exists { + return fmt.Errorf("id is required. not specified in dashboard titled %v", dd["title"]) + } + dashboardId, ok := did.(string) + if !ok { + return fmt.Errorf("id must be string in dashboard titled %v", dd["title"]) + } + if _, seen := seenDashboardIds[dashboardId]; seen { + return fmt.Errorf("multiple dashboards found with id %s", dashboardId) + } + seenDashboardIds[dashboardId] = nil + } + + // TODO(Raj): Validate all parts of plugged in integrations + + return nil +} + func hydrateFileUris(spec interface{}, basedir string) (interface{}, error) { if specMap, ok := spec.(map[string]interface{}); ok { result := map[string]interface{}{} diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json b/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json index 5b5d93ad9e..5b993cb2ca 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json @@ -1,4 +1,5 @@ { + "id": "mongo-overview", "description": "This dashboard provides a high-level overview of your MongoDB. It includes read/write performance, most-used replicas, collection metrics etc...", "layout": [ { diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json b/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json index 9799ad66e3..944e06b03f 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json @@ -1,4 +1,5 @@ { + "id": "postgres-overview", "description": "This dashboard provides a high-level overview of your PostgreSQL databases. It includes replication, locks, and throughput etc...", "layout": [ { diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json b/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json index 58e964af73..3fd2c255ce 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json @@ -1,4 +1,5 @@ { + "id": "redis-overview", "description": "This dashboard shows the Redis instance overview. It includes latency, hit/miss rate, connections, and memory information.\n", "layout": [ { diff --git a/pkg/query-service/app/integrations/controller.go b/pkg/query-service/app/integrations/controller.go index 1a347a73a5..a45ab3fb04 100644 --- a/pkg/query-service/app/integrations/controller.go +++ b/pkg/query-service/app/integrations/controller.go @@ -6,6 +6,7 @@ import ( "github.com/jmoiron/sqlx" "go.signoz.io/signoz/pkg/query-service/agentConf" + "go.signoz.io/signoz/pkg/query-service/app/dashboards" "go.signoz.io/signoz/pkg/query-service/app/logparsingpipeline" "go.signoz.io/signoz/pkg/query-service/model" ) @@ -114,3 +115,15 @@ func (c *Controller) GetPipelinesForInstalledIntegrations( ) ([]logparsingpipeline.Pipeline, *model.ApiError) { return c.mgr.GetPipelinesForInstalledIntegrations(ctx) } + +func (c *Controller) GetDashboardsForInstalledIntegrations( + ctx context.Context, +) ([]dashboards.Dashboard, *model.ApiError) { + return c.mgr.GetDashboardsForInstalledIntegrations(ctx) +} + +func (c *Controller) GetInstalledIntegrationDashboardById( + ctx context.Context, dashboardUuid string, +) (*dashboards.Dashboard, *model.ApiError) { + return c.mgr.GetInstalledIntegrationDashboardById(ctx, dashboardUuid) +} diff --git a/pkg/query-service/app/integrations/manager.go b/pkg/query-service/app/integrations/manager.go index 37541b0a19..110d370c1b 100644 --- a/pkg/query-service/app/integrations/manager.go +++ b/pkg/query-service/app/integrations/manager.go @@ -33,8 +33,8 @@ type IntegrationSummary struct { } type IntegrationAssets struct { - Logs LogsAssets `json:"logs"` - Dashboards []dashboards.Dashboard `json:"dashboards"` + Logs LogsAssets `json:"logs"` + Dashboards []dashboards.Data `json:"dashboards"` Alerts []rules.PostableRule `json:"alerts"` } @@ -248,6 +248,120 @@ func (m *Manager) UninstallIntegration( return m.installedIntegrationsRepo.delete(ctx, integrationId) } +func (m *Manager) GetPipelinesForInstalledIntegrations( + ctx context.Context, +) ([]logparsingpipeline.Pipeline, *model.ApiError) { + installedIntegrations, apiErr := m.getDetailsForInstalledIntegrations(ctx) + if apiErr != nil { + return nil, apiErr + } + + pipelines := []logparsingpipeline.Pipeline{} + for _, ii := range installedIntegrations { + for _, p := range ii.Assets.Logs.Pipelines { + pp := logparsingpipeline.Pipeline{ + // Alias is used for identifying integration pipelines. Id can't be used for this + // since versioning while saving pipelines requires a new id for each version + // to avoid altering history when pipelines are edited/reordered etc + Alias: AliasForIntegrationPipeline(ii.Id, p.Alias), + Id: uuid.NewString(), + OrderId: p.OrderId, + Enabled: p.Enabled, + Name: p.Name, + Description: &p.Description, + Filter: p.Filter, + Config: p.Config, + } + pipelines = append(pipelines, pp) + } + } + + return pipelines, nil +} + +func (m *Manager) dashboardUuid(integrationId string, dashboardId string) string { + return strings.Join([]string{"integration", integrationId, dashboardId}, "--") +} + +func (m *Manager) parseDashboardUuid(dashboardUuid string) ( + integrationId string, dashboardId string, err *model.ApiError, +) { + parts := strings.SplitN(dashboardUuid, "--", 3) + if len(parts) != 3 || parts[0] != "integration" { + return "", "", model.BadRequest(fmt.Errorf( + "invalid installed integration dashboard id", + )) + } + + return parts[1], parts[2], nil +} + +func (m *Manager) GetInstalledIntegrationDashboardById( + ctx context.Context, + dashboardUuid string, +) (*dashboards.Dashboard, *model.ApiError) { + integrationId, dashboardId, apiErr := m.parseDashboardUuid(dashboardUuid) + if apiErr != nil { + return nil, apiErr + } + + integration, apiErr := m.GetIntegration(ctx, integrationId) + if apiErr != nil { + return nil, apiErr + } + + if integration.Installation == nil { + return nil, model.BadRequest(fmt.Errorf( + "integration with id %s is not installed", integrationId, + )) + } + + for _, dd := range integration.IntegrationDetails.Assets.Dashboards { + if dId, exists := dd["id"]; exists { + if id, ok := dId.(string); ok && id == dashboardId { + isLocked := 1 + return &dashboards.Dashboard{ + Uuid: m.dashboardUuid(integrationId, string(dashboardId)), + Locked: &isLocked, + Data: dd, + }, nil + } + } + } + + return nil, model.NotFoundError(fmt.Errorf( + "integration dashboard with id %s not found", dashboardUuid, + )) +} + +func (m *Manager) GetDashboardsForInstalledIntegrations( + ctx context.Context, +) ([]dashboards.Dashboard, *model.ApiError) { + installedIntegrations, apiErr := m.getDetailsForInstalledIntegrations(ctx) + if apiErr != nil { + return nil, apiErr + } + + result := []dashboards.Dashboard{} + + for _, ii := range installedIntegrations { + for _, dd := range ii.Assets.Dashboards { + if dId, exists := dd["id"]; exists { + if dashboardId, ok := dId.(string); ok { + isLocked := 1 + result = append(result, dashboards.Dashboard{ + Uuid: m.dashboardUuid(ii.IntegrationSummary.Id, dashboardId), + Locked: &isLocked, + Data: dd, + }) + } + } + } + } + + return result, nil +} + // Helpers. func (m *Manager) getIntegrationDetails( ctx context.Context, @@ -297,9 +411,11 @@ func (m *Manager) getInstalledIntegration( return &installation, nil } -func (m *Manager) GetPipelinesForInstalledIntegrations( +func (m *Manager) getDetailsForInstalledIntegrations( ctx context.Context, -) ([]logparsingpipeline.Pipeline, *model.ApiError) { +) ( + map[string]IntegrationDetails, *model.ApiError, +) { installations, apiErr := m.installedIntegrationsRepo.list(ctx) if apiErr != nil { return nil, apiErr @@ -308,30 +424,5 @@ func (m *Manager) GetPipelinesForInstalledIntegrations( installedIds := utils.MapSlice(installations, func(i InstalledIntegration) string { return i.IntegrationId }) - installedIntegrations, apiErr := m.availableIntegrationsRepo.get(ctx, installedIds) - if apiErr != nil { - return nil, apiErr - } - - pipelines := []logparsingpipeline.Pipeline{} - for _, ii := range installedIntegrations { - for _, p := range ii.Assets.Logs.Pipelines { - pp := logparsingpipeline.Pipeline{ - // Alias is used for identifying integration pipelines. Id can't be used for this - // since versioning while saving pipelines requires a new id for each version - // to avoid altering history when pipelines are edited/reordered etc - Alias: AliasForIntegrationPipeline(ii.Id, p.Alias), - Id: uuid.NewString(), - OrderId: p.OrderId, - Enabled: p.Enabled, - Name: p.Name, - Description: &p.Description, - Filter: p.Filter, - Config: p.Config, - } - pipelines = append(pipelines, pp) - } - } - - return pipelines, nil + return m.availableIntegrationsRepo.get(ctx, installedIds) } diff --git a/pkg/query-service/app/integrations/test_utils.go b/pkg/query-service/app/integrations/test_utils.go index d06b2db75c..1ff964b3e6 100644 --- a/pkg/query-service/app/integrations/test_utils.go +++ b/pkg/query-service/app/integrations/test_utils.go @@ -92,7 +92,7 @@ func (t *TestAvailableIntegrationsRepo) list( }, }, }, - Dashboards: []dashboards.Dashboard{}, + Dashboards: []dashboards.Data{}, Alerts: []rules.PostableRule{}, }, ConnectionTests: &IntegrationConnectionTests{ @@ -170,7 +170,7 @@ func (t *TestAvailableIntegrationsRepo) list( }, }, }, - Dashboards: []dashboards.Dashboard{}, + Dashboards: []dashboards.Data{}, Alerts: []rules.PostableRule{}, }, ConnectionTests: &IntegrationConnectionTests{ diff --git a/pkg/query-service/tests/integration/signoz_integrations_test.go b/pkg/query-service/tests/integration/signoz_integrations_test.go index 8c6402f3f1..5294d06081 100644 --- a/pkg/query-service/tests/integration/signoz_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_integrations_test.go @@ -14,6 +14,7 @@ import ( mockhouse "github.com/srikanthccv/ClickHouse-go-mock" "github.com/stretchr/testify/require" "go.signoz.io/signoz/pkg/query-service/app" + "go.signoz.io/signoz/pkg/query-service/app/dashboards" "go.signoz.io/signoz/pkg/query-service/app/integrations" "go.signoz.io/signoz/pkg/query-service/app/logparsingpipeline" "go.signoz.io/signoz/pkg/query-service/auth" @@ -275,6 +276,72 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) { pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines) } +func TestDashboardsForInstalledIntegrationDashboards(t *testing.T) { + require := require.New(t) + + testDB := utils.NewQueryServiceDBForTests(t) + integrationsTB := NewIntegrationsTestBed(t, testDB) + + availableIntegrationsResp := integrationsTB.GetAvailableIntegrationsFromQS() + availableIntegrations := availableIntegrationsResp.Integrations + require.Greater( + len(availableIntegrations), 0, + "some integrations should come bundled with SigNoz", + ) + + dashboards := integrationsTB.GetDashboardsFromQS() + require.Equal( + 0, len(dashboards), + "There should be no dashboards at the start", + ) + + // Find an available integration that contains dashboards + var testAvailableIntegration *integrations.IntegrationsListItem + for _, ai := range availableIntegrations { + details := integrationsTB.GetIntegrationDetailsFromQS(ai.Id) + require.NotNil(details) + if len(details.Assets.Dashboards) > 0 { + testAvailableIntegration = &ai + break + } + } + require.NotNil(testAvailableIntegration) + + // Installing an integration should make its dashboards appear in the dashboard list + require.False(testAvailableIntegration.IsInstalled) + integrationsTB.RequestQSToInstallIntegration( + testAvailableIntegration.Id, map[string]interface{}{}, + ) + + testIntegration := integrationsTB.GetIntegrationDetailsFromQS(testAvailableIntegration.Id) + require.NotNil(testIntegration.Installation) + testIntegrationDashboards := testIntegration.Assets.Dashboards + require.Greater( + len(testIntegrationDashboards), 0, + "test integration is expected to have dashboards", + ) + + dashboards = integrationsTB.GetDashboardsFromQS() + require.Equal( + len(testIntegrationDashboards), len(dashboards), + "dashboards for installed integrations should appear in dashboards list", + ) + + // Should be able to get installed integrations dashboard by id + dd := integrationsTB.GetDashboardByIdFromQS(dashboards[0].Uuid) + require.Equal(*dd, dashboards[0]) + + // Integration dashboards should not longer appear in dashboard list after uninstallation + integrationsTB.RequestQSToUninstallIntegration( + testIntegration.Id, + ) + dashboards = integrationsTB.GetDashboardsFromQS() + require.Equal( + 0, len(dashboards), + "dashboards for uninstalled integrations should not appear in dashboards list", + ) +} + type IntegrationsTestBed struct { t *testing.T testUser *model.User @@ -373,6 +440,40 @@ func (tb *IntegrationsTestBed) RequestQSToUninstallIntegration( tb.RequestQS("/api/v1/integrations/uninstall", request) } +func (tb *IntegrationsTestBed) GetDashboardsFromQS() []dashboards.Dashboard { + result := tb.RequestQS("/api/v1/dashboards", nil) + + dataJson, err := json.Marshal(result.Data) + if err != nil { + tb.t.Fatalf("could not marshal apiResponse.Data: %v", err) + } + + dashboards := []dashboards.Dashboard{} + err = json.Unmarshal(dataJson, &dashboards) + if err != nil { + tb.t.Fatalf(" could not unmarshal apiResponse.Data json into dashboards") + } + + return dashboards +} + +func (tb *IntegrationsTestBed) GetDashboardByIdFromQS(dashboardUuid string) *dashboards.Dashboard { + result := tb.RequestQS(fmt.Sprintf("/api/v1/dashboards/%s", dashboardUuid), nil) + + dataJson, err := json.Marshal(result.Data) + if err != nil { + tb.t.Fatalf("could not marshal apiResponse.Data: %v", err) + } + + dashboard := dashboards.Dashboard{} + err = json.Unmarshal(dataJson, &dashboard) + if err != nil { + tb.t.Fatalf(" could not unmarshal apiResponse.Data json into dashboards") + } + + return &dashboard +} + func (tb *IntegrationsTestBed) RequestQS( path string, postData interface{}, @@ -441,6 +542,7 @@ func NewIntegrationsTestBed(t *testing.T, testDB *sqlx.DB) *IntegrationsTestBed router := app.NewRouter() am := app.NewAuthMiddleware(auth.GetUserFromRequest) + apiHandler.RegisterRoutes(router, am) apiHandler.RegisterIntegrationRoutes(router, am) user, apiErr := createTestUser() diff --git a/pkg/query-service/utils/testutils.go b/pkg/query-service/utils/testutils.go index 33fd76dafa..d8989d9323 100644 --- a/pkg/query-service/utils/testutils.go +++ b/pkg/query-service/utils/testutils.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/app/dashboards" "go.signoz.io/signoz/pkg/query-service/dao" ) @@ -24,6 +25,7 @@ func NewQueryServiceDBForTests(t *testing.T) *sqlx.DB { // TODO(Raj): This should not require passing in the DB file path dao.InitDao("sqlite", testDBFilePath) + dashboards.InitDB(testDBFilePath) return testDB }