Feat: qs autopopulate installed integration dashboards (#4680)

* chore: add test for dashboards for installed integrations

* feat: include dashboards for installed integrations in API response

* chore: add test expectation for getting installed integration dashboard by id

* feat: add support for retrieving installed integration dashboards by id

* chore: add dashboard id validation for integrations

pkg/query-service/app/http_handler.go

@@ -867,11 +867,15 @@ func (aH *APIHandler) listRules(w http.ResponseWriter, r *http.Request) {
 func (aH *APIHandler) getDashboards(w http.ResponseWriter, r *http.Request) {
 
 	allDashboards, err := dashboards.GetDashboards(r.Context())
-
 	if err != nil {
 		RespondError(w, err, nil)
 		return
 	}
+
+	ic := aH.IntegrationsController
+	installedIntegrationDashboards, err := ic.GetDashboardsForInstalledIntegrations(r.Context())
+	allDashboards = append(allDashboards, installedIntegrationDashboards...)
+
 	tagsFromReq, ok := r.URL.Query()["tags"]
 	if !ok || len(tagsFromReq) == 0 || tagsFromReq[0] == "" {
 		aH.Respond(w, allDashboards)
@@ -1040,8 +1044,19 @@ func (aH *APIHandler) getDashboard(w http.ResponseWriter, r *http.Request) {
 	dashboard, apiError := dashboards.GetDashboard(r.Context(), uuid)
 
 	if apiError != nil {
-		RespondError(w, apiError, nil)
-		return
+		if apiError.Type() != model.ErrorNotFound {
+			RespondError(w, apiError, nil)
+			return
+		}
+
+		dashboard, apiError = aH.IntegrationsController.GetInstalledIntegrationDashboardById(
+			r.Context(), uuid,
+		)
+		if apiError != nil {
+			RespondError(w, apiError, nil)
+			return
+		}
+
 	}
 
 	aH.Respond(w, dashboard)

pkg/query-service/app/integrations/builtin.go

@@ -127,11 +127,39 @@ func readBuiltInIntegration(dirpath string) (
 		)
 	}
 
+	err = validateIntegration(integration)
+	if err != nil {
+		return nil, fmt.Errorf("invalid integration spec %s: %w", integration.Id, err)
+	}
+
 	integration.Id = "builtin-" + integration.Id
 
 	return &integration, nil
 }
 
+func validateIntegration(i IntegrationDetails) error {
+	// Validate dashboard data
+	seenDashboardIds := map[string]interface{}{}
+	for _, dd := range i.Assets.Dashboards {
+		did, exists := dd["id"]
+		if !exists {
+			return fmt.Errorf("id is required. not specified in dashboard titled %v", dd["title"])
+		}
+		dashboardId, ok := did.(string)
+		if !ok {
+			return fmt.Errorf("id must be string in dashboard titled %v", dd["title"])
+		}
+		if _, seen := seenDashboardIds[dashboardId]; seen {
+			return fmt.Errorf("multiple dashboards found with id %s", dashboardId)
+		}
+		seenDashboardIds[dashboardId] = nil
+	}
+
+	// TODO(Raj): Validate all parts of plugged in integrations
+
+	return nil
+}
+
 func hydrateFileUris(spec interface{}, basedir string) (interface{}, error) {
 	if specMap, ok := spec.(map[string]interface{}); ok {
 		result := map[string]interface{}{}

pkg/query-service/app/integrations/builtin_integrations/mongo/assets/dashboards/overview.json

@@ -1,4 +1,5 @@
 {
+    "id": "mongo-overview",
     "description": "This dashboard provides a high-level overview of your MongoDB. It includes read/write performance, most-used replicas, collection metrics etc...",
     "layout": [
         {

pkg/query-service/app/integrations/builtin_integrations/postgres/assets/dashboards/overview.json

@@ -1,4 +1,5 @@
 {
+    "id": "postgres-overview",
     "description": "This dashboard provides a high-level overview of your PostgreSQL databases. It includes replication, locks, and throughput etc...",
     "layout": [
         {

pkg/query-service/app/integrations/builtin_integrations/redis/assets/dashboards/overview.json

@@ -1,4 +1,5 @@
 {
+    "id": "redis-overview",
     "description": "This dashboard shows the Redis instance overview. It includes latency, hit/miss rate, connections, and memory information.\n",
     "layout": [
         {

pkg/query-service/app/integrations/controller.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/jmoiron/sqlx"
 	"go.signoz.io/signoz/pkg/query-service/agentConf"
+	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
 	"go.signoz.io/signoz/pkg/query-service/app/logparsingpipeline"
 	"go.signoz.io/signoz/pkg/query-service/model"
 )
@@ -114,3 +115,15 @@ func (c *Controller) GetPipelinesForInstalledIntegrations(
 ) ([]logparsingpipeline.Pipeline, *model.ApiError) {
 	return c.mgr.GetPipelinesForInstalledIntegrations(ctx)
 }
+
+func (c *Controller) GetDashboardsForInstalledIntegrations(
+	ctx context.Context,
+) ([]dashboards.Dashboard, *model.ApiError) {
+	return c.mgr.GetDashboardsForInstalledIntegrations(ctx)
+}
+
+func (c *Controller) GetInstalledIntegrationDashboardById(
+	ctx context.Context, dashboardUuid string,
+) (*dashboards.Dashboard, *model.ApiError) {
+	return c.mgr.GetInstalledIntegrationDashboardById(ctx, dashboardUuid)
+}

pkg/query-service/app/integrations/manager.go

@@ -33,8 +33,8 @@ type IntegrationSummary struct {
 }
 
 type IntegrationAssets struct {
-	Logs       LogsAssets             `json:"logs"`
-	Dashboards []dashboards.Dashboard `json:"dashboards"`
+	Logs       LogsAssets        `json:"logs"`
+	Dashboards []dashboards.Data `json:"dashboards"`
 
 	Alerts []rules.PostableRule `json:"alerts"`
 }
@@ -248,6 +248,120 @@ func (m *Manager) UninstallIntegration(
 	return m.installedIntegrationsRepo.delete(ctx, integrationId)
 }
 
+func (m *Manager) GetPipelinesForInstalledIntegrations(
+	ctx context.Context,
+) ([]logparsingpipeline.Pipeline, *model.ApiError) {
+	installedIntegrations, apiErr := m.getDetailsForInstalledIntegrations(ctx)
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	pipelines := []logparsingpipeline.Pipeline{}
+	for _, ii := range installedIntegrations {
+		for _, p := range ii.Assets.Logs.Pipelines {
+			pp := logparsingpipeline.Pipeline{
+				// Alias is used for identifying integration pipelines. Id can't be used for this
+				// since versioning while saving pipelines requires a new id for each version
+				// to avoid altering history when pipelines are edited/reordered etc
+				Alias:       AliasForIntegrationPipeline(ii.Id, p.Alias),
+				Id:          uuid.NewString(),
+				OrderId:     p.OrderId,
+				Enabled:     p.Enabled,
+				Name:        p.Name,
+				Description: &p.Description,
+				Filter:      p.Filter,
+				Config:      p.Config,
+			}
+			pipelines = append(pipelines, pp)
+		}
+	}
+
+	return pipelines, nil
+}
+
+func (m *Manager) dashboardUuid(integrationId string, dashboardId string) string {
+	return strings.Join([]string{"integration", integrationId, dashboardId}, "--")
+}
+
+func (m *Manager) parseDashboardUuid(dashboardUuid string) (
+	integrationId string, dashboardId string, err *model.ApiError,
+) {
+	parts := strings.SplitN(dashboardUuid, "--", 3)
+	if len(parts) != 3 || parts[0] != "integration" {
+		return "", "", model.BadRequest(fmt.Errorf(
+			"invalid installed integration dashboard id",
+		))
+	}
+
+	return parts[1], parts[2], nil
+}
+
+func (m *Manager) GetInstalledIntegrationDashboardById(
+	ctx context.Context,
+	dashboardUuid string,
+) (*dashboards.Dashboard, *model.ApiError) {
+	integrationId, dashboardId, apiErr := m.parseDashboardUuid(dashboardUuid)
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	integration, apiErr := m.GetIntegration(ctx, integrationId)
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	if integration.Installation == nil {
+		return nil, model.BadRequest(fmt.Errorf(
+			"integration with id %s is not installed", integrationId,
+		))
+	}
+
+	for _, dd := range integration.IntegrationDetails.Assets.Dashboards {
+		if dId, exists := dd["id"]; exists {
+			if id, ok := dId.(string); ok && id == dashboardId {
+				isLocked := 1
+				return &dashboards.Dashboard{
+					Uuid:   m.dashboardUuid(integrationId, string(dashboardId)),
+					Locked: &isLocked,
+					Data:   dd,
+				}, nil
+			}
+		}
+	}
+
+	return nil, model.NotFoundError(fmt.Errorf(
+		"integration dashboard with id %s not found", dashboardUuid,
+	))
+}
+
+func (m *Manager) GetDashboardsForInstalledIntegrations(
+	ctx context.Context,
+) ([]dashboards.Dashboard, *model.ApiError) {
+	installedIntegrations, apiErr := m.getDetailsForInstalledIntegrations(ctx)
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	result := []dashboards.Dashboard{}
+
+	for _, ii := range installedIntegrations {
+		for _, dd := range ii.Assets.Dashboards {
+			if dId, exists := dd["id"]; exists {
+				if dashboardId, ok := dId.(string); ok {
+					isLocked := 1
+					result = append(result, dashboards.Dashboard{
+						Uuid:   m.dashboardUuid(ii.IntegrationSummary.Id, dashboardId),
+						Locked: &isLocked,
+						Data:   dd,
+					})
+				}
+			}
+		}
+	}
+
+	return result, nil
+}
+
 // Helpers.
 func (m *Manager) getIntegrationDetails(
 	ctx context.Context,
@@ -297,9 +411,11 @@ func (m *Manager) getInstalledIntegration(
 	return &installation, nil
 }
 
-func (m *Manager) GetPipelinesForInstalledIntegrations(
+func (m *Manager) getDetailsForInstalledIntegrations(
 	ctx context.Context,
-) ([]logparsingpipeline.Pipeline, *model.ApiError) {
+) (
+	map[string]IntegrationDetails, *model.ApiError,
+) {
 	installations, apiErr := m.installedIntegrationsRepo.list(ctx)
 	if apiErr != nil {
 		return nil, apiErr
@@ -308,30 +424,5 @@ func (m *Manager) GetPipelinesForInstalledIntegrations(
 	installedIds := utils.MapSlice(installations, func(i InstalledIntegration) string {
 		return i.IntegrationId
 	})
-	installedIntegrations, apiErr := m.availableIntegrationsRepo.get(ctx, installedIds)
-	if apiErr != nil {
-		return nil, apiErr
-	}
-
-	pipelines := []logparsingpipeline.Pipeline{}
-	for _, ii := range installedIntegrations {
-		for _, p := range ii.Assets.Logs.Pipelines {
-			pp := logparsingpipeline.Pipeline{
-				// Alias is used for identifying integration pipelines. Id can't be used for this
-				// since versioning while saving pipelines requires a new id for each version
-				// to avoid altering history when pipelines are edited/reordered etc
-				Alias:       AliasForIntegrationPipeline(ii.Id, p.Alias),
-				Id:          uuid.NewString(),
-				OrderId:     p.OrderId,
-				Enabled:     p.Enabled,
-				Name:        p.Name,
-				Description: &p.Description,
-				Filter:      p.Filter,
-				Config:      p.Config,
-			}
-			pipelines = append(pipelines, pp)
-		}
-	}
-
-	return pipelines, nil
+	return m.availableIntegrationsRepo.get(ctx, installedIds)
 }

pkg/query-service/app/integrations/test_utils.go

@@ -92,7 +92,7 @@ func (t *TestAvailableIntegrationsRepo) list(
 						},
 					},
 				},
-				Dashboards: []dashboards.Dashboard{},
+				Dashboards: []dashboards.Data{},
 				Alerts:     []rules.PostableRule{},
 			},
 			ConnectionTests: &IntegrationConnectionTests{
@@ -170,7 +170,7 @@ func (t *TestAvailableIntegrationsRepo) list(
 						},
 					},
 				},
-				Dashboards: []dashboards.Dashboard{},
+				Dashboards: []dashboards.Data{},
 				Alerts:     []rules.PostableRule{},
 			},
 			ConnectionTests: &IntegrationConnectionTests{

pkg/query-service/tests/integration/signoz_integrations_test.go

@@ -14,6 +14,7 @@ import (
 	mockhouse "github.com/srikanthccv/ClickHouse-go-mock"
 	"github.com/stretchr/testify/require"
 	"go.signoz.io/signoz/pkg/query-service/app"
+	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
 	"go.signoz.io/signoz/pkg/query-service/app/integrations"
 	"go.signoz.io/signoz/pkg/query-service/app/logparsingpipeline"
 	"go.signoz.io/signoz/pkg/query-service/auth"
@@ -275,6 +276,72 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines)
 }
 
+func TestDashboardsForInstalledIntegrationDashboards(t *testing.T) {
+	require := require.New(t)
+
+	testDB := utils.NewQueryServiceDBForTests(t)
+	integrationsTB := NewIntegrationsTestBed(t, testDB)
+
+	availableIntegrationsResp := integrationsTB.GetAvailableIntegrationsFromQS()
+	availableIntegrations := availableIntegrationsResp.Integrations
+	require.Greater(
+		len(availableIntegrations), 0,
+		"some integrations should come bundled with SigNoz",
+	)
+
+	dashboards := integrationsTB.GetDashboardsFromQS()
+	require.Equal(
+		0, len(dashboards),
+		"There should be no dashboards at the start",
+	)
+
+	// Find an available integration that contains dashboards
+	var testAvailableIntegration *integrations.IntegrationsListItem
+	for _, ai := range availableIntegrations {
+		details := integrationsTB.GetIntegrationDetailsFromQS(ai.Id)
+		require.NotNil(details)
+		if len(details.Assets.Dashboards) > 0 {
+			testAvailableIntegration = &ai
+			break
+		}
+	}
+	require.NotNil(testAvailableIntegration)
+
+	// Installing an integration should make its dashboards appear in the dashboard list
+	require.False(testAvailableIntegration.IsInstalled)
+	integrationsTB.RequestQSToInstallIntegration(
+		testAvailableIntegration.Id, map[string]interface{}{},
+	)
+
+	testIntegration := integrationsTB.GetIntegrationDetailsFromQS(testAvailableIntegration.Id)
+	require.NotNil(testIntegration.Installation)
+	testIntegrationDashboards := testIntegration.Assets.Dashboards
+	require.Greater(
+		len(testIntegrationDashboards), 0,
+		"test integration is expected to have dashboards",
+	)
+
+	dashboards = integrationsTB.GetDashboardsFromQS()
+	require.Equal(
+		len(testIntegrationDashboards), len(dashboards),
+		"dashboards for installed integrations should appear in dashboards list",
+	)
+
+	// Should be able to get installed integrations dashboard by id
+	dd := integrationsTB.GetDashboardByIdFromQS(dashboards[0].Uuid)
+	require.Equal(*dd, dashboards[0])
+
+	// Integration dashboards should not longer appear in dashboard list after uninstallation
+	integrationsTB.RequestQSToUninstallIntegration(
+		testIntegration.Id,
+	)
+	dashboards = integrationsTB.GetDashboardsFromQS()
+	require.Equal(
+		0, len(dashboards),
+		"dashboards for uninstalled integrations should not appear in dashboards list",
+	)
+}
+
 type IntegrationsTestBed struct {
 	t              *testing.T
 	testUser       *model.User
@@ -373,6 +440,40 @@ func (tb *IntegrationsTestBed) RequestQSToUninstallIntegration(
 	tb.RequestQS("/api/v1/integrations/uninstall", request)
 }
 
+func (tb *IntegrationsTestBed) GetDashboardsFromQS() []dashboards.Dashboard {
+	result := tb.RequestQS("/api/v1/dashboards", nil)
+
+	dataJson, err := json.Marshal(result.Data)
+	if err != nil {
+		tb.t.Fatalf("could not marshal apiResponse.Data: %v", err)
+	}
+
+	dashboards := []dashboards.Dashboard{}
+	err = json.Unmarshal(dataJson, &dashboards)
+	if err != nil {
+		tb.t.Fatalf(" could not unmarshal apiResponse.Data json into dashboards")
+	}
+
+	return dashboards
+}
+
+func (tb *IntegrationsTestBed) GetDashboardByIdFromQS(dashboardUuid string) *dashboards.Dashboard {
+	result := tb.RequestQS(fmt.Sprintf("/api/v1/dashboards/%s", dashboardUuid), nil)
+
+	dataJson, err := json.Marshal(result.Data)
+	if err != nil {
+		tb.t.Fatalf("could not marshal apiResponse.Data: %v", err)
+	}
+
+	dashboard := dashboards.Dashboard{}
+	err = json.Unmarshal(dataJson, &dashboard)
+	if err != nil {
+		tb.t.Fatalf(" could not unmarshal apiResponse.Data json into dashboards")
+	}
+
+	return &dashboard
+}
+
 func (tb *IntegrationsTestBed) RequestQS(
 	path string,
 	postData interface{},
@@ -441,6 +542,7 @@ func NewIntegrationsTestBed(t *testing.T, testDB *sqlx.DB) *IntegrationsTestBed
 
 	router := app.NewRouter()
 	am := app.NewAuthMiddleware(auth.GetUserFromRequest)
+	apiHandler.RegisterRoutes(router, am)
 	apiHandler.RegisterIntegrationRoutes(router, am)
 
 	user, apiErr := createTestUser()

pkg/query-service/utils/testutils.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/jmoiron/sqlx"
+	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
 	"go.signoz.io/signoz/pkg/query-service/dao"
 )
 
@@ -24,6 +25,7 @@ func NewQueryServiceDBForTests(t *testing.T) *sqlx.DB {
 
 	// TODO(Raj): This should not require passing in the DB file path
 	dao.InitDao("sqlite", testDBFilePath)
+	dashboards.InitDB(testDBFilePath)
 
 	return testDB
 }