diff --git a/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-query-logs.md b/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-query-logs.md index 373b1016d6..24e20a7815 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-query-logs.md +++ b/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-query-logs.md @@ -78,3 +78,5 @@ Make the collector config file available to your otel collector and use it by ad ``` Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag. +Also note that only 1 collector instance should be configured to collect query_logs. +Using multiple collector instances or replicas with this config will lead to duplicate logs. diff --git a/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/prerequisites.md b/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/prerequisites.md index 7f1019666d..f2c667edbc 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/prerequisites.md +++ b/pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/prerequisites.md @@ -30,7 +30,7 @@ To configure metrics and logs collection for a Clickhouse server, you need the f - **Ensure that an OTEL collector is running in your deployment environment** If needed, please [install SigNoz OTEL Collector](https://signoz.io/docs/tutorial/opentelemetry-binary-usage-in-virtual-machine/) If already installed, ensure that the collector version is v0.88.0 or newer. - If collecting logs from system.query_log table, ensure that the collector version is v0.88.22 or newer. + If collecting logs from system.query_log table, ensure that the collector version is v0.88.23 or newer. Also ensure that you can provide config files to the collector and that you can set environment variables and command line flags used for running it. diff --git a/pkg/query-service/app/integrations/manager.go b/pkg/query-service/app/integrations/manager.go index 56a57ee026..6cd5a0c853 100644 --- a/pkg/query-service/app/integrations/manager.go +++ b/pkg/query-service/app/integrations/manager.go @@ -258,7 +258,7 @@ func (m *Manager) UninstallIntegration( func (m *Manager) GetPipelinesForInstalledIntegrations( ctx context.Context, ) ([]logparsingpipeline.Pipeline, *model.ApiError) { - installedIntegrations, apiErr := m.getDetailsForInstalledIntegrations(ctx) + installedIntegrations, apiErr := m.getInstalledIntegrations(ctx) if apiErr != nil { return nil, apiErr } @@ -327,10 +327,15 @@ func (m *Manager) GetInstalledIntegrationDashboardById( if dId, exists := dd["id"]; exists { if id, ok := dId.(string); ok && id == dashboardId { isLocked := 1 + author := "integration" return &dashboards.Dashboard{ - Uuid: m.dashboardUuid(integrationId, string(dashboardId)), - Locked: &isLocked, - Data: dd, + Uuid: m.dashboardUuid(integrationId, string(dashboardId)), + Locked: &isLocked, + Data: dd, + CreatedAt: integration.Installation.InstalledAt, + CreateBy: &author, + UpdatedAt: integration.Installation.InstalledAt, + UpdateBy: &author, }, nil } } @@ -344,7 +349,7 @@ func (m *Manager) GetInstalledIntegrationDashboardById( func (m *Manager) GetDashboardsForInstalledIntegrations( ctx context.Context, ) ([]dashboards.Dashboard, *model.ApiError) { - installedIntegrations, apiErr := m.getDetailsForInstalledIntegrations(ctx) + installedIntegrations, apiErr := m.getInstalledIntegrations(ctx) if apiErr != nil { return nil, apiErr } @@ -356,10 +361,15 @@ func (m *Manager) GetDashboardsForInstalledIntegrations( if dId, exists := dd["id"]; exists { if dashboardId, ok := dId.(string); ok { isLocked := 1 + author := "integration" result = append(result, dashboards.Dashboard{ - Uuid: m.dashboardUuid(ii.IntegrationSummary.Id, dashboardId), - Locked: &isLocked, - Data: dd, + Uuid: m.dashboardUuid(ii.IntegrationSummary.Id, dashboardId), + Locked: &isLocked, + Data: dd, + CreatedAt: ii.Installation.InstalledAt, + CreateBy: &author, + UpdatedAt: ii.Installation.InstalledAt, + UpdateBy: &author, }) } } @@ -418,10 +428,10 @@ func (m *Manager) getInstalledIntegration( return &installation, nil } -func (m *Manager) getDetailsForInstalledIntegrations( +func (m *Manager) getInstalledIntegrations( ctx context.Context, ) ( - map[string]IntegrationDetails, *model.ApiError, + map[string]Integration, *model.ApiError, ) { installations, apiErr := m.installedIntegrationsRepo.list(ctx) if apiErr != nil { @@ -431,5 +441,24 @@ func (m *Manager) getDetailsForInstalledIntegrations( installedIds := utils.MapSlice(installations, func(i InstalledIntegration) string { return i.IntegrationId }) - return m.availableIntegrationsRepo.get(ctx, installedIds) + integrationDetails, apiErr := m.availableIntegrationsRepo.get(ctx, installedIds) + if apiErr != nil { + return nil, apiErr + } + + result := map[string]Integration{} + for _, ii := range installations { + iDetails, exists := integrationDetails[ii.IntegrationId] + if !exists { + return nil, model.InternalError(fmt.Errorf( + "couldn't find integration details for %s", ii.IntegrationId, + )) + } + + result[ii.IntegrationId] = Integration{ + Installation: &ii, + IntegrationDetails: iDetails, + } + } + return result, nil } diff --git a/pkg/query-service/tests/integration/signoz_integrations_test.go b/pkg/query-service/tests/integration/signoz_integrations_test.go index d58ccaf51a..eae9603888 100644 --- a/pkg/query-service/tests/integration/signoz_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_integrations_test.go @@ -327,6 +327,7 @@ func TestDashboardsForInstalledIntegrationDashboards(t *testing.T) { // Installing an integration should make its dashboards appear in the dashboard list require.False(testAvailableIntegration.IsInstalled) + tsBeforeInstallation := time.Now().Unix() integrationsTB.RequestQSToInstallIntegration( testAvailableIntegration.Id, map[string]interface{}{}, ) @@ -344,9 +345,13 @@ func TestDashboardsForInstalledIntegrationDashboards(t *testing.T) { len(testIntegrationDashboards), len(dashboards), "dashboards for installed integrations should appear in dashboards list", ) + require.GreaterOrEqual(dashboards[0].CreatedAt.Unix(), tsBeforeInstallation) + require.GreaterOrEqual(dashboards[0].UpdatedAt.Unix(), tsBeforeInstallation) // Should be able to get installed integrations dashboard by id dd := integrationsTB.GetDashboardByIdFromQS(dashboards[0].Uuid) + require.GreaterOrEqual(dd.CreatedAt.Unix(), tsBeforeInstallation) + require.GreaterOrEqual(dd.UpdatedAt.Unix(), tsBeforeInstallation) require.Equal(*dd, dashboards[0]) // Integration dashboards should not longer appear in dashboard list after uninstallation