feat: add support for S3 region buckets syncing (#7874)

* feat: add support for S3 region buckets syncing

* fix: hide the dropdown icon and not found dropdown for s3 buckets selector

* fix: display s3 buckets selector only for s3 sync service

* chore: tests for configure service s3 sync

---------

Co-authored-by: Piyush Singariya <piyushsingariya@gmail.com>

frontend/src/container/CloudIntegrationPage/ServicesSection/ConfigureServiceModal.tsx

@@ -8,12 +8,14 @@ import {
 	SupportedSignals,
 } from 'container/CloudIntegrationPage/ServicesSection/types';
 import { useUpdateServiceConfig } from 'hooks/integration/aws/useUpdateServiceConfig';
+import { isEqual } from 'lodash-es';
 import { useCallback, useMemo, useState } from 'react';
 import { useQueryClient } from 'react-query';
 
 import logEvent from '../../../api/common/logEvent';
+import S3BucketsSelector from './S3BucketsSelector';
 
-interface IConfigureServiceModalProps {
+export interface IConfigureServiceModalProps {
 	isOpen: boolean;
 	onClose: () => void;
 	serviceName: string;
@@ -36,18 +38,34 @@ function ConfigureServiceModal({
 	const [isLoading, setIsLoading] = useState(false);
 
 	// Track current form values
-	const initialValues = {
+	const initialValues = useMemo(
+		() => ({
 			metrics: initialConfig?.metrics?.enabled || false,
 			logs: initialConfig?.logs?.enabled || false,
-	};
+			s3Buckets: initialConfig?.logs?.s3_buckets || {},
+		}),
+		[initialConfig],
+	);
 	const [currentValues, setCurrentValues] = useState(initialValues);
 
 	const isSaveDisabled = useMemo(
 		() =>
 			// disable only if current values are same as the initial config
 			currentValues.metrics === initialValues.metrics &&
-			currentValues.logs === initialValues.logs,
-		[currentValues, initialValues.metrics, initialValues.logs],
+			currentValues.logs === initialValues.logs &&
+			isEqual(currentValues.s3Buckets, initialValues.s3Buckets),
+		[currentValues, initialValues],
+	);
+
+	const handleS3BucketsChange = useCallback(
+		(bucketsByRegion: Record<string, string[]>) => {
+			setCurrentValues((prev) => ({
+				...prev,
+				s3Buckets: bucketsByRegion,
+			}));
+			form.setFieldsValue({ s3Buckets: bucketsByRegion });
+		},
+		[form],
 	);
 
 	const {
@@ -70,6 +88,7 @@ function ConfigureServiceModal({
 						config: {
 							logs: {
 								enabled: values.logs,
+								s3_buckets: values.s3Buckets,
 							},
 							metrics: {
 								enabled: values.metrics,
@@ -144,6 +163,7 @@ function ConfigureServiceModal({
 				initialValues={{
 					metrics: initialConfig?.metrics?.enabled || false,
 					logs: initialConfig?.logs?.enabled || false,
+					s3Buckets: initialConfig?.logs?.s3_buckets || {},
 				}}
 			>
 				<div className=" configure-service-modal__body">
@@ -174,6 +194,7 @@ function ConfigureServiceModal({
 					)}
 
 					{supportedSignals.logs && (
+						<>
 							<Form.Item
 								name="logs"
 								valuePropName="checked"
@@ -195,6 +216,16 @@ function ConfigureServiceModal({
 									To ingest logs from your AWS services, you must complete several steps
 								</div>
 							</Form.Item>
+
+							{currentValues.logs && serviceId === 's3sync' && (
+								<Form.Item name="s3Buckets" noStyle>
+									<S3BucketsSelector
+										initialBucketsByRegion={currentValues.s3Buckets}
+										onChange={handleS3BucketsChange}
+									/>
+								</Form.Item>
+							)}
+						</>
 					)}
 				</div>
 			</Form>

frontend/src/container/CloudIntegrationPage/ServicesSection/S3BucketsSelector.tsx

@@ -0,0 +1,123 @@
+import { Form, Select, Skeleton, Typography } from 'antd';
+import { useAwsAccounts } from 'hooks/integration/aws/useAwsAccounts';
+import useUrlQuery from 'hooks/useUrlQuery';
+import { useCallback, useMemo, useState } from 'react';
+
+const { Title } = Typography;
+
+interface S3BucketsSelectorProps {
+	onChange?: (bucketsByRegion: Record<string, string[]>) => void;
+	initialBucketsByRegion?: Record<string, string[]>;
+}
+
+/**
+ * Component for selecting S3 buckets by AWS region
+ * Displays a multi-select input for each region in the active AWS account
+ */
+function S3BucketsSelector({
+	onChange,
+	initialBucketsByRegion = {},
+}: S3BucketsSelectorProps): JSX.Element {
+	const cloudAccountId = useUrlQuery().get('cloudAccountId');
+	const { data: accounts, isLoading } = useAwsAccounts();
+	const [bucketsByRegion, setBucketsByRegion] = useState<
+		Record<string, string[]>
+	>(initialBucketsByRegion);
+
+	// Find the active AWS account based on the URL query parameter
+	const activeAccount = useMemo(
+		() =>
+			accounts?.find((account) => account.cloud_account_id === cloudAccountId),
+		[accounts, cloudAccountId],
+	);
+
+	// Get all regions to display (union of account regions and initialBucketsByRegion regions)
+	const allRegions = useMemo(() => {
+		if (!activeAccount) return [];
+
+		// Get unique regions from both sources
+		const initialRegions = Object.keys(initialBucketsByRegion);
+		const accountRegions = activeAccount.config.regions;
+
+		// Create a Set to get unique values
+		const uniqueRegions = new Set([...accountRegions, ...initialRegions]);
+
+		return Array.from(uniqueRegions);
+	}, [activeAccount, initialBucketsByRegion]);
+
+	// Check if a region is disabled (not in account's regions)
+	const isRegionDisabled = useCallback(
+		(region: string) => !activeAccount?.config.regions.includes(region),
+		[activeAccount],
+	);
+
+	// Handle changes to bucket selections for a specific region
+	const handleRegionBucketsChange = useCallback(
+		(region: string, buckets: string[]): void => {
+			setBucketsByRegion((prevBuckets) => {
+				const updatedBuckets = { ...prevBuckets };
+
+				if (buckets.length === 0) {
+					// Remove empty bucket arrays
+					delete updatedBuckets[region];
+				} else {
+					updatedBuckets[region] = buckets;
+				}
+
+				// Notify parent component of changes
+				onChange?.(updatedBuckets);
+				return updatedBuckets;
+			});
+		},
+		[onChange],
+	);
+
+	// Show loading state while fetching account data
+	if (isLoading || !activeAccount) {
+		return <Skeleton active />;
+	}
+
+	return (
+		<div className="s3-buckets-selector">
+			<Title level={5}>Select S3 Buckets by Region</Title>
+
+			{allRegions.map((region) => {
+				const disabled = isRegionDisabled(region);
+
+				return (
+					<Form.Item
+						key={region}
+						label={region}
+						// eslint-disable-next-line react/jsx-props-no-spreading
+						{...(disabled && {
+							help:
+								'Region disabled in account settings; S3 buckets here will not be synced.',
+							validateStatus: 'warning',
+						})}
+					>
+						<Select
+							mode="tags"
+							placeholder={`Enter S3 bucket names for ${region}`}
+							value={bucketsByRegion[region] || []}
+							onChange={(value): void => handleRegionBucketsChange(region, value)}
+							tokenSeparators={[',']}
+							allowClear
+							disabled={disabled}
+							suffixIcon={null}
+							notFoundContent={null}
+							filterOption={false}
+							showSearch
+						/>
+					</Form.Item>
+				);
+			})}
+		</div>
+	);
+}
+
+S3BucketsSelector.defaultProps = {
+	onChange: undefined,
+	initialBucketsByRegion: undefined,
+};
+
+export default S3BucketsSelector;

frontend/src/container/CloudIntegrationPage/ServicesSection/__tests__/ConfigureServiceModalS3Sync.test.tsx

@@ -0,0 +1,162 @@
+import { act, fireEvent, screen, waitFor } from '@testing-library/react';
+import { server } from 'mocks-server/server';
+import { rest, RestRequest } from 'msw'; // Import RestRequest for req.json() typing
+
+import { UpdateServiceConfigPayload } from '../types';
+import { accountsResponse, CLOUD_ACCOUNT_ID, initialBuckets } from './mockData';
+import {
+	assertGenericModalElements,
+	assertS3SyncSpecificElements,
+	renderModal,
+} from './utils';
+
+// --- MOCKS ---
+jest.mock('hooks/useUrlQuery', () => ({
+	__esModule: true,
+	default: jest.fn(() => ({
+		get: jest.fn((paramName: string) => {
+			if (paramName === 'cloudAccountId') {
+				return CLOUD_ACCOUNT_ID;
+			}
+			return null;
+		}),
+	})),
+}));
+
+// --- TEST SUITE ---
+describe('ConfigureServiceModal for S3 Sync service', () => {
+	jest.setTimeout(10000);
+	beforeEach(() => {
+		server.use(
+			rest.get(
+				'http://localhost/api/v1/cloud-integrations/aws/accounts',
+				(req, res, ctx) => res(ctx.json(accountsResponse)),
+			),
+		);
+	});
+
+	it('should render with logs collection switch and bucket selectors (no buckets initially selected)', async () => {
+		act(() => {
+			renderModal({}); // No initial S3 buckets, defaults to 's3sync' serviceId
+		});
+		await assertGenericModalElements(); // Use new generic assertion
+		await assertS3SyncSpecificElements({}); // Use new S3-specific assertion
+	});
+
+	it('should render with logs collection switch and bucket selectors (some buckets initially selected)', async () => {
+		act(() => {
+			renderModal(initialBuckets); // Defaults to 's3sync' serviceId
+		});
+		await assertGenericModalElements(); // Use new generic assertion
+		await assertS3SyncSpecificElements(initialBuckets); // Use new S3-specific assertion
+	});
+
+	it('should enable save button after adding a new bucket via combobox', async () => {
+		act(() => {
+			renderModal(initialBuckets); // Defaults to 's3sync' serviceId
+		});
+		await assertGenericModalElements();
+		await assertS3SyncSpecificElements(initialBuckets);
+
+		expect(screen.getByRole('button', { name: /save/i })).toBeDisabled();
+
+		const targetCombobox = screen.getAllByRole('combobox')[0];
+		const newBucketName = 'a-newly-added-bucket';
+
+		act(() => {
+			fireEvent.change(targetCombobox, { target: { value: newBucketName } });
+			fireEvent.keyDown(targetCombobox, {
+				key: 'Enter',
+				code: 'Enter',
+				keyCode: 13,
+			});
+		});
+
+		await waitFor(() => {
+			expect(screen.getByLabelText(newBucketName)).toBeInTheDocument();
+			expect(screen.getByRole('button', { name: /save/i })).toBeEnabled();
+		});
+	});
+
+	it('should send updated bucket configuration on save', async () => {
+		let capturedPayload: UpdateServiceConfigPayload | null = null;
+		const mockUpdateConfigUrl =
+			'http://localhost/api/v1/cloud-integrations/aws/services/s3sync/config';
+
+		// Override POST handler specifically for this test to capture payload
+		server.use(
+			rest.post(mockUpdateConfigUrl, async (req: RestRequest, res, ctx) => {
+				capturedPayload = await req.json();
+				return res(ctx.status(200), ctx.json({ message: 'Config updated' }));
+			}),
+		);
+		act(() => {
+			renderModal(initialBuckets); // Defaults to 's3sync' serviceId
+		});
+		await assertGenericModalElements();
+		await assertS3SyncSpecificElements(initialBuckets);
+
+		expect(screen.getByRole('button', { name: /save/i })).toBeDisabled();
+
+		const newBucketName = 'another-new-bucket';
+		// As before, targeting the first combobox, assumed to be for 'ap-south-1'.
+		const targetCombobox = screen.getAllByRole('combobox')[0];
+
+		// eslint-disable-next-line sonarjs/no-identical-functions
+		act(() => {
+			fireEvent.change(targetCombobox, { target: { value: newBucketName } });
+			fireEvent.keyDown(targetCombobox, {
+				key: 'Enter',
+				code: 'Enter',
+				keyCode: 13,
+			});
+		});
+
+		await waitFor(() => {
+			expect(screen.getByLabelText(newBucketName)).toBeInTheDocument();
+			expect(screen.getByRole('button', { name: /save/i })).toBeEnabled();
+			act(() => {
+				fireEvent.click(screen.getByRole('button', { name: /save/i }));
+			});
+		});
+
+		await waitFor(() => {
+			expect(capturedPayload).not.toBeNull();
+		});
+
+		expect(capturedPayload).toEqual({
+			cloud_account_id: CLOUD_ACCOUNT_ID,
+			config: {
+				logs: {
+					enabled: true,
+					s3_buckets: {
+						'us-east-2': ['first-bucket', 'second-bucket'], // Existing buckets
+						'ap-south-1': [newBucketName], // Newly added bucket for the first region
+					},
+				},
+				metrics: {},
+			},
+		});
+	});
+
+	it('should not render S3 bucket region selector UI for services other than s3sync', async () => {
+		const otherServiceId = 'cloudwatch';
+		act(() => {
+			renderModal({}, otherServiceId);
+		});
+		await assertGenericModalElements();
+
+		await waitFor(() => {
+			expect(
+				screen.queryByRole('heading', { name: /select s3 buckets by region/i }),
+			).not.toBeInTheDocument();
+
+			const regions = accountsResponse.data.accounts[0]?.config?.regions || [];
+			regions.forEach((region) => {
+				expect(
+					screen.queryByText(`Enter S3 bucket names for ${region}`),
+				).not.toBeInTheDocument();
+			});
+		});
+	});
+});

frontend/src/container/CloudIntegrationPage/ServicesSection/__tests__/mockData.ts

@@ -0,0 +1,44 @@
+import { IConfigureServiceModalProps } from '../ConfigureServiceModal';
+
+const CLOUD_ACCOUNT_ID = '123456789012';
+
+const initialBuckets = { 'us-east-2': ['first-bucket', 'second-bucket'] };
+
+const accountsResponse = {
+	status: 'success',
+	data: {
+		accounts: [
+			{
+				id: 'a1b2c3d4-e5f6-7890-1234-567890abcdef',
+				cloud_account_id: CLOUD_ACCOUNT_ID,
+				config: {
+					regions: ['ap-south-1', 'ap-south-2', 'us-east-1', 'us-east-2'],
+				},
+				status: {
+					integration: {
+						last_heartbeat_ts_ms: 1747114366214,
+					},
+				},
+			},
+		],
+	},
+};
+
+const defaultModalProps: Omit<IConfigureServiceModalProps, 'initialConfig'> = {
+	isOpen: true,
+	onClose: jest.fn(),
+	serviceName: 'S3 Sync',
+	serviceId: 's3sync',
+	cloudAccountId: CLOUD_ACCOUNT_ID,
+	supportedSignals: {
+		logs: true,
+		metrics: false,
+	},
+};
+
+export {
+	accountsResponse,
+	CLOUD_ACCOUNT_ID,
+	defaultModalProps,
+	initialBuckets,
+};

frontend/src/container/CloudIntegrationPage/ServicesSection/__tests__/utils.tsx

@@ -0,0 +1,79 @@
+import { render, RenderResult, screen, waitFor } from '@testing-library/react';
+import MockQueryClientProvider from 'providers/test/MockQueryClientProvider';
+
+import ConfigureServiceModal from '../ConfigureServiceModal';
+import { accountsResponse, defaultModalProps } from './mockData';
+
+/**
+ * Renders the ConfigureServiceModal with specified S3 bucket initial configurations.
+ */
+const renderModal = (
+	initialConfigLogsS3Buckets: Record<string, string[]> = {},
+	serviceId = 's3sync',
+): RenderResult => {
+	const initialConfig = {
+		logs: { enabled: true, s3_buckets: initialConfigLogsS3Buckets },
+		metrics: { enabled: false },
+	};
+
+	return render(
+		<MockQueryClientProvider>
+			<ConfigureServiceModal
+				// eslint-disable-next-line react/jsx-props-no-spreading
+				{...defaultModalProps}
+				serviceId={serviceId}
+				initialConfig={initialConfig}
+			/>
+		</MockQueryClientProvider>,
+	);
+};
+
+/**
+ * Asserts that generic UI elements of the modal are present.
+ */
+const assertGenericModalElements = async (): Promise<void> => {
+	await waitFor(() => {
+		expect(screen.getByRole('switch')).toBeInTheDocument();
+		expect(screen.getByText(/log collection/i)).toBeInTheDocument();
+		expect(
+			screen.getByText(
+				/to ingest logs from your aws services, you must complete several steps/i,
+			),
+		).toBeInTheDocument();
+	});
+};
+
+/**
+ * Asserts the state of S3 bucket selectors for each region, specific to S3 Sync.
+ */
+const assertS3SyncSpecificElements = async (
+	expectedBucketsByRegion: Record<string, string[]> = {},
+): Promise<void> => {
+	const regions = accountsResponse.data.accounts[0]?.config?.regions || [];
+
+	await waitFor(() => {
+		expect(
+			screen.getByRole('heading', { name: /select s3 buckets by region/i }),
+		).toBeInTheDocument();
+
+		regions.forEach((region) => {
+			expect(screen.getByText(region)).toBeInTheDocument();
+			const bucketsForRegion = expectedBucketsByRegion[region] || [];
+			if (bucketsForRegion.length > 0) {
+				bucketsForRegion.forEach((bucket) => {
+					expect(screen.getByText(bucket)).toBeInTheDocument();
+				});
+			} else {
+				expect(
+					screen.getByText(`Enter S3 bucket names for ${region}`),
+				).toBeInTheDocument();
+			}
+		});
+	});
+};
+
+export {
+	assertGenericModalElements,
+	assertS3SyncSpecificElements,
+	renderModal,
+};

frontend/src/container/CloudIntegrationPage/ServicesSection/types.ts

@@ -34,9 +34,18 @@ interface DataStatus {
 	last_received_from: string;
 }
 
+interface S3BucketsByRegion {
+	[region: string]: string[];
+}
+
+interface LogsConfig extends ConfigStatus {
+	s3_buckets?: S3BucketsByRegion;
+}
+
 interface ServiceConfig {
-	logs: ConfigStatus;
+	logs: LogsConfig;
 	metrics: ConfigStatus;
+	s3_sync?: LogsConfig;
 }
 
 interface IServiceStatus {
@@ -99,6 +108,7 @@ interface UpdateServiceConfigPayload {
 	config: {
 		logs: {
 			enabled: boolean;
+			s3_buckets?: S3BucketsByRegion;
 		};
 		metrics: {
 			enabled: boolean;
@@ -113,6 +123,7 @@ interface UpdateServiceConfigResponse {
 		config: {
 			logs: {
 				enabled: boolean;
+				s3_buckets?: S3BucketsByRegion;
 			};
 			metrics: {
 				enabled: boolean;
@@ -125,6 +136,7 @@ export type {
 	CloudAccount,
 	CloudAccountsData,
 	IServiceStatus,
+	S3BucketsByRegion,
 	Service,
 	ServiceConfig,
 	ServiceData,

frontend/src/hooks/integration/aws/useUpdateServiceConfig.ts

@@ -1,4 +1,5 @@
 import { updateServiceConfig } from 'api/integration/aws';
+import { S3BucketsByRegion } from 'container/CloudIntegrationPage/ServicesSection/types';
 import { useMutation, UseMutationResult } from 'react-query';
 
 interface UpdateServiceConfigPayload {
@@ -6,6 +7,7 @@ interface UpdateServiceConfigPayload {
 	config: {
 		logs: {
 			enabled: boolean;
+			s3_buckets?: S3BucketsByRegion;
 		};
 		metrics: {
 			enabled: boolean;