Feat: fe: logs pipelines timestamp parsing processor (#4106)

* chore: add processor config for time parsing processor

* chore: add select input and processor fields with enumerated options

* feat: set timestamp layout to default value when layout_type is changed

* chore: minor cleanup

* chore: some more cleanup

* chore: some more cleanup

* chore: get jest passing

* chore: normalize ts in pipelines previews input and output

* chore: some cleanup

* fix: set correct field id for timestamp format input

frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/ProcessorForm.tsx

@@ -0,0 +1,78 @@
+import { Form, Input, Select } from 'antd';
+import { ModalFooterTitle } from 'container/PipelinePage/styles';
+import { useTranslation } from 'react-i18next';
+
+import { formValidationRules } from '../config';
+import { processorFields, ProcessorFormField } from './config';
+import {
+	Container,
+	FormWrapper,
+	PipelineIndexIcon,
+	StyledSelect,
+} from './styles';
+
+function ProcessorFieldInput({
+	fieldData,
+}: ProcessorFieldInputProps): JSX.Element | null {
+	const { t } = useTranslation('pipeline');
+
+	// Watch form values so we can evaluate shouldRender on
+	// conditional fields when form values are updated.
+	const form = Form.useFormInstance();
+	Form.useWatch(fieldData?.dependencies || [], form);
+
+	if (fieldData.shouldRender && !fieldData.shouldRender(form)) {
+		return null;
+	}
+
+	return (
+		<Container>
+			<PipelineIndexIcon size="small">
+				{Number(fieldData.id) + 1}
+			</PipelineIndexIcon>
+			<FormWrapper>
+				<Form.Item
+					required={false}
+					label={<ModalFooterTitle>{fieldData.fieldName}</ModalFooterTitle>}
+					key={fieldData.id}
+					name={fieldData.name}
+					initialValue={fieldData.initialValue}
+					rules={fieldData.rules ? fieldData.rules : formValidationRules}
+					dependencies={fieldData.dependencies || []}
+				>
+					{fieldData?.options ? (
+						<StyledSelect>
+							{fieldData.options.map(({ value, label }) => (
+								<Select.Option key={value + label} value={value}>
+									{label}
+								</Select.Option>
+							))}
+						</StyledSelect>
+					) : (
+						<Input placeholder={t(fieldData.placeholder)} />
+					)}
+				</Form.Item>
+			</FormWrapper>
+		</Container>
+	);
+}
+
+interface ProcessorFieldInputProps {
+	fieldData: ProcessorFormField;
+}
+
+function ProcessorForm({ processorType }: ProcessorFormProps): JSX.Element {
+	return (
+		<div>
+			{processorFields[processorType]?.map((fieldData: ProcessorFormField) => (
+				<ProcessorFieldInput key={fieldData.id} fieldData={fieldData} />
+			))}
+		</div>
+	);
+}
+
+interface ProcessorFormProps {
+	processorType: string;
+}
+
+export default ProcessorForm;

frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/config.ts

@@ -1,5 +1,7 @@
+import { FormInstance } from 'antd';
 import { Rule, RuleRender } from 'antd/es/form';
 import { NamePath } from 'antd/es/form/interface';
+import { ProcessorData } from 'types/api/pipeline/def';
 
 type ProcessorType = {
 	key: string;
@@ -14,6 +16,7 @@ export const processorTypes: Array<ProcessorType> = [
 	{ key: 'regex_parser', value: 'regex_parser', label: 'Regex' },
 	{ key: 'json_parser', value: 'json_parser', label: 'Json Parser' },
 	{ key: 'trace_parser', value: 'trace_parser', label: 'Trace Parser' },
+	{ key: 'time_parser', value: 'time_parser', label: 'Timestamp Parser' },
 	{ key: 'add', value: 'add', label: 'Add' },
 	{ key: 'remove', value: 'remove', label: 'Remove' },
 	// { key: 'retain', value: 'retain', label: 'Retain' }, @Chintan - Commented as per Nitya's suggestion
@@ -23,6 +26,11 @@ export const processorTypes: Array<ProcessorType> = [
 
 export const DEFAULT_PROCESSOR_TYPE = processorTypes[0].value;
 
+export type ProcessorFieldOption = {
+	label: string;
+	value: string;
+};
+
 export type ProcessorFormField = {
 	id: number;
 	fieldName: string;
@@ -31,6 +39,12 @@ export type ProcessorFormField = {
 	rules?: Array<Rule>;
 	initialValue?: string;
 	dependencies?: Array<string | NamePath>;
+	options?: Array<ProcessorFieldOption>;
+	shouldRender?: (form: FormInstance) => boolean;
+	onFormValuesChanged?: (
+		changedValues: ProcessorData,
+		form: FormInstance,
+	) => void;
 };
 
 const traceParserFieldValidator: RuleRender = (form) => ({
@@ -206,6 +220,103 @@ export const processorFields: { [key: string]: Array<ProcessorFormField> } = {
 			],
 		},
 	],
+	time_parser: [
+		{
+			id: 1,
+			fieldName: 'Name of Timestamp Parsing Processor',
+			placeholder: 'processor_name_placeholder',
+			name: 'name',
+		},
+		{
+			id: 2,
+			fieldName: 'Parse Timestamp Value From',
+			placeholder: 'processor_parsefrom_placeholder',
+			name: 'parse_from',
+			initialValue: 'attributes.timestamp',
+		},
+		{
+			id: 3,
+			fieldName: 'Timestamp Format Type',
+			placeholder: '',
+			name: 'layout_type',
+			initialValue: 'strptime',
+			options: [
+				{
+					label: 'Unix Epoch',
+					value: 'epoch',
+				},
+				{
+					label: 'strptime Format',
+					value: 'strptime',
+				},
+			],
+			onFormValuesChanged: (
+				changedValues: ProcessorData,
+				form: FormInstance,
+			): void => {
+				if (changedValues?.layout_type) {
+					const newLayoutValue =
+						changedValues.layout_type === 'strptime' ? '%Y-%m-%dT%H:%M:%S.%f%z' : 's';
+
+					form.setFieldValue('layout', newLayoutValue);
+				}
+			},
+		},
+		{
+			id: 4,
+			fieldName: 'Epoch Format',
+			placeholder: '',
+			name: 'layout',
+			dependencies: ['layout_type'],
+			shouldRender: (form: FormInstance): boolean => {
+				const layoutType = form.getFieldValue('layout_type');
+				return layoutType === 'epoch';
+			},
+			initialValue: 's',
+			options: [
+				{
+					label: 'seconds',
+					value: 's',
+				},
+				{
+					label: 'milliseconds',
+					value: 'ms',
+				},
+				{
+					label: 'microseconds',
+					value: 'us',
+				},
+				{
+					label: 'nanoseconds',
+					value: 'ns',
+				},
+				{
+					label: 'seconds.milliseconds (eg: 1136214245.123)',
+					value: 's.ms',
+				},
+				{
+					label: 'seconds.microseconds (eg: 1136214245.123456)',
+					value: 's.us',
+				},
+				{
+					label: 'seconds.nanoseconds (eg: 1136214245.123456789)',
+					value: 's.ns',
+				},
+			],
+		},
+		{
+			id: 4,
+			fieldName: 'Timestamp Format',
+			placeholder: 'strptime directives based format. Eg: %Y-%m-%dT%H:%M:%S.%f%z',
+			name: 'layout',
+			dependencies: ['layout_type'],
+			shouldRender: (form: FormInstance): boolean => {
+				const layoutType = form.getFieldValue('layout_type');
+				return layoutType === 'strptime';
+			},
+			initialValue: '%Y-%m-%dT%H:%M:%S.%f%z',
+		},
+	],
 	retain: [
 		{
 			id: 1,

frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/index.tsx

@@ -11,9 +11,9 @@ import { v4 } from 'uuid';
 
 import { ModalButtonWrapper, ModalTitle } from '../styles';
 import { getEditedDataSource, getRecordIndex } from '../utils';
-import { DEFAULT_PROCESSOR_TYPE } from './config';
+import { DEFAULT_PROCESSOR_TYPE, processorFields } from './config';
 import TypeSelect from './FormFields/TypeSelect';
-import { renderProcessorForm } from './utils';
+import ProcessorForm from './ProcessorForm';
 
 function AddNewProcessor({
 	isActionType,
@@ -141,6 +141,17 @@ function AddNewProcessor({
 
 	const isOpen = useMemo(() => isEdit || isAdd, [isAdd, isEdit]);
 
+	const onFormValuesChanged = useCallback(
+		(changedValues: ProcessorData): void => {
+			processorFields[processorType].forEach((field) => {
+				if (field.onFormValuesChanged) {
+					field.onFormValuesChanged(changedValues, form);
+				}
+			});
+		},
+		[form, processorType],
+	);
+
 	return (
 		<Modal
 			title={<ModalTitle level={4}>{modalTitle}</ModalTitle>}
@@ -157,9 +168,10 @@ function AddNewProcessor({
 				onFinish={onFinish}
 				autoComplete="off"
 				form={form}
+				onValuesChange={onFormValuesChanged}
 			>
 				<TypeSelect value={processorType} onChange={handleProcessorType} />
-				{renderProcessorForm(processorType)}
+				<ProcessorForm processorType={processorType} />
 				<Divider plain />
 				<Form.Item>
 					<ModalButtonWrapper>

frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/utils.tsx

@@ -1,9 +0,0 @@
-import { processorFields, ProcessorFormField } from './config';
-import NameInput from './FormFields/NameInput';
-
-export const renderProcessorForm = (
-	processorType: string,
-): Array<JSX.Element> =>
-	processorFields[processorType]?.map((fieldData: ProcessorFormField) => (
-		<NameInput key={fieldData.id} fieldData={fieldData} />
-	));

frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts

@@ -27,7 +27,8 @@ const usePipelinePreview = ({
 	// ILog allows both number and string while the API needs a number
 	const simulationInput = inputLogs.map((l) => ({
 		...l,
-		timestamp: new Date(l.timestamp).getTime(),
+		// log timestamps in query service API are unix nanos
+		timestamp: new Date(l.timestamp).getTime() * 10 ** 6,
 	}));
 
 	const response = useQuery<PipelineSimulationResponse, AxiosError>({
@@ -42,9 +43,15 @@ const usePipelinePreview = ({
 
 	const { isFetching, isError, data, error } = response;
 
+	const outputLogs = (data?.logs || []).map((l: ILog) => ({
+		...l,
+		// log timestamps in query service API are unix nanos
+		timestamp: (l.timestamp as number) / 10 ** 6,
+	}));
+
 	return {
 		isLoading: isFetching,
-		outputLogs: data?.logs || [],
+		outputLogs,
 		isError,
 		errorMsg: error?.response?.data?.error || '',
 	};

frontend/src/types/api/pipeline/def.ts

@@ -27,6 +27,10 @@ export interface ProcessorData {
 	trace_flags?: {
 		parse_from: string;
 	};
+
+	// time parser fields
+	layout_type?: string;
+	layout?: string;
 }
 
 export interface PipelineData {