QueryBuilder filters for log pipelines (#3587)

frontend/jest.config.ts

@@ -7,7 +7,7 @@ const config: Config.InitialOptions = {
 	moduleFileExtensions: ['ts', 'tsx', 'js', 'json'],
 	modulePathIgnorePatterns: ['dist'],
 	moduleNameMapper: {
-		'\\.(css|less)$': '<rootDir>/__mocks__/cssMock.ts',
+		'\\.(css|less|scss)$': '<rootDir>/__mocks__/cssMock.ts',
 	},
 	globals: {
 		extensionsToTreatAsEsm: ['.ts'],

frontend/public/locales/en/pipeline.json

@@ -25,6 +25,7 @@
 	"delete_processor_description": "Logs are processed sequentially in processors. Deleting a processor may change content of data processed by other processors",
 	"search_pipeline_placeholder": "Filter Pipelines",
 	"pipeline_name_placeholder": "Name",
+	"pipeline_filter_placeholder": "Filter for selecting logs to be processed by this pipeline. Example: service_name = billing",
 	"pipeline_tags_placeholder": "Tags",
 	"pipeline_description_placeholder": "Enter description for your pipeline",
 	"processor_name_placeholder": "Name",

frontend/src/container/PipelinePage/PipelineListsView/AddNewPipeline/FormFields/FilterInput.tsx

@@ -0,0 +1,64 @@
+import { Form } from 'antd';
+import { initialQueryBuilderFormValuesMap } from 'constants/queryBuilder';
+import QueryBuilderSearch from 'container/QueryBuilder/filters/QueryBuilderSearch';
+import isEqual from 'lodash-es/isEqual';
+import { useTranslation } from 'react-i18next';
+import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
+
+import { ProcessorFormField } from '../../AddNewProcessor/config';
+import { formValidationRules } from '../../config';
+import { FormLabelStyle } from '../styles';
+
+function TagFilterInput({
+	value,
+	onChange,
+	placeholder,
+}: TagFilterInputProps): JSX.Element {
+	const query = { ...initialQueryBuilderFormValuesMap.logs };
+	if (value) {
+		query.filters = value;
+	}
+
+	const onQueryChange = (newValue: TagFilter): void => {
+		// Avoid unnecessary onChange calls
+		if (!isEqual(newValue, query.filters)) {
+			onChange(newValue);
+		}
+	};
+
+	return (
+		<QueryBuilderSearch
+			query={query}
+			onChange={onQueryChange}
+			placeholder={placeholder}
+		/>
+	);
+}
+
+interface TagFilterInputProps {
+	onChange: (filter: TagFilter) => void;
+	value: TagFilter;
+	placeholder: string;
+}
+
+function FilterInput({ fieldData }: FilterInputProps): JSX.Element {
+	const { t } = useTranslation('pipeline');
+
+	return (
+		<Form.Item
+			required={false}
+			label={<FormLabelStyle>{fieldData.fieldName}</FormLabelStyle>}
+			key={fieldData.id}
+			rules={formValidationRules}
+			name={fieldData.name}
+		>
+			{/* Antd form will supply value and onChange to <TagFilterInput /> here.
+      // @ts-ignore */}
+			<TagFilterInput placeholder={t(fieldData.placeholder)} />
+		</Form.Item>
+	);
+}
+interface FilterInputProps {
+	fieldData: ProcessorFormField;
+}
+export default FilterInput;

frontend/src/container/PipelinePage/PipelineListsView/AddNewPipeline/FormFields/FilterSearch.tsx

@@ -1,31 +0,0 @@
-import { Form, Input } from 'antd';
-import { useTranslation } from 'react-i18next';
-
-import { ProcessorFormField } from '../../AddNewProcessor/config';
-import { formValidationRules } from '../../config';
-import { FormLabelStyle } from '../styles';
-
-function FilterSearch({ fieldData }: FilterSearchProps): JSX.Element {
-	const { t } = useTranslation('pipeline');
-
-	return (
-		<Form.Item
-			required={false}
-			label={<FormLabelStyle>{fieldData.fieldName}</FormLabelStyle>}
-			key={fieldData.id}
-			rules={formValidationRules}
-			name={fieldData.name}
-		>
-			<Input.Search
-				id={fieldData.id.toString()}
-				name={fieldData.name}
-				placeholder={t(fieldData.placeholder)}
-				allowClear
-			/>
-		</Form.Item>
-	);
-}
-interface FilterSearchProps {
-	fieldData: ProcessorFormField;
-}
-export default FilterSearch;

frontend/src/container/PipelinePage/PipelineListsView/TableComponents/PipelineFilterPreview/index.tsx

@@ -0,0 +1,24 @@
+import './styles.scss';
+
+import { queryFilterTags } from 'hooks/queryBuilder/useTag';
+import { PipelineData } from 'types/api/pipeline/def';
+
+function PipelineFilterPreview({
+	filter,
+}: PipelineFilterPreviewProps): JSX.Element {
+	return (
+		<div className="pipeline-filter-preview-container">
+			{queryFilterTags(filter).map((tag) => (
+				<div className="pipeline-filter-preview-condition" key={tag}>
+					{tag}
+				</div>
+			))}
+		</div>
+	);
+}
+
+interface PipelineFilterPreviewProps {
+	filter: PipelineData['filter'];
+}
+
+export default PipelineFilterPreview;

frontend/src/container/PipelinePage/PipelineListsView/TableComponents/PipelineFilterPreview/styles.scss

@@ -0,0 +1,10 @@
+.pipeline-filter-preview-condition {
+	padding: 0 0.2em;
+}
+
+.pipeline-filter-preview-container {
+	display: flex;
+	flex-wrap: wrap;
+	gap: 0.4em;
+	font-size: 0.75rem;
+}

frontend/src/container/PipelinePage/PipelineListsView/TableComponents/index.tsx

@@ -4,6 +4,7 @@ import { PipelineData, ProcessorData } from 'types/api/pipeline/def';
 
 import { PipelineIndexIcon } from '../AddNewProcessor/styles';
 import { ColumnDataStyle, ListDataStyle, ProcessorIndexIcon } from '../styles';
+import PipelineFilterPreview from './PipelineFilterPreview';
 
 const componentMap: ComponentMap = {
 	orderId: ({ record }) => <PipelineIndexIcon>{record}</PipelineIndexIcon>,
@@ -14,6 +15,7 @@ const componentMap: ComponentMap = {
 	),
 	id: ({ record }) => <ProcessorIndexIcon>{record}</ProcessorIndexIcon>,
 	name: ({ record }) => <ListDataStyle>{record}</ListDataStyle>,
+	filter: ({ record }) => <PipelineFilterPreview filter={record} />,
 };
 
 function TableComponents({
@@ -31,7 +33,9 @@ type ComponentMap = {
 	[key: string]: React.FC<{ record: Record }>;
 };
 
-export type Record = PipelineData['orderId'] & ProcessorData;
+export type Record = PipelineData['orderId'] &
+	PipelineData['filter'] &
+	ProcessorData;
 
 interface TableComponentsProps {
 	columnKey: string;

frontend/src/container/PipelinePage/PipelineListsView/config.ts

@@ -8,15 +8,16 @@ import {
 import DeploymentStage from '../Layouts/ChangeHistory/DeploymentStage';
 import DeploymentTime from '../Layouts/ChangeHistory/DeploymentTime';
 import DescriptionTextArea from './AddNewPipeline/FormFields/DescriptionTextArea';
+import FilterInput from './AddNewPipeline/FormFields/FilterInput';
 import NameInput from './AddNewPipeline/FormFields/NameInput';
 
 export const pipelineFields = [
 	{
 		id: 1,
 		fieldName: 'Filter',
-		placeholder: 'search_pipeline_placeholder',
+		placeholder: 'pipeline_filter_placeholder',
 		name: 'filter',
-		component: NameInput,
+		component: FilterInput,
 	},
 	{
 		id: 2,

frontend/src/container/PipelinePage/mocks/pipeline.ts

@@ -1,7 +1,33 @@
 import { Pipeline, PipelineData } from 'types/api/pipeline/def';
+import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
+import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
 
 export const configurationVersion = '1.0';
 
+export function mockPipelineFilter(
+	key: string,
+	op: string,
+	value: string,
+): TagFilter {
+	return {
+		op: 'AND',
+		items: [
+			{
+				id: `${key}-${value}`,
+				key: {
+					key,
+					dataType: DataTypes.String,
+					type: '',
+					isColumn: false,
+					isJSON: false,
+				},
+				op,
+				value,
+			},
+		],
+	};
+}
+
 export const pipelineMockData: Array<PipelineData> = [
 	{
 		id: '4453c8b0-c0fd-42bf-bf09-7cc1b04ccdc9',
@@ -10,7 +36,7 @@ export const pipelineMockData: Array<PipelineData> = [
 		alias: 'apachecommonparser',
 		description: 'This is a desc',
 		enabled: false,
-		filter: 'attributes.source == nginx',
+		filter: mockPipelineFilter('source', '=', 'nginx'),
 		config: [
 			{
 				orderId: 1,
@@ -43,7 +69,7 @@ export const pipelineMockData: Array<PipelineData> = [
 		alias: 'movingpipelinenew',
 		description: 'This is a desc of move',
 		enabled: false,
-		filter: 'attributes.method == POST',
+		filter: mockPipelineFilter('method', '=', 'POST'),
 		config: [
 			{
 				orderId: 1,

frontend/src/container/PipelinePage/tests/AddNewPipeline.test.tsx

@@ -1,5 +1,6 @@
 import { render } from '@testing-library/react';
 import { I18nextProvider } from 'react-i18next';
+import { QueryClient, QueryClientProvider } from 'react-query';
 import { Provider } from 'react-redux';
 import { MemoryRouter } from 'react-router-dom';
 import i18n from 'ReactI18';
@@ -27,25 +28,36 @@ beforeAll(() => {
 	matchMedia();
 });
 
+const queryClient = new QueryClient({
+	defaultOptions: {
+		queries: {
+			refetchOnWindowFocus: false,
+		},
+	},
+});
+
 describe('PipelinePage container test', () => {
 	it('should render AddNewPipeline section', () => {
 		const setActionType = jest.fn();
 		const selectedPipelineData = pipelineMockData[0];
 		const isActionType = 'add-pipeline';
+
 		const { asFragment } = render(
 			<MemoryRouter>
-				<Provider store={store}>
+				<QueryClientProvider client={queryClient}>
-					<I18nextProvider i18n={i18n}>
+					<Provider store={store}>
-						<AddNewPipeline
+						<I18nextProvider i18n={i18n}>
-							isActionType={isActionType}
+							<AddNewPipeline
-							setActionType={setActionType}
+								isActionType={isActionType}
-							selectedPipelineData={selectedPipelineData}
+								setActionType={setActionType}
-							setShowSaveButton={jest.fn()}
+								selectedPipelineData={selectedPipelineData}
-							setCurrPipelineData={jest.fn()}
+								setShowSaveButton={jest.fn()}
-							currPipelineData={pipelineMockData}
+								setCurrPipelineData={jest.fn()}
-						/>
+								currPipelineData={pipelineMockData}
-					</I18nextProvider>
+							/>
-				</Provider>
+						</I18nextProvider>
+					</Provider>
+				</QueryClientProvider>
 			</MemoryRouter>,
 		);
 		expect(asFragment()).toMatchSnapshot();

frontend/src/container/PipelinePage/tests/utils.test.ts

@@ -1,4 +1,4 @@
-import { pipelineMockData } from '../mocks/pipeline';
+import { mockPipelineFilter, pipelineMockData } from '../mocks/pipeline';
 import {
 	processorFields,
 	processorTypes,
@@ -68,7 +68,7 @@ describe('Utils testing of Pipeline Page', () => {
 			...pipelineMockData[findRecordIndex],
 			name: 'updated name',
 			description: 'changed description',
-			filter: 'value == test',
+			filter: mockPipelineFilter('value', '=', 'test'),
 			tags: ['test'],
 		};
 		const editedData = getEditedDataSource(

frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx

@@ -41,6 +41,7 @@ function QueryBuilderSearch({
 	onChange,
 	whereClauseConfig,
 	className,
+	placeholder,
 }: QueryBuilderSearchProps): JSX.Element {
 	const {
 		updateTag,
@@ -190,7 +191,7 @@ function QueryBuilderSearch({
 			filterOption={false}
 			autoClearSearchValue={false}
 			mode="multiple"
-			placeholder={PLACEHOLDER}
+			placeholder={placeholder}
 			value={queryTags}
 			searchValue={searchValue}
 			className={className}
@@ -218,11 +219,13 @@ interface QueryBuilderSearchProps {
 	onChange: (value: TagFilter) => void;
 	whereClauseConfig?: WhereClauseConfig;
 	className?: string;
+	placeholder?: string;
 }
 
 QueryBuilderSearch.defaultProps = {
 	whereClauseConfig: undefined,
 	className: '',
+	placeholder: PLACEHOLDER,
 };
 
 export interface CustomTagProps {

frontend/src/hooks/queryBuilder/useTag.ts

@@ -7,10 +7,32 @@ import {
 import { unparse } from 'papaparse';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { useCallback, useEffect, useMemo, useState } from 'react';
-import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
+import {
+	IBuilderQuery,
+	TagFilter,
+} from 'types/api/queryBuilder/queryBuilderData';
 
 import { WhereClauseConfig } from './useAutoComplete';
 
+/**
+ * Helper for formatting a TagFilter object into filter item strings
+ * @param {TagFilter} filters - query filter object to be converted
+ * @returns {string[]} An array of formatted conditions. Eg: `["service = web", "severity_text = INFO"]`)
+ */
+export function queryFilterTags(filter: TagFilter): string[] {
+	return (filter?.items || []).map((ele) => {
+		if (isInNInOperator(getOperatorFromValue(ele.op))) {
+			try {
+				const csvString = unparse([ele.value]);
+				return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${csvString}`;
+			} catch {
+				return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${ele.value}`;
+			}
+		}
+		return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${ele.value}`;
+	});
+}
+
 type IUseTag = {
 	handleAddTag: (value: string) => void;
 	handleClearTag: (value: string) => void;
@@ -33,21 +55,9 @@ export const useTag = (
 	setSearchKey: (value: string) => void,
 	whereClauseConfig?: WhereClauseConfig,
 ): IUseTag => {
-	const initTagsData = useMemo(
+	const initTagsData = useMemo(() => queryFilterTags(query?.filters), [
-		() =>
+		query?.filters,
-			(query?.filters?.items || []).map((ele) => {
+	]);
-				if (isInNInOperator(getOperatorFromValue(ele.op))) {
-					try {
-						const csvString = unparse([ele.value]);
-						return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${csvString}`;
-					} catch {
-						return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${ele.value}`;
-					}
-				}
-				return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${ele.value}`;
-			}),
-		[query.filters],
-	);
 
 	const [tags, setTags] = useState<string[]>(initTagsData);
 

frontend/src/types/api/pipeline/def.ts

@@ -1,3 +1,5 @@
+import { TagFilter } from '../queryBuilder/queryBuilderData';
+
 export interface ProcessorData {
 	type: string;
 	id?: string;
@@ -23,7 +25,7 @@ export interface PipelineData {
 	description?: string;
 	createdBy: string;
 	enabled: boolean;
-	filter: string;
+	filter: TagFilter;
 	id?: string;
 	name: string;
 	orderId: number;