fix: query builder edit mode doesn't send correct payload (#2804)

* fix: where clause getting values

* fix: group by filter custom option

* fix: id for group by and aggregate filters

* fix: repeating values

* refactor: group by uniq items

* fix: removing source key

* fix: keep where clause filter on operator change

* chore: clean up for console log and additional variables

---------

Co-authored-by: Chintan Sudani <46838508+techchintan@users.noreply.github.com>
Co-authored-by: Vishal Sharma <makeavish786@gmail.com>
Co-authored-by: Palash Gupta <palashgdev@gmail.com>

frontend/src/api/queryBuilder/getAggregateAttribute.ts

@@ -1,6 +1,8 @@
 import { ApiV3Instance } from 'api';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError, AxiosResponse } from 'axios';
+import { baseAutoCompleteIdKeysOrder } from 'constants/queryBuilder';
+import { createIdFromObjectFields } from 'lib/createIdFromObjectFields';
 import createQueryParams from 'lib/createQueryParams';
 // ** Helpers
 import { ErrorResponse, SuccessResponse } from 'types/api';
@@ -10,7 +12,6 @@ import {
 	BaseAutocompleteData,
 	IQueryAutocompleteResponse,
 } from 'types/api/queryBuilder/queryAutocompleteResponse';
-import { v4 as uuid } from 'uuid';
 
 export const getAggregateAttribute = async ({
 	aggregateOperator,
@@ -31,8 +32,10 @@ export const getAggregateAttribute = async ({
 		);
 
 		const payload: BaseAutocompleteData[] =
-			response.data.data.attributeKeys?.map((item) => ({ ...item, id: uuid() })) ||
+			response.data.data.attributeKeys?.map(({ id: _, ...item }) => ({
-			[];
+				...item,
+				id: createIdFromObjectFields(item, baseAutoCompleteIdKeysOrder),
+			})) || [];
 
 		return {
 			statusCode: 200,

frontend/src/api/queryBuilder/getAttributeKeys.ts

@@ -1,6 +1,8 @@
 import { ApiV3Instance } from 'api';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError, AxiosResponse } from 'axios';
+import { baseAutoCompleteIdKeysOrder } from 'constants/queryBuilder';
+import { createIdFromObjectFields } from 'lib/createIdFromObjectFields';
 import createQueryParams from 'lib/createQueryParams';
 import { ErrorResponse, SuccessResponse } from 'types/api';
 // ** Types
@@ -9,7 +11,6 @@ import {
 	BaseAutocompleteData,
 	IQueryAutocompleteResponse,
 } from 'types/api/queryBuilder/queryAutocompleteResponse';
-import { v4 as uuid } from 'uuid';
 
 export const getAggregateKeys = async ({
 	aggregateOperator,
@@ -33,8 +34,10 @@ export const getAggregateKeys = async ({
 		);
 
 		const payload: BaseAutocompleteData[] =
-			response.data.data.attributeKeys?.map((item) => ({ ...item, id: uuid() })) ||
+			response.data.data.attributeKeys?.map(({ id: _, ...item }) => ({
-			[];
+				...item,
+				id: createIdFromObjectFields(item, baseAutoCompleteIdKeysOrder),
+			})) || [];
 
 		return {
 			statusCode: 200,

frontend/src/constants/queryBuilder.ts

@@ -1,7 +1,10 @@
 // ** Helpers
 import { GRAPH_TYPES } from 'container/NewDashboard/ComponentsSlider';
 import { createNewBuilderItemName } from 'lib/newQueryBuilder/createNewBuilderItemName';
-import { LocalDataType } from 'types/api/queryBuilder/queryAutocompleteResponse';
+import {
+	BaseAutocompleteData,
+	LocalDataType,
+} from 'types/api/queryBuilder/queryAutocompleteResponse';
 import {
 	HavingForm,
 	IBuilderFormula,
@@ -35,7 +38,13 @@ import {
 export const MAX_FORMULAS = 20;
 export const MAX_QUERIES = 26;
 
-export const selectValueDivider = '--';
+export const idDivider = '--';
+export const selectValueDivider = '__';
+
+export const baseAutoCompleteIdKeysOrder: (keyof Omit<
+	BaseAutocompleteData,
+	'id'
+>)[] = ['key', 'dataType', 'type', 'isColumn'];
 
 export const formulasNames: string[] = Array.from(
 	Array(MAX_FORMULAS),
@@ -90,7 +99,7 @@ export const initialHavingValues: HavingForm = {
 	value: [],
 };
 
-export const initialAggregateAttribute: IBuilderQuery['aggregateAttribute'] = {
+export const initialAutocompleteData: BaseAutocompleteData = {
 	id: uuid(),
 	dataType: null,
 	key: '',
@@ -102,7 +111,7 @@ export const initialQueryBuilderFormValues: IBuilderQuery = {
 	dataSource: DataSource.METRICS,
 	queryName: createNewBuilderItemName({ existNames: [], sourceNames: alphabet }),
 	aggregateOperator: MetricAggregateOperator.NOOP,
-	aggregateAttribute: initialAggregateAttribute,
+	aggregateAttribute: initialAutocompleteData,
 	filters: { items: [], op: 'AND' },
 	expression: createNewBuilderItemName({
 		existNames: [],

frontend/src/container/QueryBuilder/filters/AggregatorFilter/AggregatorFilter.tsx

@@ -3,19 +3,25 @@ import { AutoComplete, Spin } from 'antd';
 // ** Api
 import { getAggregateAttribute } from 'api/queryBuilder/getAggregateAttribute';
 import {
-	initialAggregateAttribute,
+	baseAutoCompleteIdKeysOrder,
+	idDivider,
+	initialAutocompleteData,
 	QueryBuilderKeys,
 	selectValueDivider,
 } from 'constants/queryBuilder';
 import useDebounce from 'hooks/useDebounce';
-import { getFilterObjectValue } from 'lib/newQueryBuilder/getFilterObjectValue';
+import { createIdFromObjectFields } from 'lib/createIdFromObjectFields';
 import { transformStringWithPrefix } from 'lib/query/transformStringWithPrefix';
 import { memo, useCallback, useMemo, useState } from 'react';
 import { useQuery } from 'react-query';
+import {
+	AutocompleteType,
+	BaseAutocompleteData,
+	DataType,
+} from 'types/api/queryBuilder/queryAutocompleteResponse';
 import { DataSource } from 'types/common/queryBuilder';
 import { ExtendedSelectOption } from 'types/common/select';
 import { transformToUpperCase } from 'utils/transformToUpperCase';
-import { v4 as uuid } from 'uuid';
 
 import { selectStyle } from '../QueryBuilderSearch/config';
 // ** Types
@@ -27,7 +33,7 @@ export const AggregatorFilter = memo(function AggregatorFilter({
 }: AgregatorFilterProps): JSX.Element {
 	const [optionsData, setOptionsData] = useState<ExtendedSelectOption[]>([]);
 	const debouncedValue = useDebounce(query.aggregateAttribute.key, 300);
-	const { data, isFetching } = useQuery(
+	const { isFetching } = useQuery(
 		[
 			QueryBuilderKeys.GET_AGGREGATE_ATTRIBUTE,
 			debouncedValue,
@@ -44,18 +50,17 @@ export const AggregatorFilter = memo(function AggregatorFilter({
 			enabled: !!query.aggregateOperator && !!query.dataSource,
 			onSuccess: (data) => {
 				const options: ExtendedSelectOption[] =
-					data?.payload?.attributeKeys?.map((item) => ({
+					data?.payload?.attributeKeys?.map(({ id: _, ...item }) => ({
 						label: transformStringWithPrefix({
 							str: item.key,
 							prefix: item.type || '',
 							condition: !item.isColumn,
 						}),
-						value: `${transformStringWithPrefix({
+						value: `${item.key}${selectValueDivider}${createIdFromObjectFields(
-							str: item.key,
+							item,
-							prefix: item.type || '',
+							baseAutoCompleteIdKeysOrder,
-							condition: !item.isColumn,
+						)}`,
-						})}${selectValueDivider}${item.id || uuid()}`,
+						key: createIdFromObjectFields(item, baseAutoCompleteIdKeysOrder),
-						key: item.id || uuid(),
 					})) || [];
 
 				setOptionsData(options);
@@ -70,15 +75,23 @@ export const AggregatorFilter = memo(function AggregatorFilter({
 		): void => {
 			const currentOption = option as ExtendedSelectOption;
 
-			const { key } = getFilterObjectValue(value);
+			if (currentOption.key) {
+				const [key, dataType, type, isColumn] = currentOption.key.split(idDivider);
+				const attribute: BaseAutocompleteData = {
+					key,
+					dataType: dataType as DataType,
+					type: type as AutocompleteType,
+					isColumn: isColumn === 'true',
+				};
 
-			const currentAttributeObj = data?.payload?.attributeKeys?.find(
+				onChange(attribute);
-				(item) => currentOption.key === item.id,
+			} else {
-			) || { ...initialAggregateAttribute, key };
+				const attribute = { ...initialAutocompleteData, key: value };
 
-			onChange(currentAttributeObj);
+				onChange(attribute);
+			}
 		},
-		[data, onChange],
+		[onChange],
 	);
 
 	const value = useMemo(

frontend/src/container/QueryBuilder/filters/GroupByFilter/GroupByFilter.tsx

@@ -1,17 +1,25 @@
 import { Select, Spin } from 'antd';
 import { getAggregateKeys } from 'api/queryBuilder/getAttributeKeys';
 // ** Constants
-import { QueryBuilderKeys, selectValueDivider } from 'constants/queryBuilder';
+import {
+	idDivider,
+	initialAutocompleteData,
+	QueryBuilderKeys,
+	selectValueDivider,
+} from 'constants/queryBuilder';
 import useDebounce from 'hooks/useDebounce';
-import { getFilterObjectValue } from 'lib/newQueryBuilder/getFilterObjectValue';
 // ** Components
 // ** Helpers
 import { transformStringWithPrefix } from 'lib/query/transformStringWithPrefix';
-import { memo, useEffect, useState } from 'react';
+import { isEqual, uniqWith } from 'lodash-es';
+import { memo, useCallback, useEffect, useState } from 'react';
 import { useQuery } from 'react-query';
-import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
+import {
+	AutocompleteType,
+	BaseAutocompleteData,
+	DataType,
+} from 'types/api/queryBuilder/queryAutocompleteResponse';
 import { SelectOption } from 'types/common/select';
-import { v4 as uuid } from 'uuid';
 
 import { selectStyle } from '../QueryBuilderSearch/config';
 import { GroupByFilterProps } from './GroupByFilter.interfaces';
@@ -32,7 +40,7 @@ export const GroupByFilter = memo(function GroupByFilter({
 
 	const debouncedValue = useDebounce(searchText, 300);
 
-	const { data, isFetching } = useQuery(
+	const { isFetching } = useQuery(
 		[QueryBuilderKeys.GET_AGGREGATE_KEYS, debouncedValue, isFocused],
 		async () =>
 			getAggregateKeys({
@@ -65,7 +73,7 @@ export const GroupByFilter = memo(function GroupByFilter({
 							str: item.key,
 							prefix: item.type || '',
 							condition: !item.isColumn,
-						})}${selectValueDivider}${item.id || uuid()}`,
+						})}${selectValueDivider}${item.id}`,
 					})) || [];
 
 				setOptionsData(options);
@@ -87,36 +95,32 @@ export const GroupByFilter = memo(function GroupByFilter({
 	};
 
 	const handleChange = (values: SelectOption<string, string>[]): void => {
-		const responseKeys = data?.payload?.attributeKeys || [];
-
 		const groupByValues: BaseAutocompleteData[] = values.map((item) => {
 			const [currentValue, id] = item.value.split(selectValueDivider);
-			const { key, type, isColumn } = getFilterObjectValue(currentValue);
+			if (id && id.includes(idDivider)) {
+				const [key, dataType, type, isColumn] = id.split(idDivider);
 
-			const existGroupQuery = query.groupBy.find((group) => group.id === id);
+				return {
-
+					id,
-			if (existGroupQuery) {
+					key,
-				return existGroupQuery;
+					dataType: dataType as DataType,
+					type: type as AutocompleteType,
+					isColumn: isColumn === 'true',
+				};
 			}
 
-			const existGroupResponse = responseKeys.find((group) => group.id === id);
+			return { ...initialAutocompleteData, key: currentValue };
-
-			if (existGroupResponse) {
-				return existGroupResponse;
-			}
-
-			return {
-				id: uuid(),
-				isColumn,
-				key,
-				dataType: null,
-				type,
-			};
 		});
 
-		onChange(groupByValues);
+		const result = uniqWith(groupByValues, isEqual);
+
+		onChange(result);
 	};
 
+	const clearSearch = useCallback(() => {
+		setSearchText('');
+	}, []);
+
 	useEffect(() => {
 		const currentValues: SelectOption<string, string>[] = query.groupBy.map(
 			(item) => ({
@@ -129,7 +133,7 @@ export const GroupByFilter = memo(function GroupByFilter({
 					str: item.key,
 					prefix: item.type || '',
 					condition: !item.isColumn,
-				})}${selectValueDivider}${item.id || uuid()}`,
+				})}${selectValueDivider}${item.id}`,
 			}),
 		);
 
@@ -147,6 +151,7 @@ export const GroupByFilter = memo(function GroupByFilter({
 			filterOption={false}
 			onBlur={handleBlur}
 			onFocus={handleFocus}
+			onDeselect={clearSearch}
 			options={optionsData}
 			value={localValues}
 			labelInValue

frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx

@@ -5,9 +5,11 @@ import {
 	KeyboardEvent,
 	ReactElement,
 	ReactNode,
+	useCallback,
 	useEffect,
 	useMemo,
 } from 'react';
+import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
 import {
 	IBuilderQuery,
 	TagFilter,
@@ -44,7 +46,11 @@ function QueryBuilderSearch({
 		searchKey,
 	} = useAutoComplete(query);
 
-	const { sourceKeys } = useFetchKeysAndValues(searchValue, query, searchKey);
+	const { sourceKeys, handleRemoveSourceKey } = useFetchKeysAndValues(
+		searchValue,
+		query,
+		searchKey,
+	);
 
 	const onTagRender = ({
 		value,
@@ -94,6 +100,14 @@ function QueryBuilderSearch({
 		if (isExistsNotExistsOperator(searchValue)) handleKeyDown(event);
 	};
 
+	const handleDeselect = useCallback(
+		(deselectedItem: string) => {
+			handleClearTag(deselectedItem);
+			handleRemoveSourceKey(deselectedItem);
+		},
+		[handleClearTag, handleRemoveSourceKey],
+	);
+
 	const isMetricsDataSource = useMemo(
 		() => query.dataSource === DataSource.METRICS,
 		[query.dataSource],
@@ -106,9 +120,12 @@ function QueryBuilderSearch({
 
 	useEffect(() => {
 		const initialTagFilters: TagFilter = { items: [], op: 'AND' };
+		const initialSourceKeys = query.filters.items.map(
+			(item) => item.key as BaseAutocompleteData,
+		);
 		initialTagFilters.items = tags.map((tag) => {
 			const { tagKey, tagOperator, tagValue } = getTagToken(tag);
-			const filterAttribute = sourceKeys.find(
+			const filterAttribute = [...initialSourceKeys, ...sourceKeys].find(
 				(key) => key.key === getRemovePrefixFromKey(tagKey),
 			);
 			return {
@@ -128,7 +145,7 @@ function QueryBuilderSearch({
 		});
 		onChange(initialTagFilters);
 		/* eslint-disable react-hooks/exhaustive-deps */
-	}, [sourceKeys, tags]);
+	}, [sourceKeys]);
 
 	return (
 		<Select
@@ -146,7 +163,7 @@ function QueryBuilderSearch({
 			onSearch={handleSearch}
 			onChange={onChangeHandler}
 			onSelect={handleSelect}
-			onDeselect={handleClearTag}
+			onDeselect={handleDeselect}
 			onInputKeyDown={onInputKeyDownHandler}
 			notFoundContent={isFetching ? <Spin size="small" /> : null}
 		>

frontend/src/hooks/queryBuilder/useFetchKeysAndValues.ts

@@ -8,7 +8,7 @@ import {
 } from 'container/QueryBuilder/filters/QueryBuilderSearch/utils';
 import { isEqual, uniqWith } from 'lodash-es';
 import debounce from 'lodash-es/debounce';
-import { useEffect, useMemo, useRef, useState } from 'react';
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useQuery } from 'react-query';
 import { useDebounce } from 'react-use';
 import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
@@ -20,6 +20,7 @@ type IuseFetchKeysAndValues = {
 	results: string[];
 	isFetching: boolean;
 	sourceKeys: BaseAutocompleteData[];
+	handleRemoveSourceKey: (newSourceKey: string) => void;
 };
 
 /**
@@ -127,6 +128,12 @@ export const useFetchKeysAndValues = (
 		}
 	};
 
+	const handleRemoveSourceKey = useCallback((sourceKey: string) => {
+		setSourceKeys((prevState) =>
+			prevState.filter((item) => item.key !== sourceKey),
+		);
+	}, []);
+
 	// creates a ref to the fetch function so that it doesn't change on every render
 	const clearFetcher = useRef(handleFetchOption).current;
 
@@ -155,5 +162,6 @@ export const useFetchKeysAndValues = (
 		results,
 		isFetching,
 		sourceKeys,
+		handleRemoveSourceKey,
 	};
 };

frontend/src/hooks/queryBuilder/useQueryOperations.ts

@@ -1,5 +1,5 @@
 import {
-	initialAggregateAttribute,
+	initialAutocompleteData,
 	initialQueryBuilderFormValues,
 	mapOfFilters,
 } from 'constants/queryBuilder';
@@ -44,9 +44,8 @@ export const useQueryOperations: UseQueryOperations = ({ query, index }) => {
 				aggregateOperator: value,
 				having: [],
 				limit: null,
-				filters: { items: [], op: 'AND' },
 				...(shouldResetAggregateAttribute
-					? { aggregateAttribute: initialAggregateAttribute }
+					? { aggregateAttribute: initialAutocompleteData }
 					: {}),
 			};
 

frontend/src/hooks/queryBuilder/useTag.ts

@@ -5,7 +5,7 @@ import {
 } from 'container/QueryBuilder/filters/QueryBuilderSearch/utils';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as Papa from 'papaparse';
-import { useCallback, useEffect, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
 
 type IUseTag = {
@@ -30,7 +30,19 @@ export const useTag = (
 	query: IBuilderQuery,
 	setSearchKey: (value: string) => void,
 ): IUseTag => {
-	const [tags, setTags] = useState<string[]>([]);
+	const initTagsData = useMemo(
+		() =>
+			(query?.filters?.items || []).map((ele) => {
+				if (isInNInOperator(getOperatorFromValue(ele.op))) {
+					const csvString = Papa.unparse([ele.value]);
+					return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${csvString}`;
+				}
+				return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${ele.value}`;
+			}),
+		[query.filters],
+	);
+
+	const [tags, setTags] = useState<string[]>(initTagsData);
 
 	const updateTag = (value: string): void => {
 		const newTags = tags?.filter((item: string) => item !== value);
@@ -61,16 +73,8 @@ export const useTag = (
 	}, []);
 
 	useEffect(() => {
-		const initialTags = (query?.filters?.items || []).map((ele) => {
+		setTags(initTagsData);
-			if (isInNInOperator(getOperatorFromValue(ele.op))) {
+	}, [initTagsData]);
-				const csvString = Papa.unparse([ele.value]);
-				return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${csvString}`;
-			}
-			return `${ele.key?.key} ${getOperatorFromValue(ele.op)} ${ele.value}`;
-		});
-		setTags(initialTags);
-		// eslint-disable-next-line react-hooks/exhaustive-deps
-	}, []);
 
 	return { handleAddTag, handleClearTag, tags, updateTag };
 };

frontend/src/lib/createIdFromObjectFields.ts

@@ -0,0 +1,6 @@
+import { idDivider } from 'constants/queryBuilder';
+
+export const createIdFromObjectFields = <T, K extends keyof T>(
+	obj: T,
+	orderedKeys: K[],
+): string => orderedKeys.map((key) => obj[key]).join(idDivider);

frontend/src/lib/newQueryBuilder/getFilterObjectValue.ts

@@ -1,33 +0,0 @@
-import { TYPE_ADDON_REGEXP } from 'constants/regExp';
-import {
-	AutocompleteType,
-	BaseAutocompleteData,
-} from 'types/api/queryBuilder/queryAutocompleteResponse';
-
-const getType = (str: string): AutocompleteType | null => {
-	const types: AutocompleteType[] = ['tag', 'resource'];
-
-	let currentType: AutocompleteType | null = null;
-
-	types.forEach((type) => {
-		if (str.includes(type)) {
-			currentType = type;
-		}
-	});
-
-	return currentType;
-};
-
-export const getFilterObjectValue = (
-	value: string,
-): Omit<BaseAutocompleteData, 'dataType' | 'id'> => {
-	const type = getType(value);
-
-	if (!type) {
-		return { isColumn: true, key: value, type: null };
-	}
-
-	const splittedValue = value.split(TYPE_ADDON_REGEXP);
-
-	return { isColumn: false, key: splittedValue[1], type };
-};

frontend/src/providers/QueryBuilder.tsx

@@ -1,5 +1,6 @@
 import {
 	alphabet,
+	baseAutoCompleteIdKeysOrder,
 	formulasNames,
 	initialClickHouseData,
 	initialFormulaBuilderFormValues,
@@ -15,6 +16,7 @@ import {
 import { COMPOSITE_QUERY } from 'constants/queryBuilderQueryNames';
 import { GRAPH_TYPES } from 'container/NewDashboard/ComponentsSlider';
 import useUrlQuery from 'hooks/useUrlQuery';
+import { createIdFromObjectFields } from 'lib/createIdFromObjectFields';
 import { createNewBuilderItemName } from 'lib/newQueryBuilder/createNewBuilderItemName';
 import { getOperatorsBySourceAndPanelType } from 'lib/newQueryBuilder/getOperatorsBySourceAndPanelType';
 import { replaceIncorrectObjectFields } from 'lib/replaceIncorrectObjectFields';
@@ -100,6 +102,7 @@ export function QueryBuilderProvider({
 
 	const initQueryBuilderData = useCallback((query: Partial<Query>): void => {
 		const { queryType, ...queryState } = query;
+
 		const builder: QueryBuilderData = {
 			queryData: queryState.builder
 				? queryState.builder.queryData.map((item) => ({
@@ -129,7 +132,27 @@ export function QueryBuilderProvider({
 			  }))
 			: initialQuery.clickhouse_sql;
 
-		setCurrentQuery({ builder, clickhouse_sql: clickHouse, promql });
+		setCurrentQuery({
+			clickhouse_sql: clickHouse,
+			promql,
+			builder: {
+				...builder,
+				queryData: builder.queryData.map((q) => ({
+					...q,
+					groupBy: q.groupBy.map(({ id: _, ...item }) => ({
+						...item,
+						id: createIdFromObjectFields(item, baseAutoCompleteIdKeysOrder),
+					})),
+					aggregateAttribute: {
+						...q.aggregateAttribute,
+						id: createIdFromObjectFields(
+							q.aggregateAttribute,
+							baseAutoCompleteIdKeysOrder,
+						),
+					},
+				})),
+			},
+		});
 
 		setQueryType(queryType || EQueryType.QUERY_BUILDER);
 	}, []);