feat: Add Generic Table View in the logs explorer (#2936)

* feat: add dynamic table based on query

* fix: group by repeating

* fix: change view when groupBy exist in the list

* fix: table scroll

* fix: filters for explorer page (#2959)

---------

Co-authored-by: Palash Gupta <palashgdev@gmail.com>

frontend/src/components/ResizeTable/ResizeTable.tsx

@@ -1,14 +1,20 @@
 import { Table } from 'antd';
 import type { TableProps } from 'antd/es/table';
 import { ColumnsType } from 'antd/lib/table';
-import { SyntheticEvent, useCallback, useMemo, useState } from 'react';
+import {
+	SyntheticEvent,
+	useCallback,
+	useEffect,
+	useMemo,
+	useState,
+} from 'react';
 import { ResizeCallbackData } from 'react-resizable';
 
 import ResizableHeader from './ResizableHeader';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function ResizeTable({ columns, ...restprops }: TableProps<any>): JSX.Element {
-	const [columnsData, setColumns] = useState<ColumnsType>(columns || []);
+	const [columnsData, setColumns] = useState<ColumnsType>([]);
 
 	const handleResize = useCallback(
 		(index: number) => (
@@ -37,6 +43,12 @@ function ResizeTable({ columns, ...restprops }: TableProps<any>): JSX.Element {
 		[columnsData, handleResize],
 	);
 
+	useEffect(() => {
+		if (columns) {
+			setColumns(columns);
+		}
+	}, [columns]);
+
 	return (
 		<Table
 			// eslint-disable-next-line react/jsx-props-no-spreading

frontend/src/container/LogsExplorerTable/LogsExplorerTable.tsx

@@ -0,0 +1,44 @@
+import { initialQueriesMap, PANEL_TYPES } from 'constants/queryBuilder';
+import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
+import { QueryTable } from 'container/QueryTable';
+import { useGetPanelTypesQueryParam } from 'hooks/queryBuilder/useGetPanelTypesQueryParam';
+import { useGetQueryRange } from 'hooks/queryBuilder/useGetQueryRange';
+import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
+import { useSelector } from 'react-redux';
+import { AppState } from 'store/reducers';
+import { GlobalReducer } from 'types/reducer/globalTime';
+
+export function LogsExplorerTable(): JSX.Element {
+	const { stagedQuery } = useQueryBuilder();
+
+	const { selectedTime } = useSelector<AppState, GlobalReducer>(
+		(state) => state.globalTime,
+	);
+
+	const panelTypeParam = useGetPanelTypesQueryParam(PANEL_TYPES.LIST);
+
+	const { data, isFetching } = useGetQueryRange(
+		{
+			query: stagedQuery || initialQueriesMap.metrics,
+			graphType: panelTypeParam,
+			globalSelectedInterval: selectedTime,
+			selectedTime: 'GLOBAL_TIME',
+		},
+		{
+			queryKey: [
+				REACT_QUERY_KEY.GET_QUERY_RANGE,
+				selectedTime,
+				stagedQuery,
+				panelTypeParam,
+			],
+			enabled: !!stagedQuery,
+		},
+	);
+	return (
+		<QueryTable
+			query={stagedQuery || initialQueriesMap.metrics}
+			queryTableData={data?.payload.data.newResult.data.result || []}
+			loading={isFetching}
+		/>
+	);
+}

frontend/src/container/LogsExplorerTable/index.ts

@@ -0,0 +1 @@
+export { LogsExplorerTable } from './LogsExplorerTable';

frontend/src/container/LogsExplorerViews/LogsExplorerViews.tsx

@@ -1,6 +1,7 @@
 import { TabsProps } from 'antd';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import { PANEL_TYPES_QUERY } from 'constants/queryBuilderQueryNames';
+import { LogsExplorerTable } from 'container/LogsExplorerTable';
 import { GRAPH_TYPES } from 'container/NewDashboard/ComponentsSlider';
 import { useGetPanelTypesQueryParam } from 'hooks/queryBuilder/useGetPanelTypesQueryParam';
 import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
@@ -25,17 +26,26 @@ export function LogsExplorerViews(): JSX.Element {
 		[currentQuery],
 	);
 
+	const isGroupByExist = useMemo(() => {
+		const groupByCount: number = currentQuery.builder.queryData.reduce<number>(
+			(acc, query) => acc + query.groupBy.length,
+			0,
+		);
+
+		return groupByCount > 0;
+	}, [currentQuery]);
+
 	const tabsItems: TabsProps['items'] = useMemo(
 		() => [
 			{
 				label: 'List View',
 				key: PANEL_TYPES.LIST,
-				disabled: isMultipleQueries,
+				disabled: isMultipleQueries || isGroupByExist,
 			},
 			{ label: 'TimeSeries', key: PANEL_TYPES.TIME_SERIES },
-			{ label: 'Table', key: PANEL_TYPES.TABLE },
+			{ label: 'Table', key: PANEL_TYPES.TABLE, children: <LogsExplorerTable /> },
 		],
-		[isMultipleQueries],
+		[isMultipleQueries, isGroupByExist],
 	);
 
 	const handleChangeView = useCallback(
@@ -57,10 +67,12 @@ export function LogsExplorerViews(): JSX.Element {
 	);
 
 	useEffect(() => {
-		if (panelTypeParams === 'list' && isMultipleQueries) {
+		const shouldChangeView = isMultipleQueries || isGroupByExist;
+
+		if (panelTypeParams === 'list' && shouldChangeView) {
 			handleChangeView(PANEL_TYPES.TIME_SERIES);
 		}
-	}, [panelTypeParams, isMultipleQueries, handleChangeView]);
+	}, [panelTypeParams, isMultipleQueries, isGroupByExist, handleChangeView]);
 
 	return (
 		<div>

frontend/src/container/QueryBuilder/components/Query/Query.tsx

@@ -196,7 +196,54 @@ export const Query = memo(function Query({
 			}
 
 			default: {
-				return null;
+				return (
+					<>
+						<Col span={11}>
+							<Row gutter={[11, 5]}>
+								<Col flex="5.93rem">
+									<FilterLabel label="Limit" />
+								</Col>
+								<Col flex="1 1 12.5rem">
+									<LimitFilter query={query} onChange={handleChangeLimit} />
+								</Col>
+							</Row>
+						</Col>
+						<Col span={11}>
+							<Row gutter={[11, 5]}>
+								<Col flex="5.93rem">
+									<FilterLabel label="HAVING" />
+								</Col>
+								<Col flex="1 1 12.5rem">
+									<HavingFilter onChange={handleChangeHavingFilter} query={query} />
+								</Col>
+							</Row>
+						</Col>
+						<Col span={11}>
+							<Row gutter={[11, 5]}>
+								<Col flex="5.93rem">
+									<FilterLabel label="Order by" />
+								</Col>
+								<Col flex="1 1 12.5rem">
+									<OrderByFilter query={query} onChange={handleChangeOrderByKeys} />
+								</Col>
+							</Row>
+						</Col>
+
+						<Col span={11}>
+							<Row gutter={[11, 5]}>
+								<Col flex="5.93rem">
+									<FilterLabel label="Aggregate Every" />
+								</Col>
+								<Col flex="1 1 6rem">
+									<AggregateEveryFilter
+										query={query}
+										onChange={handleChangeAggregateEvery}
+									/>
+								</Col>
+							</Row>
+						</Col>
+					</>
+				);
 			}
 		}
 	}, [

frontend/src/container/QueryBuilder/filters/GroupByFilter/GroupByFilter.tsx

@@ -102,10 +102,12 @@ export const GroupByFilter = memo(function GroupByFilter({
 
 				return {
 					id,
-					key,
+					key: key || currentValue,
-					dataType: dataType as DataType,
+					dataType: (dataType as DataType) || initialAutocompleteData.dataType,
-					type: type as AutocompleteType,
+					type: (type as AutocompleteType) || initialAutocompleteData.type,
-					isColumn: isColumn === 'true',
+					isColumn: isColumn
+						? isColumn === 'true'
+						: initialAutocompleteData.isColumn,
 				};
 			}
 

frontend/src/container/QueryTable/QueryTable.intefaces.ts

@@ -0,0 +1,14 @@
+import { TableProps } from 'antd';
+import { RowData } from 'lib/query/createTableColumnsFromQuery';
+import { ReactNode } from 'react';
+import { Query } from 'types/api/queryBuilder/queryBuilderData';
+import { QueryDataV3 } from 'types/api/widgets/getQuery';
+
+export type QueryTableProps = Omit<
+	TableProps<RowData>,
+	'columns' | 'dataSource'
+> & {
+	queryTableData: QueryDataV3[];
+	query: Query;
+	renderActionCell?: (record: RowData) => ReactNode;
+};

frontend/src/container/QueryTable/QueryTable.tsx

@@ -0,0 +1,52 @@
+import type { ColumnsType } from 'antd/es/table';
+import { ResizeTable } from 'components/ResizeTable';
+import dayjs from 'dayjs';
+import {
+	createTableColumnsFromQuery,
+	RowData,
+} from 'lib/query/createTableColumnsFromQuery';
+import { useMemo } from 'react';
+
+import { QueryTableProps } from './QueryTable.intefaces';
+
+export function QueryTable({
+	queryTableData,
+	query,
+	renderActionCell,
+	...props
+}: QueryTableProps): JSX.Element {
+	const { columns, dataSource } = useMemo(
+		() =>
+			createTableColumnsFromQuery({
+				query,
+				queryTableData,
+				renderActionCell,
+			}),
+		[query, queryTableData, renderActionCell],
+	);
+
+	const modifiedColumns = useMemo(() => {
+		const currentColumns: ColumnsType<RowData> = columns.map((column) =>
+			column.key === 'timestamp'
+				? {
+						...column,
+						render: (_, record): string =>
+							dayjs(new Date(record.timestamp)).format('MMM DD, YYYY, HH:mm:ss'),
+				  }
+				: column,
+		);
+
+		return currentColumns;
+	}, [columns]);
+
+	return (
+		<ResizeTable
+			columns={modifiedColumns}
+			tableLayout="fixed"
+			dataSource={dataSource}
+			scroll={{ x: true }}
+			// eslint-disable-next-line react/jsx-props-no-spreading
+			{...props}
+		/>
+	);
+}

frontend/src/container/QueryTable/index.ts

@@ -0,0 +1 @@
+export { QueryTable } from './QueryTable';

frontend/src/lib/newQueryBuilder/convertNewDataToOld.ts

@@ -35,5 +35,9 @@ export const convertNewDataToOld = (
 	});
 	const oldResultType = resultType;
 
-	return { data: { result: oldResult, resultType: oldResultType } };
+	// TODO: fix it later for using only v3 version of api
+
+	return {
+		data: { result: oldResult, resultType: oldResultType, newResult: newData },
+	};
 };

frontend/src/lib/query/createTableColumnsFromQuery.ts

@@ -0,0 +1,294 @@
+import { ColumnsType } from 'antd/es/table';
+import { ColumnType } from 'antd/lib/table';
+import { FORMULA_REGEXP } from 'constants/regExp';
+import { QueryTableProps } from 'container/QueryTable/QueryTable.intefaces';
+import { toCapitalize } from 'lib/toCapitalize';
+import { ReactNode } from 'react';
+import { IBuilderQuery, Query } from 'types/api/queryBuilder/queryBuilderData';
+import { QueryDataV3, SeriesItem } from 'types/api/widgets/getQuery';
+import { v4 as uuid } from 'uuid';
+
+type CreateTableDataFromQueryParams = Pick<
+	QueryTableProps,
+	'queryTableData' | 'query' | 'renderActionCell'
+>;
+
+export type RowData = {
+	timestamp: number;
+	key: string;
+	[key: string]: string | number;
+};
+
+type DynamicColumn = {
+	key: keyof RowData;
+	data: (string | number)[];
+	type: 'field' | 'operator';
+	sortable: boolean;
+};
+
+type DynamicColumns = DynamicColumn[];
+
+type CreateTableDataFromQuery = (
+	params: CreateTableDataFromQueryParams,
+) => {
+	columns: ColumnsType<RowData>;
+	dataSource: RowData[];
+	rowsLength: number;
+};
+
+type FillColumnData = (
+	queryTableData: QueryDataV3[],
+	dynamicColumns: DynamicColumns,
+	query: Query,
+) => { filledDynamicColumns: DynamicColumns; rowsLength: number };
+
+type GetDynamicColumns = (
+	queryTableData: QueryDataV3[],
+	query: Query,
+) => DynamicColumns;
+
+const isFormula = (queryName: string): boolean =>
+	FORMULA_REGEXP.test(queryName);
+
+const isColumnExist = (
+	columnName: string,
+	columns: DynamicColumns,
+): boolean => {
+	const columnKeys = columns.map((item) => item.key);
+
+	return columnKeys.includes(columnName);
+};
+
+const prepareColumnTitle = (title: string): string => {
+	const haveUnderscore = title.includes('_');
+
+	if (haveUnderscore) {
+		return title
+			.split('_')
+			.map((str) => toCapitalize(str))
+			.join(' ');
+	}
+
+	return toCapitalize(title);
+};
+
+const getDynamicColumns: GetDynamicColumns = (queryTableData, query) => {
+	const dynamicColumns: DynamicColumns = [];
+
+	queryTableData.forEach((currentQuery) => {
+		if (!currentQuery.series) return;
+
+		if (!isColumnExist('timestamp', dynamicColumns)) {
+			dynamicColumns.push({
+				key: 'timestamp',
+				data: [],
+				type: 'field',
+				sortable: true,
+			});
+		}
+
+		currentQuery.series.forEach((seria) => {
+			Object.keys(seria.labels).forEach((label) => {
+				if (isColumnExist(label, dynamicColumns)) return;
+				if (isFormula(label)) return;
+
+				const labelValue = seria.labels[label];
+
+				const isNumber = !Number.isNaN(parseFloat(labelValue));
+
+				const fieldObj: DynamicColumn = {
+					key: label,
+					data: [],
+					type: 'field',
+					sortable: isNumber,
+				};
+
+				dynamicColumns.push(fieldObj);
+			});
+		});
+
+		if (!isFormula(currentQuery.queryName)) {
+			const builderQuery = query.builder.queryData.find(
+				(q) => q.queryName === currentQuery.queryName,
+			);
+
+			const operator = builderQuery ? builderQuery.aggregateOperator : '';
+
+			if (isColumnExist(operator, dynamicColumns)) return;
+
+			const operatorColumn: DynamicColumn = {
+				key: operator,
+				data: [],
+				type: 'operator',
+				sortable: true,
+			};
+			dynamicColumns.push(operatorColumn);
+		}
+	});
+
+	return dynamicColumns;
+};
+
+const getQueryOperator = (
+	queryData: IBuilderQuery[],
+	currentQueryName: string,
+): string => {
+	const builderQuery = queryData.find((q) => q.queryName === currentQueryName);
+
+	return builderQuery ? builderQuery.aggregateOperator : '';
+};
+
+const fillEmptyRowCells = (
+	unusedColumnsKeys: Set<keyof RowData>,
+	sourceColumns: DynamicColumns,
+	currentColumn: DynamicColumn,
+): void => {
+	unusedColumnsKeys.forEach((key) => {
+		if (key === currentColumn.key) {
+			const unusedCol = sourceColumns.find((item) => item.key === key);
+
+			if (unusedCol) {
+				unusedCol.data.push('N/A');
+				unusedColumnsKeys.delete(key);
+			}
+		}
+	});
+};
+
+const fillDataFromSeria = (
+	seria: SeriesItem,
+	columns: DynamicColumns,
+	currentOperator: string,
+): void => {
+	const labelEntries = Object.entries(seria.labels);
+
+	seria.values.forEach((value) => {
+		const unusedColumnsKeys = new Set<keyof RowData>(
+			columns.map((item) => item.key),
+		);
+
+		columns.forEach((column) => {
+			if (isFormula(column.key as string)) return;
+
+			if (column.key === 'timestamp') {
+				column.data.push(value.timestamp);
+				unusedColumnsKeys.delete('timestamp');
+				return;
+			}
+
+			if (column.key === currentOperator) {
+				column.data.push(parseFloat(value.value).toFixed(2));
+				unusedColumnsKeys.delete(column.key);
+				return;
+			}
+
+			labelEntries.forEach(([key, currentValue]) => {
+				if (column.key === key) {
+					column.data.push(currentValue);
+					unusedColumnsKeys.delete(key);
+				}
+			});
+
+			fillEmptyRowCells(unusedColumnsKeys, columns, column);
+		});
+	});
+};
+
+const fillColumnsData: FillColumnData = (queryTableData, cols, query) => {
+	const fields = cols.filter((item) => item.type === 'field');
+	const operators = cols.filter((item) => item.type === 'operator');
+	const resultColumns = [...fields, ...operators];
+
+	queryTableData.forEach((currentQuery) => {
+		if (!currentQuery.series) return;
+
+		const currentOperator = getQueryOperator(
+			query.builder.queryData,
+			currentQuery.queryName,
+		);
+
+		currentQuery.series.forEach((seria) => {
+			fillDataFromSeria(seria, resultColumns, currentOperator);
+		});
+	});
+
+	const rowsLength = resultColumns.length > 0 ? resultColumns[0].data.length : 0;
+
+	return { filledDynamicColumns: resultColumns, rowsLength };
+};
+
+const generateData = (
+	dynamicColumns: DynamicColumns,
+	rowsLength: number,
+): RowData[] => {
+	const data: RowData[] = [];
+
+	for (let i = 0; i < rowsLength; i += 1) {
+		const rowData: RowData = dynamicColumns.reduce((acc, item) => {
+			const { key } = item;
+
+			acc[key] = item.data[i];
+			acc.key = uuid();
+
+			return acc;
+		}, {} as RowData);
+
+		data.push(rowData);
+	}
+
+	return data;
+};
+
+const generateTableColumns = (
+	dynamicColumns: DynamicColumns,
+): ColumnsType<RowData> => {
+	const columns: ColumnsType<RowData> = dynamicColumns.reduce<
+		ColumnsType<RowData>
+	>((acc, item) => {
+		const column: ColumnType<RowData> = {
+			dataIndex: item.key,
+			key: item.key,
+			title: prepareColumnTitle(item.key as string),
+			sorter: item.sortable
+				? (a: RowData, b: RowData): number =>
+						(a[item.key] as number) - (b[item.key] as number)
+				: false,
+		};
+
+		return [...acc, column];
+	}, []);
+
+	return columns;
+};
+
+export const createTableColumnsFromQuery: CreateTableDataFromQuery = ({
+	query,
+	queryTableData,
+	renderActionCell,
+}) => {
+	const dynamicColumns = getDynamicColumns(queryTableData, query);
+
+	const { filledDynamicColumns, rowsLength } = fillColumnsData(
+		queryTableData,
+		dynamicColumns,
+		query,
+	);
+
+	const dataSource = generateData(filledDynamicColumns, rowsLength);
+
+	const columns = generateTableColumns(filledDynamicColumns);
+
+	const actionsCell: ColumnType<RowData> | null = renderActionCell
+		? {
+				key: 'actions',
+				title: 'Actions',
+				render: (_, record): ReactNode => renderActionCell(record),
+		  }
+		: null;
+
+	if (actionsCell && dataSource.length > 0) {
+		columns.push(actionsCell);
+	}
+
+	return { columns, dataSource, rowsLength };
+};

frontend/src/lib/toCapitalize.ts

@@ -0,0 +1,2 @@
+export const toCapitalize = (str: string): string =>
+	str[0].toUpperCase() + str.slice(1);

frontend/src/types/api/metrics/getQueryRange.ts

@@ -5,6 +5,7 @@ export interface MetricRangePayloadProps {
 	data: {
 		result: QueryData[];
 		resultType: string;
+		newResult: MetricRangePayloadV3;
 	};
 }
 

frontend/src/types/api/widgets/getQuery.ts

@@ -23,7 +23,7 @@ export interface QueryDataV3 {
 	list: null;
 	queryName: string;
 	legend?: string;
-	series: SeriesItem[];
+	series: SeriesItem[] | null;
 }
 
 export interface Props {