From c5d23336a7b690979696001fc4870f7016393aa4 Mon Sep 17 00:00:00 2001 From: Vikrant Gupta Date: Thu, 27 Jun 2024 22:04:14 +0530 Subject: [PATCH] chore: move the table calculation to backend (#5351) --- .../GridCard/FullView/index.tsx | 2 + .../GridCardLayout/GridCard/index.tsx | 1 + .../GridTableComponent/__tests__/response.ts | 215 ++++++++++++++++++ .../__tests__/utils.test.tsx | 42 ++++ .../container/GridTableComponent/index.tsx | 45 ++-- .../src/container/GridTableComponent/utils.ts | 89 ++++++++ .../WidgetGraph/WidgetGraphContainer.tsx | 10 +- .../NewWidget/LeftContainer/index.tsx | 6 + .../PanelWrapper/TablePanelWrapper.tsx | 2 +- frontend/src/lib/dashboard/getQueryResults.ts | 7 +- .../lib/dashboard/prepareQueryRangePayload.ts | 2 + .../src/types/api/metrics/getQueryRange.ts | 1 + 12 files changed, 397 insertions(+), 25 deletions(-) create mode 100644 frontend/src/container/GridTableComponent/__tests__/response.ts create mode 100644 frontend/src/container/GridTableComponent/__tests__/utils.test.tsx diff --git a/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx b/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx index 02d71ed9bb..83ba3b7d4b 100644 --- a/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx +++ b/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx @@ -80,6 +80,8 @@ function FullView({ query: updatedQuery, globalSelectedInterval: globalSelectedTime, variables: getDashboardVariables(selectedDashboard?.data.variables), + fillGaps: widget.fillSpans, + formatForWeb: getGraphType(widget.panelTypes) === PANEL_TYPES.TABLE, }; } updatedQuery.builder.queryData[0].pageSize = 10; diff --git a/frontend/src/container/GridCardLayout/GridCard/index.tsx b/frontend/src/container/GridCardLayout/GridCard/index.tsx index 28c67ae92e..2daac8c9b4 100644 --- a/frontend/src/container/GridCardLayout/GridCard/index.tsx +++ b/frontend/src/container/GridCardLayout/GridCard/index.tsx @@ -109,6 +109,7 @@ function GridCardGraph({ globalSelectedInterval, variables: getDashboardVariables(variables), fillGaps: widget.fillSpans, + formatForWeb: getGraphType(widget.panelTypes) === PANEL_TYPES.TABLE, }; } updatedQuery.builder.queryData[0].pageSize = 10; diff --git a/frontend/src/container/GridTableComponent/__tests__/response.ts b/frontend/src/container/GridTableComponent/__tests__/response.ts new file mode 100644 index 0000000000..677a1b18e3 --- /dev/null +++ b/frontend/src/container/GridTableComponent/__tests__/response.ts @@ -0,0 +1,215 @@ +export const tableDataMultipleQueriesSuccessResponse = { + columns: [ + { + name: 'service_name', + queryName: '', + isValueColumn: false, + }, + { + name: 'A', + queryName: 'A', + isValueColumn: true, + }, + { + name: 'B', + queryName: 'B', + isValueColumn: true, + }, + ], + rows: [ + { + data: { + A: 4196.71, + B: 'n/a', + service_name: 'demo-app', + }, + }, + { + data: { + A: 500.83, + B: 'n/a', + service_name: 'customer', + }, + }, + { + data: { + A: 499.5, + B: 'n/a', + service_name: 'mysql', + }, + }, + { + data: { + A: 293.22, + B: 'n/a', + service_name: 'frontend', + }, + }, + { + data: { + A: 230.03, + B: 'n/a', + service_name: 'driver', + }, + }, + { + data: { + A: 67.09, + B: 'n/a', + service_name: 'route', + }, + }, + { + data: { + A: 30.96, + B: 'n/a', + service_name: 'redis', + }, + }, + { + data: { + A: 'n/a', + B: 112.27, + service_name: 'n/a', + }, + }, + ], +}; + +export const widgetQueryWithLegend = { + clickhouse_sql: [ + { + name: 'A', + legend: '', + disabled: false, + query: '', + }, + ], + promql: [ + { + name: 'A', + query: '', + legend: '', + disabled: false, + }, + ], + builder: { + queryData: [ + { + dataSource: 'metrics', + queryName: 'A', + aggregateOperator: 'count', + aggregateAttribute: { + dataType: 'float64', + id: 'signoz_latency--float64--ExponentialHistogram--true', + isColumn: true, + isJSON: false, + key: 'signoz_latency', + type: 'ExponentialHistogram', + }, + timeAggregation: '', + spaceAggregation: 'p90', + functions: [], + filters: { + items: [], + op: 'AND', + }, + expression: 'A', + disabled: false, + stepInterval: 60, + having: [], + limit: null, + orderBy: [], + groupBy: [ + { + dataType: 'string', + isColumn: false, + isJSON: false, + key: 'service_name', + type: 'tag', + id: 'service_name--string--tag--false', + }, + ], + legend: 'p99', + reduceTo: 'avg', + }, + { + dataSource: 'metrics', + queryName: 'B', + aggregateOperator: 'rate', + aggregateAttribute: { + dataType: 'float64', + id: 'system_disk_operations--float64--Sum--true', + isColumn: true, + isJSON: false, + key: 'system_disk_operations', + type: 'Sum', + }, + timeAggregation: 'rate', + spaceAggregation: 'sum', + functions: [], + filters: { + items: [], + op: 'AND', + }, + expression: 'B', + disabled: false, + stepInterval: 60, + having: [], + limit: null, + orderBy: [], + groupBy: [], + legend: '', + reduceTo: 'avg', + }, + ], + queryFormulas: [], + }, + id: '48ad5a67-9a3c-49d4-a886-d7a34f8b875d', + queryType: 'builder', +}; + +export const expectedOutputWithLegends = { + dataSource: [ + { + A: 4196.71, + B: 'n/a', + service_name: 'demo-app', + }, + { + A: 500.83, + B: 'n/a', + service_name: 'customer', + }, + { + A: 499.5, + B: 'n/a', + service_name: 'mysql', + }, + { + A: 293.22, + B: 'n/a', + service_name: 'frontend', + }, + { + A: 230.03, + B: 'n/a', + service_name: 'driver', + }, + { + A: 67.09, + B: 'n/a', + service_name: 'route', + }, + { + A: 30.96, + B: 'n/a', + service_name: 'redis', + }, + { + A: 'n/a', + B: 112.27, + service_name: 'n/a', + }, + ], +}; diff --git a/frontend/src/container/GridTableComponent/__tests__/utils.test.tsx b/frontend/src/container/GridTableComponent/__tests__/utils.test.tsx new file mode 100644 index 0000000000..f0582a51fb --- /dev/null +++ b/frontend/src/container/GridTableComponent/__tests__/utils.test.tsx @@ -0,0 +1,42 @@ +import { Query } from 'types/api/queryBuilder/queryBuilderData'; + +import { createColumnsAndDataSource, getQueryLegend } from '../utils'; +import { + expectedOutputWithLegends, + tableDataMultipleQueriesSuccessResponse, + widgetQueryWithLegend, +} from './response'; + +describe('Table Panel utils', () => { + it('createColumnsAndDataSource function', () => { + const data = tableDataMultipleQueriesSuccessResponse; + const query = widgetQueryWithLegend as Query; + + const { columns, dataSource } = createColumnsAndDataSource(data, query); + + expect(dataSource).toStrictEqual(expectedOutputWithLegends.dataSource); + + // this makes sure that the columns are rendered in the same order as response + expect(columns[0].title).toBe('service_name'); + // the next specifically makes sure that the legends are properly applied in multiple queries + expect(columns[1].title).toBe('p99'); + // this makes sure that the query without a legend takes the title from the query response + expect(columns[2].title).toBe('B'); + + // this is to ensure that the rows properly map to the column data indexes as the dataIndex should be equal to name of the columns + // returned in the response as the rows will be mapped with them + expect((columns[0] as any).dataIndex).toBe('service_name'); + expect((columns[1] as any).dataIndex).toBe('A'); + expect((columns[2] as any).dataIndex).toBe('B'); + }); + + it('getQueryLegend function', () => { + const query = widgetQueryWithLegend as Query; + + // query A has a legend of p99 + expect(getQueryLegend(query, 'A')).toBe('p99'); + + // should return undefined when legend not present + expect(getQueryLegend(query, 'B')).toBe(undefined); + }); +}); diff --git a/frontend/src/container/GridTableComponent/index.tsx b/frontend/src/container/GridTableComponent/index.tsx index 171f09da07..fab4d85e8b 100644 --- a/frontend/src/container/GridTableComponent/index.tsx +++ b/frontend/src/container/GridTableComponent/index.tsx @@ -3,10 +3,7 @@ import { Space, Tooltip } from 'antd'; import { getYAxisFormattedValue } from 'components/Graph/yAxisConfig'; import { Events } from 'constants/events'; import { QueryTable } from 'container/QueryTable'; -import { - createTableColumnsFromQuery, - RowData, -} from 'lib/query/createTableColumnsFromQuery'; +import { RowData } from 'lib/query/createTableColumnsFromQuery'; import { cloneDeep, get, isEmpty, set } from 'lodash-es'; import { memo, ReactNode, useCallback, useEffect, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; @@ -14,7 +11,11 @@ import { eventEmitter } from 'utils/getEventEmitter'; import { WrapperStyled } from './styles'; import { GridTableComponentProps } from './types'; -import { findMatchingThreshold } from './utils'; +import { + createColumnsAndDataSource, + findMatchingThreshold, + TableData, +} from './utils'; function GridTableComponent({ data, @@ -25,28 +26,26 @@ function GridTableComponent({ ...props }: GridTableComponentProps): JSX.Element { const { t } = useTranslation(['valueGraph']); + + // create columns and dataSource in the ui friendly structure + // use the query from the widget here to extract the legend information const { columns, dataSource: originalDataSource } = useMemo( - () => - createTableColumnsFromQuery({ - query, - queryTableData: data, - }), - [data, query], + () => createColumnsAndDataSource((data as unknown) as TableData, query), + [query, data], ); + const createDataInCorrectFormat = useCallback( (dataSource: RowData[]): RowData[] => dataSource.map((d) => { const finalObject = {}; - const keys = Object.keys(d); - keys.forEach((k) => { - const label = get( - columns.find((c) => get(c, 'dataIndex', '') === k) || {}, - 'title', - '', + + // we use the order of the columns here to have similar download as the user view + columns.forEach((k) => { + set( + finalObject, + get(k, 'title', '') as string, + get(d, get(k, 'dataIndex', ''), 'n/a'), ); - if (label) { - set(finalObject, label as string, d[k]); - } }); return finalObject as RowData; }), @@ -65,7 +64,11 @@ function GridTableComponent({ const newValue = { ...val }; Object.keys(val).forEach((k) => { if (columnUnits[k]) { - newValue[k] = getYAxisFormattedValue(String(val[k]), columnUnits[k]); + // the check below takes care of not adding units for rows that have n/a values + newValue[k] = + val[k] !== 'n/a' + ? getYAxisFormattedValue(String(val[k]), columnUnits[k]) + : val[k]; newValue[`${k}_without_unit`] = val[k]; } }); diff --git a/frontend/src/container/GridTableComponent/utils.ts b/frontend/src/container/GridTableComponent/utils.ts index e60cac6a24..089a35fe00 100644 --- a/frontend/src/container/GridTableComponent/utils.ts +++ b/frontend/src/container/GridTableComponent/utils.ts @@ -1,4 +1,11 @@ +import { ColumnsType, ColumnType } from 'antd/es/table'; import { ThresholdProps } from 'container/NewWidget/RightContainer/Threshold/types'; +import { QUERY_TABLE_CONFIG } from 'container/QueryTable/config'; +import { QueryTableProps } from 'container/QueryTable/QueryTable.intefaces'; +import { RowData } from 'lib/query/createTableColumnsFromQuery'; +import { isEmpty, isNaN } from 'lodash-es'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; +import { EQueryType } from 'types/common/dashboard'; // Helper function to evaluate the condition based on the operator function evaluateCondition( @@ -56,3 +63,85 @@ export function findMatchingThreshold( hasMultipleMatches, }; } + +export interface TableData { + columns: { name: string; queryName: string; isValueColumn: boolean }[]; + rows: { data: any }[]; +} + +export function getQueryLegend( + currentQuery: Query, + queryName: string, +): string | undefined { + let legend: string | undefined; + switch (currentQuery.queryType) { + case EQueryType.QUERY_BUILDER: + // check if the value is present in the queries + legend = currentQuery.builder.queryData.find( + (query) => query.queryName === queryName, + )?.legend; + + if (!legend) { + // check if the value is present in the formula + legend = currentQuery.builder.queryFormulas.find( + (query) => query.queryName === queryName, + )?.legend; + } + break; + case EQueryType.CLICKHOUSE: + legend = currentQuery.clickhouse_sql.find( + (query) => query.name === queryName, + )?.legend; + break; + case EQueryType.PROM: + legend = currentQuery.promql.find((query) => query.name === queryName) + ?.legend; + break; + default: + legend = undefined; + break; + } + + return legend; +} + +export function createColumnsAndDataSource( + data: TableData, + currentQuery: Query, + renderColumnCell?: QueryTableProps['renderColumnCell'], +): { columns: ColumnsType; dataSource: RowData[] } { + const columns: ColumnsType = + data.columns?.reduce>((acc, item) => { + // is the column is the value column then we need to check for the available legend + const legend = item.isValueColumn + ? getQueryLegend(currentQuery, item.queryName) + : undefined; + + const column: ColumnType = { + dataIndex: item.name, + // if no legend present then rely on the column name value + title: !isEmpty(legend) ? legend : item.name, + width: QUERY_TABLE_CONFIG.width, + render: renderColumnCell && renderColumnCell[item.name], + sorter: (a: RowData, b: RowData): number => { + const valueA = Number(a[`${item.name}_without_unit`] ?? a[item.name]); + const valueB = Number(b[`${item.name}_without_unit`] ?? b[item.name]); + + if (!isNaN(valueA) && !isNaN(valueB)) { + return valueA - valueB; + } + + return ((a[item.name] as string) || '').localeCompare( + (b[item.name] as string) || '', + ); + }, + }; + + return [...acc, column]; + }, []) || []; + + // the rows returned have data encapsulation hence removing the same here + const dataSource = data.rows?.map((d) => d.data) || []; + + return { columns, dataSource }; +} diff --git a/frontend/src/container/NewWidget/LeftContainer/WidgetGraph/WidgetGraphContainer.tsx b/frontend/src/container/NewWidget/LeftContainer/WidgetGraph/WidgetGraphContainer.tsx index d0b69fcd8d..14a2419915 100644 --- a/frontend/src/container/NewWidget/LeftContainer/WidgetGraph/WidgetGraphContainer.tsx +++ b/frontend/src/container/NewWidget/LeftContainer/WidgetGraph/WidgetGraphContainer.tsx @@ -2,8 +2,6 @@ import { Card, Typography } from 'antd'; import Spinner from 'components/Spinner'; import { PANEL_TYPES } from 'constants/queryBuilder'; import { WidgetGraphContainerProps } from 'container/NewWidget/types'; -// import useUrlQuery from 'hooks/useUrlQuery'; -// import { useDashboard } from 'providers/Dashboard/Dashboard'; import { getSortedSeriesData } from 'utils/getSortedSeriesData'; import { NotFoundContainer } from './styles'; @@ -59,6 +57,14 @@ function WidgetGraphContainer({ ); } + if (queryResponse.isIdle) { + return ( + + No Data + + ); + } + return ( ; params?: Record; fillGaps?: boolean; + formatForWeb?: boolean; tableParams?: { pagination?: Pagination; selectColumns?: any; diff --git a/frontend/src/lib/dashboard/prepareQueryRangePayload.ts b/frontend/src/lib/dashboard/prepareQueryRangePayload.ts index 244e096079..181a83914b 100644 --- a/frontend/src/lib/dashboard/prepareQueryRangePayload.ts +++ b/frontend/src/lib/dashboard/prepareQueryRangePayload.ts @@ -16,6 +16,7 @@ export const prepareQueryRangePayload = ({ query, globalSelectedInterval, graphType, + formatForWeb, selectedTime, tableParams, variables = {}, @@ -102,6 +103,7 @@ export const prepareQueryRangePayload = ({ inputFormat: 'ns', }), variables, + formatForWeb, compositeQuery, ...restParams, }; diff --git a/frontend/src/types/api/metrics/getQueryRange.ts b/frontend/src/types/api/metrics/getQueryRange.ts index 5409eba346..47c8fcba4a 100644 --- a/frontend/src/types/api/metrics/getQueryRange.ts +++ b/frontend/src/types/api/metrics/getQueryRange.ts @@ -24,6 +24,7 @@ export type QueryRangePayload = { start: number; step: number; variables?: Record; + formatForWeb?: boolean; [param: string]: unknown; }; export interface MetricRangePayloadProps {