From a393ea4d686377cb53a0b572e0fe25f557cdda97 Mon Sep 17 00:00:00 2001 From: CKanishka Date: Tue, 22 Aug 2023 23:08:52 +0530 Subject: [PATCH 01/21] fix(FE/logs): enable horizontal scroll in logs context --- .../src/components/Logs/RawLogView/index.tsx | 11 ++++++++++- .../src/components/Logs/RawLogView/styles.ts | 16 ++++++++++------ frontend/src/container/LogsContextList/index.tsx | 8 +++++++- .../src/container/LogsExplorerContext/index.tsx | 12 ++++++++++-- .../src/container/LogsExplorerContext/styles.ts | 4 ++++ 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/Logs/RawLogView/index.tsx b/frontend/src/components/Logs/RawLogView/index.tsx index e926e73643..42c7aae4ae 100644 --- a/frontend/src/components/Logs/RawLogView/index.tsx +++ b/frontend/src/components/Logs/RawLogView/index.tsx @@ -37,12 +37,19 @@ const convert = new Convert(); interface RawLogViewProps { isActiveLog?: boolean; isReadOnly?: boolean; + isTextOverflowEllipsisDisabled?: boolean; data: ILog; linesPerRow: number; } function RawLogView(props: RawLogViewProps): JSX.Element { - const { isActiveLog = false, isReadOnly = false, data, linesPerRow } = props; + const { + isActiveLog = false, + isReadOnly = false, + data, + linesPerRow, + isTextOverflowEllipsisDisabled = false, + } = props; const { isHighlighted, isLogsExplorerPage, onLogCopy } = useCopyLogLink( data.id, @@ -143,6 +150,7 @@ function RawLogView(props: RawLogViewProps): JSX.Element { @@ -181,6 +189,7 @@ function RawLogView(props: RawLogViewProps): JSX.Element { RawLogView.defaultProps = { isActiveLog: false, isReadOnly: false, + isTextOverflowEllipsisDisabled: false, }; export default RawLogView; diff --git a/frontend/src/components/Logs/RawLogView/styles.ts b/frontend/src/components/Logs/RawLogView/styles.ts index b4be783a2a..e47e736d61 100644 --- a/frontend/src/components/Logs/RawLogView/styles.ts +++ b/frontend/src/components/Logs/RawLogView/styles.ts @@ -35,6 +35,7 @@ interface RawLogContentProps { linesPerRow: number; $isReadOnly: boolean; $isActiveLog: boolean; + $isTextOverflowEllipsisDisabled: boolean; } export const RawLogContent = styled.div` @@ -42,12 +43,15 @@ export const RawLogContent = styled.div` font-family: Fira Code, monospace; font-weight: 300; - overflow: hidden; - text-overflow: ellipsis; - display: -webkit-box; - -webkit-line-clamp: ${(props): number => props.linesPerRow}; - line-clamp: ${(props): number => props.linesPerRow}; - -webkit-box-orient: vertical; + ${(props): string => + props.$isTextOverflowEllipsisDisabled + ? 'white-space: nowrap' + : `overflow: hidden; + text-overflow: ellipsis; + display: -webkit-box; + -webkit-line-clamp: ${props.linesPerRow}; + line-clamp: ${props.linesPerRow}; + -webkit-box-orient: vertical;`}; font-size: 1rem; line-height: 2rem; diff --git a/frontend/src/container/LogsContextList/index.tsx b/frontend/src/container/LogsContextList/index.tsx index 7037a386aa..25e0414498 100644 --- a/frontend/src/container/LogsContextList/index.tsx +++ b/frontend/src/container/LogsContextList/index.tsx @@ -154,7 +154,13 @@ function LogsContextList({ const getItemContent = useCallback( (_: number, log: ILog): JSX.Element => ( - + ), [], ); diff --git a/frontend/src/container/LogsExplorerContext/index.tsx b/frontend/src/container/LogsExplorerContext/index.tsx index 746148f3d1..0631c78ea6 100644 --- a/frontend/src/container/LogsExplorerContext/index.tsx +++ b/frontend/src/container/LogsExplorerContext/index.tsx @@ -9,7 +9,7 @@ import { useIsDarkMode } from 'hooks/useDarkMode'; import { memo, useCallback, useMemo, useState } from 'react'; import { Query, TagFilter } from 'types/api/queryBuilder/queryBuilderData'; -import { EditButton, TitleWrapper } from './styles'; +import { EditButton, LogContainer, TitleWrapper } from './styles'; import { LogsExplorerContextProps } from './types'; import useInitialQuery from './useInitialQuery'; @@ -96,7 +96,15 @@ function LogsExplorerContext({ // eslint-disable-next-line react/jsx-props-no-spreading {...contextListParams} /> - + + + ` ? getAlphaColor(themeColors.white)[45] : getAlphaColor(themeColors.black)[45]}; `; + +export const LogContainer = styled.div` + overflow-x: auto; +`; From d41805a3b0e7572d95fde6e3304b664487234d7a Mon Sep 17 00:00:00 2001 From: CKanishka Date: Wed, 23 Aug 2023 11:43:16 +0530 Subject: [PATCH 02/21] refactor(FE/RawLogView): prop destructure at param level --- .../src/components/Logs/RawLogView/index.tsx | 16 +++++------- .../src/components/Logs/RawLogView/styles.ts | 26 +++++++++---------- frontend/src/utils/logs.ts | 4 +-- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/frontend/src/components/Logs/RawLogView/index.tsx b/frontend/src/components/Logs/RawLogView/index.tsx index 42c7aae4ae..194ce53d28 100644 --- a/frontend/src/components/Logs/RawLogView/index.tsx +++ b/frontend/src/components/Logs/RawLogView/index.tsx @@ -42,15 +42,13 @@ interface RawLogViewProps { linesPerRow: number; } -function RawLogView(props: RawLogViewProps): JSX.Element { - const { - isActiveLog = false, - isReadOnly = false, - data, - linesPerRow, - isTextOverflowEllipsisDisabled = false, - } = props; - +function RawLogView({ + isActiveLog, + isReadOnly, + data, + linesPerRow, + isTextOverflowEllipsisDisabled, +}: RawLogViewProps): JSX.Element { const { isHighlighted, isLogsExplorerPage, onLogCopy } = useCopyLogLink( data.id, ); diff --git a/frontend/src/components/Logs/RawLogView/styles.ts b/frontend/src/components/Logs/RawLogView/styles.ts index e47e736d61..8e1394af89 100644 --- a/frontend/src/components/Logs/RawLogView/styles.ts +++ b/frontend/src/components/Logs/RawLogView/styles.ts @@ -5,8 +5,8 @@ import { getActiveLogBackground, getDefaultLogBackground } from 'utils/logs'; export const RawLogViewContainer = styled(Row)<{ $isDarkMode: boolean; - $isReadOnly: boolean; - $isActiveLog: boolean; + $isReadOnly?: boolean; + $isActiveLog?: boolean; }>` position: relative; width: 100%; @@ -33,9 +33,9 @@ export const ExpandIconWrapper = styled(Col)` interface RawLogContentProps { linesPerRow: number; - $isReadOnly: boolean; - $isActiveLog: boolean; - $isTextOverflowEllipsisDisabled: boolean; + $isReadOnly?: boolean; + $isActiveLog?: boolean; + $isTextOverflowEllipsisDisabled?: boolean; } export const RawLogContent = styled.div` @@ -43,24 +43,24 @@ export const RawLogContent = styled.div` font-family: Fira Code, monospace; font-weight: 300; - ${(props): string => - props.$isTextOverflowEllipsisDisabled + ${({ $isTextOverflowEllipsisDisabled, linesPerRow }): string => + $isTextOverflowEllipsisDisabled ? 'white-space: nowrap' : `overflow: hidden; text-overflow: ellipsis; display: -webkit-box; - -webkit-line-clamp: ${props.linesPerRow}; - line-clamp: ${props.linesPerRow}; + -webkit-line-clamp: ${linesPerRow}; + line-clamp: ${linesPerRow}; -webkit-box-orient: vertical;`}; font-size: 1rem; line-height: 2rem; - cursor: ${(props): string => - props.$isActiveLog || props.$isReadOnly ? 'initial' : 'pointer'}; + cursor: ${({ $isActiveLog, $isReadOnly }): string => + $isActiveLog || $isReadOnly ? 'initial' : 'pointer'}; - ${(props): string => - props.$isReadOnly && !props.$isActiveLog ? 'padding: 0 1.5rem;' : ''} + ${({ $isActiveLog, $isReadOnly }): string => + $isReadOnly && $isActiveLog ? 'padding: 0 1.5rem;' : ''} `; export const ActionButtonsWrapper = styled(Space)` diff --git a/frontend/src/utils/logs.ts b/frontend/src/utils/logs.ts index 66a6ba28a6..bfe79a3177 100644 --- a/frontend/src/utils/logs.ts +++ b/frontend/src/utils/logs.ts @@ -3,8 +3,8 @@ import { themeColors } from 'constants/theme'; import getAlphaColor from 'utils/getAlphaColor'; export const getDefaultLogBackground = ( - isReadOnly: boolean, - isDarkMode: boolean, + isReadOnly?: boolean, + isDarkMode?: boolean, ): string => { if (isReadOnly) return ''; return `&:hover { From a37476a09b35f5e4ba0cc5d440775b20e1d43aba Mon Sep 17 00:00:00 2001 From: CKanishka Date: Wed, 23 Aug 2023 11:49:05 +0530 Subject: [PATCH 03/21] refactor(FE/RawLogView): move types to seperate file --- .../src/components/Logs/RawLogView/index.tsx | 11 +---------- .../src/components/Logs/RawLogView/styles.ts | 9 ++------- frontend/src/components/Logs/RawLogView/types.ts | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 frontend/src/components/Logs/RawLogView/types.ts diff --git a/frontend/src/components/Logs/RawLogView/index.tsx b/frontend/src/components/Logs/RawLogView/index.tsx index 194ce53d28..7c630a2939 100644 --- a/frontend/src/components/Logs/RawLogView/index.tsx +++ b/frontend/src/components/Logs/RawLogView/index.tsx @@ -21,8 +21,6 @@ import { useMemo, useState, } from 'react'; -// interfaces -import { ILog } from 'types/api/logs/log'; // styles import { @@ -31,17 +29,10 @@ import { RawLogContent, RawLogViewContainer, } from './styles'; +import { RawLogViewProps } from './types'; const convert = new Convert(); -interface RawLogViewProps { - isActiveLog?: boolean; - isReadOnly?: boolean; - isTextOverflowEllipsisDisabled?: boolean; - data: ILog; - linesPerRow: number; -} - function RawLogView({ isActiveLog, isReadOnly, diff --git a/frontend/src/components/Logs/RawLogView/styles.ts b/frontend/src/components/Logs/RawLogView/styles.ts index 8e1394af89..4944d05f74 100644 --- a/frontend/src/components/Logs/RawLogView/styles.ts +++ b/frontend/src/components/Logs/RawLogView/styles.ts @@ -3,6 +3,8 @@ import { Col, Row, Space } from 'antd'; import styled from 'styled-components'; import { getActiveLogBackground, getDefaultLogBackground } from 'utils/logs'; +import { RawLogContentProps } from './types'; + export const RawLogViewContainer = styled(Row)<{ $isDarkMode: boolean; $isReadOnly?: boolean; @@ -31,13 +33,6 @@ export const ExpandIconWrapper = styled(Col)` font-size: 12px; `; -interface RawLogContentProps { - linesPerRow: number; - $isReadOnly?: boolean; - $isActiveLog?: boolean; - $isTextOverflowEllipsisDisabled?: boolean; -} - export const RawLogContent = styled.div` margin-bottom: 0; font-family: Fira Code, monospace; diff --git a/frontend/src/components/Logs/RawLogView/types.ts b/frontend/src/components/Logs/RawLogView/types.ts new file mode 100644 index 0000000000..2c70c0ddb2 --- /dev/null +++ b/frontend/src/components/Logs/RawLogView/types.ts @@ -0,0 +1,16 @@ +import { ILog } from 'types/api/logs/log'; + +export interface RawLogViewProps { + isActiveLog?: boolean; + isReadOnly?: boolean; + isTextOverflowEllipsisDisabled?: boolean; + data: ILog; + linesPerRow: number; +} + +export interface RawLogContentProps { + linesPerRow: number; + $isReadOnly?: boolean; + $isActiveLog?: boolean; + $isTextOverflowEllipsisDisabled?: boolean; +} From 02cd069bb2d9a0af3465d1c258bed762dde997b1 Mon Sep 17 00:00:00 2001 From: Yevhen Shevchenko <90138953+yeshev@users.noreply.github.com> Date: Wed, 23 Aug 2023 19:44:33 +0300 Subject: [PATCH 04/21] fix: export table panel (#3403) * fix: export table panel * fix: create generate link helper --------- Co-authored-by: Vishal Sharma Co-authored-by: Palash Gupta --- frontend/src/constants/panelTypes.ts | 5 ++++ .../src/container/LogsExplorerViews/index.tsx | 24 ++++++++++++------- frontend/src/hooks/dashboard/utils.ts | 3 ++- frontend/src/pages/TracesExplorer/index.tsx | 24 +++++++++++-------- .../generateExportToDashboardLink.ts | 23 ++++++++++++++++++ 5 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 frontend/src/utils/dashboard/generateExportToDashboardLink.ts diff --git a/frontend/src/constants/panelTypes.ts b/frontend/src/constants/panelTypes.ts index 811e06f0d0..93d66e5019 100644 --- a/frontend/src/constants/panelTypes.ts +++ b/frontend/src/constants/panelTypes.ts @@ -12,3 +12,8 @@ export const PANEL_TYPES_COMPONENT_MAP = { [PANEL_TYPES.LIST]: null, [PANEL_TYPES.EMPTY_WIDGET]: null, } as const; + +export const AVAILABLE_EXPORT_PANEL_TYPES = [ + PANEL_TYPES.TIME_SERIES, + PANEL_TYPES.TABLE, +]; diff --git a/frontend/src/container/LogsExplorerViews/index.tsx b/frontend/src/container/LogsExplorerViews/index.tsx index 51244e7959..fac0441486 100644 --- a/frontend/src/container/LogsExplorerViews/index.tsx +++ b/frontend/src/container/LogsExplorerViews/index.tsx @@ -1,6 +1,6 @@ import { Tabs, TabsProps } from 'antd'; import TabLabel from 'components/TabLabel'; -import { QueryParams } from 'constants/query'; +import { AVAILABLE_EXPORT_PANEL_TYPES } from 'constants/panelTypes'; import { initialAutocompleteData, initialFilters, @@ -9,7 +9,6 @@ import { PANEL_TYPES, } from 'constants/queryBuilder'; import { queryParamNamesMap } from 'constants/queryBuilderQueryNames'; -import ROUTES from 'constants/routes'; import { DEFAULT_PER_PAGE_VALUE } from 'container/Controls/config'; import ExportPanel from 'container/ExportPanel'; import GoToTop from 'container/GoToTop'; @@ -30,7 +29,7 @@ import useUrlQueryData from 'hooks/useUrlQueryData'; import { getPaginationQueryData } from 'lib/newQueryBuilder/getPaginationQueryData'; import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useSelector } from 'react-redux'; -import { generatePath, useHistory } from 'react-router-dom'; +import { useHistory } from 'react-router-dom'; import { AppState } from 'store/reducers'; import { Dashboard } from 'types/api/dashboard/getAll'; import { ILog } from 'types/api/logs/log'; @@ -42,6 +41,7 @@ import { } from 'types/api/queryBuilder/queryBuilderData'; import { DataSource, StringOperators } from 'types/common/queryBuilder'; import { GlobalReducer } from 'types/reducer/globalTime'; +import { generateExportToDashboardLink } from 'utils/dashboard/generateExportToDashboardLink'; import { ActionsWrapper } from './LogsExplorerViews.styled'; @@ -299,11 +299,16 @@ function LogsExplorerViews(): JSX.Element { const handleExport = useCallback( (dashboard: Dashboard | null): void => { - if (!dashboard) return; + if (!dashboard || !panelType) return; + + const panelTypeParam = AVAILABLE_EXPORT_PANEL_TYPES.includes(panelType) + ? panelType + : PANEL_TYPES.TIME_SERIES; const updatedDashboard = addEmptyWidgetInDashboardJSONWithQuery( dashboard, exportDefaultQuery, + panelTypeParam, ); updateDashboard(updatedDashboard, { @@ -332,11 +337,11 @@ function LogsExplorerViews(): JSX.Element { return; } - const dashboardEditView = `${generatePath(ROUTES.DASHBOARD, { - dashboardId: data?.payload?.uuid, - })}/new?${QueryParams.graphType}=graph&${QueryParams.widgetId}=empty&${ - queryParamNamesMap.compositeQuery - }=${encodeURIComponent(JSON.stringify(exportDefaultQuery))}`; + const dashboardEditView = generateExportToDashboardLink({ + query: exportDefaultQuery, + panelType: panelTypeParam, + dashboardId: data.payload?.uuid || '', + }); history.push(dashboardEditView); }, @@ -347,6 +352,7 @@ function LogsExplorerViews(): JSX.Element { exportDefaultQuery, history, notifications, + panelType, updateDashboard, handleAxisError, ], diff --git a/frontend/src/hooks/dashboard/utils.ts b/frontend/src/hooks/dashboard/utils.ts index ea2457daa8..84fa59332d 100644 --- a/frontend/src/hooks/dashboard/utils.ts +++ b/frontend/src/hooks/dashboard/utils.ts @@ -5,6 +5,7 @@ import { Query } from 'types/api/queryBuilder/queryBuilderData'; export const addEmptyWidgetInDashboardJSONWithQuery = ( dashboard: Dashboard, query: Query, + panelTypes?: PANEL_TYPES, ): Dashboard => ({ ...dashboard, data: { @@ -30,7 +31,7 @@ export const addEmptyWidgetInDashboardJSONWithQuery = ( opacity: '', title: '', timePreferance: 'GLOBAL_TIME', - panelTypes: PANEL_TYPES.TIME_SERIES, + panelTypes: panelTypes || PANEL_TYPES.TIME_SERIES, }, ], }, diff --git a/frontend/src/pages/TracesExplorer/index.tsx b/frontend/src/pages/TracesExplorer/index.tsx index 6cb81093f3..6a89825fd1 100644 --- a/frontend/src/pages/TracesExplorer/index.tsx +++ b/frontend/src/pages/TracesExplorer/index.tsx @@ -1,14 +1,13 @@ import { Tabs } from 'antd'; import axios from 'axios'; import ExplorerCard from 'components/ExplorerCard'; -import { QueryParams } from 'constants/query'; +import { AVAILABLE_EXPORT_PANEL_TYPES } from 'constants/panelTypes'; import { initialAutocompleteData, initialQueriesMap, PANEL_TYPES, } from 'constants/queryBuilder'; import { queryParamNamesMap } from 'constants/queryBuilderQueryNames'; -import ROUTES from 'constants/routes'; import ExportPanel from 'container/ExportPanel'; import { SIGNOZ_VALUE } from 'container/QueryBuilder/filters/OrderByFilter/constants'; import QuerySection from 'container/TracesExplorer/QuerySection'; @@ -19,10 +18,10 @@ import { useShareBuilderUrl } from 'hooks/queryBuilder/useShareBuilderUrl'; import { useNotifications } from 'hooks/useNotifications'; import history from 'lib/history'; import { useCallback, useEffect, useMemo } from 'react'; -import { generatePath } from 'react-router-dom'; import { Dashboard } from 'types/api/dashboard/getAll'; import { Query } from 'types/api/queryBuilder/queryBuilderData'; import { DataSource } from 'types/common/queryBuilder'; +import { generateExportToDashboardLink } from 'utils/dashboard/generateExportToDashboardLink'; import { ActionsWrapper, Container } from './styles'; import { getTabsItems } from './utils'; @@ -95,11 +94,16 @@ function TracesExplorer(): JSX.Element { const handleExport = useCallback( (dashboard: Dashboard | null): void => { - if (!dashboard) return; + if (!dashboard || !panelType) return; + + const panelTypeParam = AVAILABLE_EXPORT_PANEL_TYPES.includes(panelType) + ? panelType + : PANEL_TYPES.TIME_SERIES; const updatedDashboard = addEmptyWidgetInDashboardJSONWithQuery( dashboard, exportDefaultQuery, + panelTypeParam, ); updateDashboard(updatedDashboard, { @@ -127,11 +131,11 @@ function TracesExplorer(): JSX.Element { return; } - const dashboardEditView = `${generatePath(ROUTES.DASHBOARD, { - dashboardId: data?.payload?.uuid, - })}/new?${QueryParams.graphType}=graph&${QueryParams.widgetId}=empty&${ - queryParamNamesMap.compositeQuery - }=${encodeURIComponent(JSON.stringify(exportDefaultQuery))}`; + const dashboardEditView = generateExportToDashboardLink({ + query: exportDefaultQuery, + panelType: panelTypeParam, + dashboardId: data.payload?.uuid || '', + }); history.push(dashboardEditView); }, @@ -144,7 +148,7 @@ function TracesExplorer(): JSX.Element { }, }); }, - [exportDefaultQuery, notifications, updateDashboard], + [exportDefaultQuery, notifications, panelType, updateDashboard], ); const getUpdateQuery = useCallback( diff --git a/frontend/src/utils/dashboard/generateExportToDashboardLink.ts b/frontend/src/utils/dashboard/generateExportToDashboardLink.ts new file mode 100644 index 0000000000..c8b85166af --- /dev/null +++ b/frontend/src/utils/dashboard/generateExportToDashboardLink.ts @@ -0,0 +1,23 @@ +import { QueryParams } from 'constants/query'; +import { PANEL_TYPES } from 'constants/queryBuilder'; +import { queryParamNamesMap } from 'constants/queryBuilderQueryNames'; +import ROUTES from 'constants/routes'; +import { generatePath } from 'react-router-dom'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; + +type GenerateExportToDashboardLinkParams = { + dashboardId: string; + panelType: PANEL_TYPES; + query: Query; +}; + +export const generateExportToDashboardLink = ({ + query, + dashboardId, + panelType, +}: GenerateExportToDashboardLinkParams): string => + `${generatePath(ROUTES.DASHBOARD, { + dashboardId, + })}/new?${QueryParams.graphType}=${panelType}&${QueryParams.widgetId}=empty&${ + queryParamNamesMap.compositeQuery + }=${encodeURIComponent(JSON.stringify(query))}`; From 4d14416a08a0a7a49518b28317a2fe2acf3ce4a3 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Thu, 24 Aug 2023 11:09:51 +0530 Subject: [PATCH 05/21] fix: update apdex query to use placeholder parameters (#3409) --- pkg/query-service/dao/sqlite/apdex.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pkg/query-service/dao/sqlite/apdex.go b/pkg/query-service/dao/sqlite/apdex.go index 8c74553fb8..65c1eae350 100644 --- a/pkg/query-service/dao/sqlite/apdex.go +++ b/pkg/query-service/dao/sqlite/apdex.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/jmoiron/sqlx" "go.signoz.io/signoz/pkg/query-service/model" ) @@ -11,19 +12,16 @@ const defaultApdexThreshold = 0.5 func (mds *ModelDaoSqlite) GetApdexSettings(ctx context.Context, services []string) ([]model.ApdexSettings, *model.ApiError) { var apdexSettings []model.ApdexSettings - var serviceName string - for i, service := range services { - if i == 0 { - serviceName = fmt.Sprintf("'%s'", service) - } else { - serviceName = fmt.Sprintf("%s, '%s'", serviceName, service) + query, args, err := sqlx.In("SELECT * FROM apdex_settings WHERE service_name IN (?)", services) + if err != nil { + return nil, &model.ApiError{ + Err: err, } } + query = mds.db.Rebind(query) - query := fmt.Sprintf("SELECT * FROM apdex_settings WHERE service_name IN (%s)", serviceName) - - err := mds.db.Select(&apdexSettings, query) + err = mds.db.Select(&apdexSettings, query, args...) if err != nil { return nil, &model.ApiError{ Err: err, From 598e71eb8eca1a62b33771ee5aa434dd278082eb Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Thu, 24 Aug 2023 11:32:06 +0530 Subject: [PATCH 06/21] chore: remove ee import (#3425) --- pkg/query-service/integrations/signozio/dynamic_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query-service/integrations/signozio/dynamic_config.go b/pkg/query-service/integrations/signozio/dynamic_config.go index 42827b73d5..fec8414f7a 100644 --- a/pkg/query-service/integrations/signozio/dynamic_config.go +++ b/pkg/query-service/integrations/signozio/dynamic_config.go @@ -6,8 +6,8 @@ import ( "net/http" "time" - "go.signoz.io/signoz/ee/query-service/model" "go.signoz.io/signoz/pkg/query-service/constants" + "go.signoz.io/signoz/pkg/query-service/model" ) var C *Client From 0bee0a6d90cd04a27eeb4db4f62923acc1b8de91 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Thu, 24 Aug 2023 12:14:16 +0530 Subject: [PATCH 07/21] fix: update dashboards to use placeholder params (#3408) --- pkg/query-service/app/dashboards/model.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/query-service/app/dashboards/model.go b/pkg/query-service/app/dashboards/model.go index 398b151399..2403775cb8 100644 --- a/pkg/query-service/app/dashboards/model.go +++ b/pkg/query-service/app/dashboards/model.go @@ -49,7 +49,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating dashboard table: %s", err.Error()) + return nil, fmt.Errorf("error in creating dashboard table: %s", err.Error()) } table_schema = `CREATE TABLE IF NOT EXISTS rules ( @@ -61,7 +61,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating rules table: %s", err.Error()) + return nil, fmt.Errorf("error in creating rules table: %s", err.Error()) } table_schema = `CREATE TABLE IF NOT EXISTS notification_channels ( @@ -76,7 +76,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating notification_channles table: %s", err.Error()) + return nil, fmt.Errorf("error in creating notification_channles table: %s", err.Error()) } table_schema = `CREATE TABLE IF NOT EXISTS ttl_status ( @@ -92,7 +92,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating ttl_status table: %s", err.Error()) + return nil, fmt.Errorf("error in creating ttl_status table: %s", err.Error()) } return db, nil @@ -179,7 +179,7 @@ func CreateDashboard(data map[string]interface{}, fm interfaces.FeatureLookup) ( func GetDashboards() ([]Dashboard, *model.ApiError) { dashboards := []Dashboard{} - query := fmt.Sprintf("SELECT * FROM dashboards;") + query := `SELECT * FROM dashboards` err := db.Select(&dashboards, query) if err != nil { @@ -197,9 +197,9 @@ func DeleteDashboard(uuid string, fm interfaces.FeatureLookup) *model.ApiError { return dErr } - query := fmt.Sprintf("DELETE FROM dashboards WHERE uuid='%s';", uuid) + query := `DELETE FROM dashboards WHERE uuid=?` - result, err := db.Exec(query) + result, err := db.Exec(query, uuid) if err != nil { return &model.ApiError{Typ: model.ErrorExec, Err: err} @@ -224,9 +224,9 @@ func DeleteDashboard(uuid string, fm interfaces.FeatureLookup) *model.ApiError { func GetDashboard(uuid string) (*Dashboard, *model.ApiError) { dashboard := Dashboard{} - query := fmt.Sprintf("SELECT * FROM dashboards WHERE uuid='%s';", uuid) + query := `SELECT * FROM dashboards WHERE uuid=?` - err := db.Get(&dashboard, query) + err := db.Get(&dashboard, query, uuid) if err != nil { return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)} } From 7586b50c5a7360b2c494afde159a1e6de975ed39 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Thu, 24 Aug 2023 17:14:42 +0530 Subject: [PATCH 08/21] fix: use query-service API to fetch triggered alerts (#3417) --- deploy/docker-swarm/common/nginx-config.conf | 4 ---- deploy/docker/common/nginx-config.conf | 4 ---- frontend/src/api/alerts/getTriggered.ts | 10 +++++----- pkg/query-service/app/http_handler.go | 21 ++++++++++++++++++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/deploy/docker-swarm/common/nginx-config.conf b/deploy/docker-swarm/common/nginx-config.conf index 158effc8bf..f7943e21aa 100644 --- a/deploy/docker-swarm/common/nginx-config.conf +++ b/deploy/docker-swarm/common/nginx-config.conf @@ -24,10 +24,6 @@ server { try_files $uri $uri/ /index.html; } - location /api/alertmanager { - proxy_pass http://alertmanager:9093/api/v2; - } - location ~ ^/api/(v1|v3)/logs/(tail|livetail){ proxy_pass http://query-service:8080; proxy_http_version 1.1; diff --git a/deploy/docker/common/nginx-config.conf b/deploy/docker/common/nginx-config.conf index 158effc8bf..f7943e21aa 100644 --- a/deploy/docker/common/nginx-config.conf +++ b/deploy/docker/common/nginx-config.conf @@ -24,10 +24,6 @@ server { try_files $uri $uri/ /index.html; } - location /api/alertmanager { - proxy_pass http://alertmanager:9093/api/v2; - } - location ~ ^/api/(v1|v3)/logs/(tail|livetail){ proxy_pass http://query-service:8080; proxy_http_version 1.1; diff --git a/frontend/src/api/alerts/getTriggered.ts b/frontend/src/api/alerts/getTriggered.ts index 160b9a3b93..6955cc315c 100644 --- a/frontend/src/api/alerts/getTriggered.ts +++ b/frontend/src/api/alerts/getTriggered.ts @@ -1,4 +1,4 @@ -import { AxiosAlertManagerInstance } from 'api'; +import axios from 'api'; import { ErrorResponseHandler } from 'api/ErrorResponseHandler'; import { AxiosError } from 'axios'; import convertObjectIntoParams from 'lib/query/convertObjectIntoParams'; @@ -11,15 +11,15 @@ const getTriggered = async ( try { const queryParams = convertObjectIntoParams(props); - const response = await AxiosAlertManagerInstance.get( - `/alerts?${queryParams}`, - ); + const response = await axios.get(`/alerts?${queryParams}`); + + const amData = JSON.parse(response.data.data); return { statusCode: 200, error: null, message: response.data.status, - payload: response.data, + payload: amData.data, }; } catch (error) { return ErrorResponseHandler(error as AxiosError); diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 46029a9e39..fa55154ac2 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -319,6 +319,8 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) { router.HandleFunc("/api/v1/channels", am.EditAccess(aH.createChannel)).Methods(http.MethodPost) router.HandleFunc("/api/v1/testChannel", am.EditAccess(aH.testChannel)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/alerts", am.ViewAccess(aH.getAlerts)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rules", am.ViewAccess(aH.listRules)).Methods(http.MethodGet) router.HandleFunc("/api/v1/rules/{id}", am.ViewAccess(aH.getRule)).Methods(http.MethodGet) router.HandleFunc("/api/v1/rules", am.EditAccess(aH.createRule)).Methods(http.MethodPost) @@ -1195,6 +1197,25 @@ func (aH *APIHandler) createChannel(w http.ResponseWriter, r *http.Request) { } +func (aH *APIHandler) getAlerts(w http.ResponseWriter, r *http.Request) { + params := r.URL.Query() + amEndpoint := constants.GetAlertManagerApiPrefix() + resp, err := http.Get(amEndpoint + "v1/alerts" + "?" + params.Encode()) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) + return + } + + aH.Respond(w, string(body)) +} + func (aH *APIHandler) createRule(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() From 25fc7b83ec745985a3498ace009d4c77009eeae9 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Fri, 25 Aug 2023 09:22:46 +0530 Subject: [PATCH 09/21] chore: update saved views endpoints (#3314) --- pkg/query-service/app/explorer/db.go | 191 +++++++++++++++++++------- pkg/query-service/app/http_handler.go | 59 ++++---- pkg/query-service/model/v3/v3.go | 22 +-- 3 files changed, 185 insertions(+), 87 deletions(-) diff --git a/pkg/query-service/app/explorer/db.go b/pkg/query-service/app/explorer/db.go index d7b1193508..7e520abfe8 100644 --- a/pkg/query-service/app/explorer/db.go +++ b/pkg/query-service/app/explorer/db.go @@ -1,26 +1,32 @@ package explorer import ( + "context" "encoding/json" "fmt" + "strings" "time" "github.com/google/uuid" "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/auth" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) var db *sqlx.DB -type ExplorerQuery struct { +type SavedView struct { UUID string `json:"uuid" db:"uuid"` + Name string `json:"name" db:"name"` + Category string `json:"category" db:"category"` CreatedAt time.Time `json:"created_at" db:"created_at"` + CreatedBy string `json:"created_by" db:"created_by"` UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + UpdatedBy string `json:"updated_by" db:"updated_by"` SourcePage string `json:"source_page" db:"source_page"` - // 0 - false, 1 - true - IsView int8 `json:"is_view" db:"is_view"` - Data string `json:"data" db:"data"` - ExtraData string `json:"extra_data" db:"extra_data"` + Tags string `json:"tags" db:"tags"` + Data string `json:"data" db:"data"` + ExtraData string `json:"extra_data" db:"extra_data"` } // InitWithDSN sets up setting up the connection pool global variable. @@ -32,19 +38,23 @@ func InitWithDSN(dataSourceName string) (*sqlx.DB, error) { return nil, err } - tableSchema := `CREATE TABLE IF NOT EXISTS explorer_queries ( + tableSchema := `CREATE TABLE IF NOT EXISTS saved_views ( uuid TEXT PRIMARY KEY, + name TEXT NOT NULL, + category TEXT NOT NULL, created_at datetime NOT NULL, + created_by TEXT, updated_at datetime NOT NULL, + updated_by TEXT, source_page TEXT NOT NULL, - is_view INTEGER NOT NULL, + tags TEXT, data TEXT NOT NULL, extra_data TEXT );` _, err = db.Exec(tableSchema) if err != nil { - return nil, fmt.Errorf("Error in creating explorer queries table: %s", err.Error()) + return nil, fmt.Errorf("error in creating saved views table: %s", err.Error()) } return db, nil @@ -54,38 +64,79 @@ func InitWithDB(sqlDB *sqlx.DB) { db = sqlDB } -func GetQueries() ([]*v3.ExplorerQuery, error) { - var queries []ExplorerQuery - err := db.Select(&queries, "SELECT * FROM explorer_queries") +func GetViews() ([]*v3.SavedView, error) { + var views []SavedView + err := db.Select(&views, "SELECT * FROM saved_views") if err != nil { - return nil, fmt.Errorf("Error in getting explorer queries: %s", err.Error()) + return nil, fmt.Errorf("error in getting saved views: %s", err.Error()) } - var explorerQueries []*v3.ExplorerQuery - for _, query := range queries { + var savedViews []*v3.SavedView + for _, view := range views { var compositeQuery v3.CompositeQuery - err = json.Unmarshal([]byte(query.Data), &compositeQuery) + err = json.Unmarshal([]byte(view.Data), &compositeQuery) if err != nil { - return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error()) + return nil, fmt.Errorf("error in unmarshalling explorer query data: %s", err.Error()) } - explorerQueries = append(explorerQueries, &v3.ExplorerQuery{ - UUID: query.UUID, - SourcePage: query.SourcePage, + savedViews = append(savedViews, &v3.SavedView{ + UUID: view.UUID, + Name: view.Name, + Category: view.Category, + CreatedAt: view.CreatedAt, + CreatedBy: view.CreatedBy, + UpdatedAt: view.UpdatedAt, + UpdatedBy: view.UpdatedBy, + Tags: strings.Split(view.Tags, ","), + SourcePage: view.SourcePage, CompositeQuery: &compositeQuery, - IsView: query.IsView, - ExtraData: query.ExtraData, + ExtraData: view.ExtraData, }) } - return explorerQueries, nil + return savedViews, nil } -func CreateQuery(query v3.ExplorerQuery) (string, error) { - data, err := json.Marshal(query.CompositeQuery) +func GetViewsForFilters(sourcePage string, name string, category string) ([]*v3.SavedView, error) { + var views []SavedView + var err error + if len(category) == 0 { + err = db.Select(&views, "SELECT * FROM saved_views WHERE source_page = ? AND name LIKE ?", sourcePage, "%"+name+"%") + } else { + err = db.Select(&views, "SELECT * FROM saved_views WHERE source_page = ? AND category LIKE ? AND name LIKE ?", sourcePage, "%"+category+"%", "%"+name+"%") + } if err != nil { - return "", fmt.Errorf("Error in marshalling explorer query data: %s", err.Error()) + return nil, fmt.Errorf("error in getting saved views: %s", err.Error()) } - uuid_ := query.UUID + var savedViews []*v3.SavedView + for _, view := range views { + var compositeQuery v3.CompositeQuery + err = json.Unmarshal([]byte(view.Data), &compositeQuery) + if err != nil { + return nil, fmt.Errorf("error in unmarshalling explorer query data: %s", err.Error()) + } + savedViews = append(savedViews, &v3.SavedView{ + UUID: view.UUID, + Name: view.Name, + CreatedAt: view.CreatedAt, + CreatedBy: view.CreatedBy, + UpdatedAt: view.UpdatedAt, + UpdatedBy: view.UpdatedBy, + Tags: strings.Split(view.Tags, ","), + SourcePage: view.SourcePage, + CompositeQuery: &compositeQuery, + ExtraData: view.ExtraData, + }) + } + return savedViews, nil +} + +func CreateView(ctx context.Context, view v3.SavedView) (string, error) { + data, err := json.Marshal(view.CompositeQuery) + if err != nil { + return "", fmt.Errorf("error in marshalling explorer query data: %s", err.Error()) + } + + uuid_ := view.UUID if uuid_ == "" { uuid_ = uuid.New().String() @@ -93,63 +144,101 @@ func CreateQuery(query v3.ExplorerQuery) (string, error) { createdAt := time.Now() updatedAt := time.Now() + email, err := getEmailFromJwt(ctx) + if err != nil { + return "", err + } + + createBy := email + updatedBy := email + _, err = db.Exec( - "INSERT INTO explorer_queries (uuid, created_at, updated_at, source_page, is_view, data, extra_data) VALUES (?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO saved_views (uuid, name, category, created_at, created_by, updated_at, updated_by, source_page, tags, data, extra_data) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", uuid_, + view.Name, + view.Category, createdAt, + createBy, updatedAt, - query.SourcePage, - query.IsView, + updatedBy, + view.SourcePage, + strings.Join(view.Tags, ","), data, - query.ExtraData, + view.ExtraData, ) if err != nil { - return "", fmt.Errorf("Error in creating explorer query: %s", err.Error()) + return "", fmt.Errorf("error in creating saved view: %s", err.Error()) } return uuid_, nil } -func GetQuery(uuid_ string) (*v3.ExplorerQuery, error) { - var query ExplorerQuery - err := db.Get(&query, "SELECT * FROM explorer_queries WHERE uuid = ?", uuid_) +func GetView(uuid_ string) (*v3.SavedView, error) { + var view SavedView + err := db.Get(&view, "SELECT * FROM saved_views WHERE uuid = ?", uuid_) if err != nil { - return nil, fmt.Errorf("Error in getting explorer query: %s", err.Error()) + return nil, fmt.Errorf("error in getting saved view: %s", err.Error()) } var compositeQuery v3.CompositeQuery - err = json.Unmarshal([]byte(query.Data), &compositeQuery) + err = json.Unmarshal([]byte(view.Data), &compositeQuery) if err != nil { - return nil, fmt.Errorf("Error in unmarshalling explorer query data: %s", err.Error()) + return nil, fmt.Errorf("error in unmarshalling explorer query data: %s", err.Error()) } - return &v3.ExplorerQuery{ - UUID: query.UUID, - SourcePage: query.SourcePage, + return &v3.SavedView{ + UUID: view.UUID, + Name: view.Name, + Category: view.Category, + CreatedAt: view.CreatedAt, + CreatedBy: view.CreatedBy, + UpdatedAt: view.UpdatedAt, + UpdatedBy: view.UpdatedBy, + SourcePage: view.SourcePage, + Tags: strings.Split(view.Tags, ","), CompositeQuery: &compositeQuery, - IsView: query.IsView, - ExtraData: query.ExtraData, + ExtraData: view.ExtraData, }, nil } -func UpdateQuery(uuid_ string, query v3.ExplorerQuery) error { - data, err := json.Marshal(query.CompositeQuery) +func UpdateView(ctx context.Context, uuid_ string, view v3.SavedView) error { + data, err := json.Marshal(view.CompositeQuery) if err != nil { - return fmt.Errorf("Error in marshalling explorer query data: %s", err.Error()) + return fmt.Errorf("error in marshalling explorer query data: %s", err.Error()) + } + + email, err := getEmailFromJwt(ctx) + if err != nil { + return err } updatedAt := time.Now() + updatedBy := email - _, err = db.Exec("UPDATE explorer_queries SET updated_at = ?, source_page = ?, is_view = ?, data = ?, extra_data = ? WHERE uuid = ?", - updatedAt, query.SourcePage, query.IsView, data, query.ExtraData, uuid_) + _, err = db.Exec("UPDATE saved_views SET updated_at = ?, updated_by = ?, name = ?, category = ?, source_page = ?, tags = ?, data = ?, extra_data = ? WHERE uuid = ?", + updatedAt, updatedBy, view.Name, view.Category, view.SourcePage, strings.Join(view.Tags, ","), data, view.ExtraData, uuid_) if err != nil { - return fmt.Errorf("Error in updating explorer query: %s", err.Error()) + return fmt.Errorf("error in updating saved view: %s", err.Error()) } return nil } -func DeleteQuery(uuid_ string) error { - _, err := db.Exec("DELETE FROM explorer_queries WHERE uuid = ?", uuid_) +func DeleteView(uuid_ string) error { + _, err := db.Exec("DELETE FROM saved_views WHERE uuid = ?", uuid_) if err != nil { - return fmt.Errorf("Error in deleting explorer query: %s", err.Error()) + return fmt.Errorf("error in deleting explorer query: %s", err.Error()) } return nil } + +func getEmailFromJwt(ctx context.Context) (string, error) { + jwt, err := auth.ExtractJwtFromContext(ctx) + if err != nil { + return "", err + } + + claims, err := auth.ParseJWT(jwt) + if err != nil { + return "", err + } + + return claims["email"].(string), nil +} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index fa55154ac2..45ba2ea970 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -338,11 +338,11 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) { router.HandleFunc("/api/v1/variables/query", am.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet) router.HandleFunc("/api/v2/variables/query", am.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/explorer/queries", am.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/explorer/queries", am.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/explorer/views", am.ViewAccess(aH.getSavedViews)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/views", am.ViewAccess(aH.createSavedViews)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/explorer/views/{viewId}", am.ViewAccess(aH.getSavedView)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/views/{viewId}", am.ViewAccess(aH.updateSavedView)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/explorer/views/{viewId}", am.ViewAccess(aH.deleteSavedView)).Methods(http.MethodDelete) router.HandleFunc("/api/v1/feedback", am.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost) // router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet) @@ -2522,8 +2522,13 @@ func (ah *APIHandler) createLogsPipeline(w http.ResponseWriter, r *http.Request) ah.Respond(w, res) } -func (aH *APIHandler) getExplorerQueries(w http.ResponseWriter, r *http.Request) { - queries, err := explorer.GetQueries() +func (aH *APIHandler) getSavedViews(w http.ResponseWriter, r *http.Request) { + // get sourcePage, name, and category from the query params + sourcePage := r.URL.Query().Get("sourcePage") + name := r.URL.Query().Get("name") + category := r.URL.Query().Get("category") + + queries, err := explorer.GetViewsForFilters(sourcePage, name, category) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -2531,19 +2536,20 @@ func (aH *APIHandler) getExplorerQueries(w http.ResponseWriter, r *http.Request) aH.Respond(w, queries) } -func (aH *APIHandler) createExplorerQueries(w http.ResponseWriter, r *http.Request) { - var query v3.ExplorerQuery - err := json.NewDecoder(r.Body).Decode(&query) +func (aH *APIHandler) createSavedViews(w http.ResponseWriter, r *http.Request) { + var view v3.SavedView + err := json.NewDecoder(r.Body).Decode(&view) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) return } // validate the query - if err := query.Validate(); err != nil { + if err := view.Validate(); err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) return } - uuid, err := explorer.CreateQuery(query) + ctx := auth.AttachJwtToContext(context.Background(), r) + uuid, err := explorer.CreateView(ctx, view) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -2552,44 +2558,45 @@ func (aH *APIHandler) createExplorerQueries(w http.ResponseWriter, r *http.Reque aH.Respond(w, uuid) } -func (aH *APIHandler) getExplorerQuery(w http.ResponseWriter, r *http.Request) { - queryID := mux.Vars(r)["queryId"] - query, err := explorer.GetQuery(queryID) +func (aH *APIHandler) getSavedView(w http.ResponseWriter, r *http.Request) { + viewID := mux.Vars(r)["viewId"] + view, err := explorer.GetView(viewID) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return } - aH.Respond(w, query) + aH.Respond(w, view) } -func (aH *APIHandler) updateExplorerQuery(w http.ResponseWriter, r *http.Request) { - queryID := mux.Vars(r)["queryId"] - var query v3.ExplorerQuery - err := json.NewDecoder(r.Body).Decode(&query) +func (aH *APIHandler) updateSavedView(w http.ResponseWriter, r *http.Request) { + viewID := mux.Vars(r)["viewId"] + var view v3.SavedView + err := json.NewDecoder(r.Body).Decode(&view) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) return } // validate the query - if err := query.Validate(); err != nil { + if err := view.Validate(); err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) return } - err = explorer.UpdateQuery(queryID, query) + ctx := auth.AttachJwtToContext(context.Background(), r) + err = explorer.UpdateView(ctx, viewID, view) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return } - aH.Respond(w, query) + aH.Respond(w, view) } -func (aH *APIHandler) deleteExplorerQuery(w http.ResponseWriter, r *http.Request) { +func (aH *APIHandler) deleteSavedView(w http.ResponseWriter, r *http.Request) { - queryID := mux.Vars(r)["queryId"] - err := explorer.DeleteQuery(queryID) + viewID := mux.Vars(r)["viewId"] + err := explorer.DeleteView(viewID) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index 55242199b3..d46c644eb4 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -638,24 +638,26 @@ func (p *Point) UnmarshalJSON(data []byte) error { return err } -// ExploreQuery is a query for the explore page -// It is a composite query with a source page name +// SavedView is a saved query for the explore page +// It is a composite query with a source page name and user defined tags // The source page name is used to identify the page that initiated the query -// The source page could be "traces", "logs", "metrics" or "dashboards", "alerts" etc. -type ExplorerQuery struct { +// The source page could be "traces", "logs", "metrics". +type SavedView struct { UUID string `json:"uuid,omitempty"` + Name string `json:"name"` + Category string `json:"category"` + CreatedAt time.Time `json:"createdAt"` + CreatedBy string `json:"createdBy"` + UpdatedAt time.Time `json:"updatedAt"` + UpdatedBy string `json:"updatedBy"` SourcePage string `json:"sourcePage"` + Tags []string `json:"tags"` CompositeQuery *CompositeQuery `json:"compositeQuery"` // ExtraData is JSON encoded data used by frontend to store additional data ExtraData string `json:"extraData"` - // 0 - false, 1 - true; this is int8 because sqlite doesn't support bool - IsView int8 `json:"isView"` } -func (eq *ExplorerQuery) Validate() error { - if eq.IsView != 0 && eq.IsView != 1 { - return fmt.Errorf("isView must be 0 or 1") - } +func (eq *SavedView) Validate() error { if eq.CompositeQuery == nil { return fmt.Errorf("composite query is required") From 988dd1bcf063898eac1fea2cddc998b06a483983 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Mon, 28 Aug 2023 15:48:39 +0530 Subject: [PATCH 10/21] fix: exists, nexists support for top level logs columns (#3434) * fix: exists, nexists support for top level logs columns * fix: dont check value for exists and nexists * fix: exists, nexists updated for materialized columns * fix: remove unnecesary variable * fix: exists check updated in all places --- .../app/logs/v3/query_builder.go | 66 ++++++++--- .../app/logs/v3/query_builder_test.go | 109 +++++++++++++----- 2 files changed, 132 insertions(+), 43 deletions(-) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index cfe67046ba..a8227512a4 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -123,23 +123,59 @@ func getSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attribu return strings.Join(selectLabels, ",") } -func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey) (string, error) { +func GetExistsNexistsFilter(op v3.FilterOperator, item v3.FilterItem) string { + if item.Key.Type == v3.AttributeKeyTypeUnspecified { + top := "!=" + if op == v3.FilterOperatorNotExists { + top = "=" + } + if val, ok := constants.StaticFieldsLogsV3[item.Key.Key]; ok { + // skip for timestamp and id + if val.Key == "" { + return "" + } + + columnName := getClickhouseColumnName(item.Key) + if val.DataType == v3.AttributeKeyDataTypeString { + return fmt.Sprintf("%s %s ''", columnName, top) + } else { + // we just have two types, number and string + return fmt.Sprintf("%s %s 0", columnName, top) + } + } + + } else if item.Key.IsColumn { + val := true + if op == v3.FilterOperatorNotExists { + val = false + } + return fmt.Sprintf("%s_exists=%v", getClickhouseColumnName(item.Key), val) + } + columnType := getClickhouseLogsColumnType(item.Key.Type) + columnDataType := getClickhouseLogsColumnDataType(item.Key.DataType) + return fmt.Sprintf(logOperators[op], columnType, columnDataType, item.Key.Key) +} + +func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey) (string, error) { var conditions []string if fs != nil && len(fs.Items) != 0 { for _, item := range fs.Items { op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) - value, err := utils.ValidateAndCastValue(item.Value, item.Key.DataType) - if err != nil { - return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) + + var value interface{} + var err error + if op != v3.FilterOperatorExists && op != v3.FilterOperatorNotExists { + value, err = utils.ValidateAndCastValue(item.Value, item.Key.DataType) + if err != nil { + return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err) + } } if logsOp, ok := logOperators[op]; ok { switch op { case v3.FilterOperatorExists, v3.FilterOperatorNotExists: - columnType := getClickhouseLogsColumnType(item.Key.Type) - columnDataType := getClickhouseLogsColumnDataType(item.Key.DataType) - conditions = append(conditions, fmt.Sprintf(logsOp, columnType, columnDataType, item.Key.Key)) + conditions = append(conditions, GetExistsNexistsFilter(op, item)) case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: columnName := getClickhouseColumnName(item.Key) fmtVal := utils.ClickHouseFormattedValue(value) @@ -170,6 +206,12 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey) } } + // add conditions for aggregate attribute + if aggregateAttribute.Key != "" { + existsFilter := GetExistsNexistsFilter(v3.FilterOperatorExists, v3.FilterItem{Key: aggregateAttribute}) + conditions = append(conditions, existsFilter) + } + queryString := strings.Join(conditions, " AND ") if len(queryString) > 0 { @@ -180,7 +222,7 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey) func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.BuilderQuery, graphLimitQtype string, preferRPM bool) (string, error) { - filterSubQuery, err := buildLogsTimeSeriesFilterQuery(mq.Filters, mq.GroupBy) + filterSubQuery, err := buildLogsTimeSeriesFilterQuery(mq.Filters, mq.GroupBy, mq.AggregateAttribute) if err != nil { return "", err } @@ -280,12 +322,6 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCount: - if mq.AggregateAttribute.Key != "" { - columnType := getClickhouseLogsColumnType(mq.AggregateAttribute.Type) - columnDataType := getClickhouseLogsColumnDataType(mq.AggregateAttribute.DataType) - filterSubQuery = fmt.Sprintf("%s AND has(%s_%s_key, '%s')", filterSubQuery, columnType, columnDataType, mq.AggregateAttribute.Key) - } - op := "toFloat64(count(*))" query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil @@ -303,7 +339,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build } func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { - filterSubQuery, err := buildLogsTimeSeriesFilterQuery(mq.Filters, mq.GroupBy) + filterSubQuery, err := buildLogsTimeSeriesFilterQuery(mq.Filters, mq.GroupBy, v3.AttributeKey{}) if err != nil { return "", err } diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index bb8c6e670b..5cc0b319a2 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -233,12 +233,55 @@ var timeSeriesFilterQueryData = []struct { }}, ExpectedFilter: " AND attributes_string_value[indexOf(attributes_string_key, 'body')] ILIKE '%test%'", }, + { + Name: "Test exists on top level field", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Operator: "exists"}, + }}, + ExpectedFilter: " AND trace_id != ''", + }, + { + Name: "Test not exists on top level field", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Operator: "nexists"}, + }}, + ExpectedFilter: " AND span_id = ''", + }, + { + Name: "Test exists on top level field number", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "trace_flags", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Operator: "exists"}, + }}, + ExpectedFilter: " AND trace_flags != 0", + }, + { + Name: "Test not exists on top level field number", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "severity_number", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Operator: "nexists"}, + }}, + ExpectedFilter: " AND severity_number = 0", + }, + { + Name: "Test exists on materiazlied column", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: "exists"}, + }}, + ExpectedFilter: " AND attribute_string_method_exists=true", + }, + { + Name: "Test nexists on materiazlied column", + FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "status", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: "nexists"}, + }}, + ExpectedFilter: " AND attribute_int64_status_exists=false", + }, + // add new tests } func TestBuildLogsTimeSeriesFilterQuery(t *testing.T) { for _, tt := range timeSeriesFilterQueryData { Convey("TestBuildLogsTimeSeriesFilterQuery", t, func() { - query, err := buildLogsTimeSeriesFilterQuery(tt.FilterSet, tt.GroupBy) + query, err := buildLogsTimeSeriesFilterQuery(tt.FilterSet, tt.GroupBy, v3.AttributeKey{}) if tt.Error != "" { So(err.Error(), ShouldEqual, tt.Error) } else { @@ -319,13 +362,13 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) group by ts order by value ASC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attribute_string_name))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attribute_string_name_exists=true group by ts order by value ASC", }, { Name: "Test aggregate count distinct on non selected field", @@ -340,7 +383,7 @@ var testBuildLogsQueryData = []struct { Expression: "A", }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) group by ts order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(attributes_string_key, 'name') group by ts order by value DESC", }, { Name: "Test aggregate count distinct with filter and groupBy", @@ -350,7 +393,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -364,10 +407,11 @@ var testBuildLogsQueryData = []struct { TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts," + " attributes_string_value[indexOf(attributes_string_key, 'method')] as method, " + - "toFloat64(count(distinct(name))) as value from signoz_logs.distributed_logs " + + "toFloat64(count(distinct(attribute_string_name))) as value from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND resources_string_value[indexOf(resources_string_key, 'x')] != 'abc' " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND attribute_string_name_exists=true " + "group by method,ts " + "order by method ASC", }, @@ -379,7 +423,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorCountDistinct, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -394,11 +438,12 @@ var testBuildLogsQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts," + " attributes_string_value[indexOf(attributes_string_key, 'method')] as method, " + "resources_string_value[indexOf(resources_string_key, 'x')] as x, " + - "toFloat64(count(distinct(name))) as value from signoz_logs.distributed_logs " + + "toFloat64(count(distinct(attribute_string_name))) as value from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND resources_string_value[indexOf(resources_string_key, 'x')] != 'abc' " + "AND indexOf(attributes_string_key, 'method') > 0 " + "AND indexOf(resources_string_key, 'x') > 0 " + + "AND attribute_string_name_exists=true " + "group by method,x,ts " + "order by method ASC,x ASC", }, @@ -428,6 +473,7 @@ var testBuildLogsQueryData = []struct { "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND has(attributes_float64_key, 'bytes') " + "group by method,ts " + "order by method ASC", }, @@ -439,7 +485,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorSum, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -452,11 +498,12 @@ var testBuildLogsQueryData = []struct { TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts," + " attributes_string_value[indexOf(attributes_string_key, 'method')] as method, " + - "sum(bytes) as value " + + "sum(attribute_float64_bytes) as value " + "from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND attribute_float64_bytes_exists=true " + "group by method,ts " + "order by method ASC", }, @@ -468,7 +515,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorMin, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -481,11 +528,12 @@ var testBuildLogsQueryData = []struct { TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts," + " attributes_string_value[indexOf(attributes_string_key, 'method')] as method, " + - "min(bytes) as value " + + "min(attribute_float64_bytes) as value " + "from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND attribute_float64_bytes_exists=true " + "group by method,ts " + "order by method ASC", }, @@ -497,7 +545,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorMax, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ @@ -510,11 +558,12 @@ var testBuildLogsQueryData = []struct { TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts," + " attributes_string_value[indexOf(attributes_string_key, 'method')] as method, " + - "max(bytes) as value " + + "max(attribute_float64_bytes) as value " + "from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND attribute_float64_bytes_exists=true " + "group by method,ts " + "order by method ASC", }, @@ -526,7 +575,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorP05, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, @@ -536,10 +585,11 @@ var testBuildLogsQueryData = []struct { TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts," + " attributes_string_value[indexOf(attributes_string_key, 'method')] as method, " + - "quantile(0.05)(bytes) as value " + + "quantile(0.05)(attribute_float64_bytes) as value " + "from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND attribute_float64_bytes_exists=true " + "group by method,ts " + "order by method ASC", }, @@ -551,7 +601,7 @@ var testBuildLogsQueryData = []struct { BuilderQuery: &v3.BuilderQuery{ QueryName: "A", StepInterval: 60, - AggregateAttribute: v3.AttributeKey{Key: "bytes", IsColumn: true}, + AggregateAttribute: v3.AttributeKey{Key: "bytes", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true}, AggregateOperator: v3.AggregateOperatorRateSum, Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, @@ -561,9 +611,10 @@ var testBuildLogsQueryData = []struct { TableName: "logs", PreferRPM: true, ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method" + - ", sum(bytes)/1.000000 as value from signoz_logs.distributed_logs " + + ", sum(attribute_float64_bytes)/1.000000 as value from signoz_logs.distributed_logs " + "where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND attribute_float64_bytes_exists=true " + "group by method,ts order by method ASC", }, { @@ -587,6 +638,7 @@ var testBuildLogsQueryData = []struct { ", count(attributes_float64_value[indexOf(attributes_float64_key, 'bytes')])/60.000000 as value " + "from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND has(attributes_float64_key, 'bytes') " + "group by method,ts " + "order by method ASC", }, @@ -612,6 +664,7 @@ var testBuildLogsQueryData = []struct { "sum(attributes_float64_value[indexOf(attributes_float64_key, 'bytes')])/1.000000 as value " + "from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) " + "AND indexOf(attributes_string_key, 'method') > 0 " + + "AND has(attributes_float64_key, 'bytes') " + "group by method,ts " + "order by method ASC", }, @@ -690,7 +743,7 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) group by ts having value > 10 order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(attributes_string_key, 'name') group by ts having value > 10 order by value DESC", }, { Name: "Test aggregate with having clause and filters", @@ -716,7 +769,7 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' group by ts having value > 10 order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'name') group by ts having value > 10 order by value DESC", }, { Name: "Test top level key", @@ -742,7 +795,7 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND body ILIKE '%test%' group by ts having value > 10 order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND body ILIKE '%test%' AND has(attributes_string_key, 'name') group by ts having value > 10 order by value DESC", }, { Name: "Test attribute with same name as top level key", @@ -768,10 +821,10 @@ var testBuildLogsQueryData = []struct { }, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'body')] ILIKE '%test%' group by ts having value > 10 order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'body')] ILIKE '%test%' AND has(attributes_string_key, 'name') group by ts having value > 10 order by value DESC", }, - // Tests for table panel type + // // Tests for table panel type { Name: "TABLE: Test count", PanelType: v3.PanelTypeTable, @@ -1004,7 +1057,7 @@ var testPrepLogsQueryData = []struct { GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, }, TableName: "logs", - ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by value DESC) LIMIT 10", + ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND has(attributes_string_key, 'name') group by method order by value DESC) LIMIT 10", Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { @@ -1027,7 +1080,7 @@ var testPrepLogsQueryData = []struct { OrderBy: []v3.OrderBy{{ColumnName: constants.SigNozOrderByValue, Order: "ASC"}}, }, TableName: "logs", - ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by value ASC) LIMIT 10", + ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND has(attributes_string_key, 'name') group by method order by value ASC) LIMIT 10", Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { @@ -1050,7 +1103,7 @@ var testPrepLogsQueryData = []struct { OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, }, TableName: "logs", - ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 group by method order by method ASC) LIMIT 10", + ExpectedQuery: "SELECT method from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND has(attributes_string_key, 'name') group by method order by method ASC) LIMIT 10", Options: Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { @@ -1072,7 +1125,7 @@ var testPrepLogsQueryData = []struct { Limit: 2, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND (method) GLOBAL IN (%s) group by method,ts order by value DESC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND has(attributes_string_key, 'name') AND (method) GLOBAL IN (%s) group by method,ts order by value DESC", Options: Options{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, { @@ -1095,7 +1148,7 @@ var testPrepLogsQueryData = []struct { Limit: 2, }, TableName: "logs", - ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND (method) GLOBAL IN (%s) group by method,ts order by method ASC", + ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as method, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND indexOf(attributes_string_key, 'method') > 0 AND has(attributes_string_key, 'name') AND (method) GLOBAL IN (%s) group by method,ts order by method ASC", Options: Options{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, // Live tail From 337e33eb8a90154d43bef207344f6c0c122ee165 Mon Sep 17 00:00:00 2001 From: Yunus M Date: Mon, 28 Aug 2023 17:54:14 +0530 Subject: [PATCH 11/21] fix(FE): tooltip link not opening in apdex tooltip (#3441) * Fix tooltip link not opening in apdex tooltip modal * chore: onClick is updated from inline to one to one --------- Co-authored-by: Palash Gupta --- frontend/src/components/TextToolTip/index.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/TextToolTip/index.tsx b/frontend/src/components/TextToolTip/index.tsx index 39ce9072eb..70e5c04685 100644 --- a/frontend/src/components/TextToolTip/index.tsx +++ b/frontend/src/components/TextToolTip/index.tsx @@ -18,12 +18,24 @@ function TextToolTip({ }: TextToolTipProps): JSX.Element { const isDarkMode = useIsDarkMode(); + const onClickHandler = ( + event: React.MouseEvent, + ): void => { + event.stopPropagation(); + }; + const overlay = useMemo( () => (
{`${text} `} {url && ( - + {urlText || 'here'} )} From d1844869782918d80643c010094d60017ff50654 Mon Sep 17 00:00:00 2001 From: Yevhen Shevchenko <90138953+yeshev@users.noreply.github.com> Date: Tue, 29 Aug 2023 15:23:22 +0300 Subject: [PATCH 12/21] feat: create live logs page and custom top nav (#3315) * feat: create live logs page and custom top nav * fix: success button color * fix: turn back color * feat: add live logs where clause (#3325) * feat: add live logs where clause * fix: undefined scenario * feat: get live data (#3337) * feat: get live data * fix: change color, change number format * chore: useMemo is updated * feat: add live logs list (#3341) * feat: add live logs list * feat: hide view if error, clear logs * feat: add condition for disable initial loading * fix: double request * fix: render id in the where clause * fix: render where clause and live list * fix: last log padding * fix: list data loading * fix: no logs text * fix: logs list size * fix: small issues * fix: render view with memo --------- Co-authored-by: Palash Gupta --------- Co-authored-by: Palash Gupta --------- Co-authored-by: Palash Gupta * fix: build is fixed --------- Co-authored-by: Palash Gupta Co-authored-by: Yunus M --- frontend/public/locales/en-GB/logs.json | 1 + frontend/public/locales/en-GB/titles.json | 1 + frontend/public/locales/en/logs.json | 1 + frontend/public/locales/en/titles.json | 1 + frontend/src/AppRoutes/pageComponents.ts | 4 + frontend/src/AppRoutes/routes.ts | 8 + frontend/src/api/metrics/getQueryRange.ts | 4 +- frontend/src/constants/liveTail.ts | 5 + frontend/src/constants/routes.ts | 1 + frontend/src/constants/theme.ts | 2 + .../container/LiveLogs/BackButton/index.tsx | 35 ++++ .../container/LiveLogs/FiltersInput/index.tsx | 78 ++++++++ .../container/LiveLogs/FiltersInput/styles.ts | 24 +++ .../LiveLogs/ListViewPanel/index.tsx | 58 ++++++ .../LiveLogs/LiveLogsContainer/index.tsx | 167 ++++++++++++++++++ .../LiveLogs/LiveLogsContainer/styles.ts | 17 ++ .../container/LiveLogs/LiveLogsList/index.tsx | 131 ++++++++++++++ .../container/LiveLogs/LiveLogsList/types.ts | 5 + .../LiveLogs/LiveLogsListChart/index.tsx | 70 ++++++++ .../LiveLogs/LiveLogsListChart/types.ts | 3 + frontend/src/container/LiveLogs/constants.ts | 31 ++++ frontend/src/container/LiveLogs/utils.ts | 71 ++++++++ .../src/container/LiveLogsTopNav/index.tsx | 71 ++++++++ .../src/container/LiveLogsTopNav/styles.ts | 20 +++ .../src/container/LiveLogsTopNav/types.ts | 5 + frontend/src/container/LocalTopNav/index.tsx | 34 ++++ frontend/src/container/LocalTopNav/styles.ts | 9 + frontend/src/container/LocalTopNav/types.ts | 6 + .../LogsExplorerChart.interfaces.ts | 2 + .../src/container/LogsExplorerChart/index.tsx | 32 ++-- .../InfinityTableView/index.tsx | 9 +- .../InfinityTableView/types.ts | 2 +- .../src/container/LogsExplorerViews/index.tsx | 4 +- frontend/src/container/LogsTopNav/index.tsx | 43 +++++ frontend/src/container/LogsTopNav/styles.ts | 20 +++ .../{TopNav => }/NewExplorerCTA/config.ts | 0 .../{TopNav => }/NewExplorerCTA/index.tsx | 0 .../OptionsMenu/FormatField/index.tsx | 13 +- frontend/src/container/OptionsMenu/types.ts | 4 +- .../container/OptionsMenu/useOptionsMenu.ts | 6 +- .../filters/QueryBuilderSearch/index.tsx | 4 + .../container/TopNav/Breadcrumbs/index.tsx | 1 + .../TopNav/DateTimeSelection/config.ts | 2 + frontend/src/container/TopNav/index.tsx | 15 +- .../hooks/queryBuilder/useShareBuilderUrl.ts | 6 +- .../src/hooks/useEventSourceEvent/index.ts | 19 +- frontend/src/pages/LiveLogs/index.tsx | 25 +++ frontend/src/pages/LogsExplorer/index.tsx | 28 +-- frontend/src/providers/EventSource.tsx | 78 +++++--- frontend/src/providers/QueryBuilder.tsx | 12 +- .../actions/dashboard/getQueryResults.ts | 98 +--------- .../dashboard/prepareQueryRangePayload.ts | 103 +++++++++++ .../src/types/api/metrics/getQueryRange.ts | 26 ++- .../queryBuilder/queryAutocompleteResponse.ts | 2 +- frontend/src/types/common/queryBuilder.ts | 3 +- frontend/src/utils/permission/index.ts | 1 + 56 files changed, 1254 insertions(+), 167 deletions(-) create mode 100644 frontend/public/locales/en-GB/logs.json create mode 100644 frontend/public/locales/en/logs.json create mode 100644 frontend/src/constants/liveTail.ts create mode 100644 frontend/src/container/LiveLogs/BackButton/index.tsx create mode 100644 frontend/src/container/LiveLogs/FiltersInput/index.tsx create mode 100644 frontend/src/container/LiveLogs/FiltersInput/styles.ts create mode 100644 frontend/src/container/LiveLogs/ListViewPanel/index.tsx create mode 100644 frontend/src/container/LiveLogs/LiveLogsContainer/index.tsx create mode 100644 frontend/src/container/LiveLogs/LiveLogsContainer/styles.ts create mode 100644 frontend/src/container/LiveLogs/LiveLogsList/index.tsx create mode 100644 frontend/src/container/LiveLogs/LiveLogsList/types.ts create mode 100644 frontend/src/container/LiveLogs/LiveLogsListChart/index.tsx create mode 100644 frontend/src/container/LiveLogs/LiveLogsListChart/types.ts create mode 100644 frontend/src/container/LiveLogs/constants.ts create mode 100644 frontend/src/container/LiveLogs/utils.ts create mode 100644 frontend/src/container/LiveLogsTopNav/index.tsx create mode 100644 frontend/src/container/LiveLogsTopNav/styles.ts create mode 100644 frontend/src/container/LiveLogsTopNav/types.ts create mode 100644 frontend/src/container/LocalTopNav/index.tsx create mode 100644 frontend/src/container/LocalTopNav/styles.ts create mode 100644 frontend/src/container/LocalTopNav/types.ts create mode 100644 frontend/src/container/LogsTopNav/index.tsx create mode 100644 frontend/src/container/LogsTopNav/styles.ts rename frontend/src/container/{TopNav => }/NewExplorerCTA/config.ts (100%) rename frontend/src/container/{TopNav => }/NewExplorerCTA/index.tsx (100%) create mode 100644 frontend/src/pages/LiveLogs/index.tsx create mode 100644 frontend/src/store/actions/dashboard/prepareQueryRangePayload.ts diff --git a/frontend/public/locales/en-GB/logs.json b/frontend/public/locales/en-GB/logs.json new file mode 100644 index 0000000000..804f66f494 --- /dev/null +++ b/frontend/public/locales/en-GB/logs.json @@ -0,0 +1 @@ +{ "fetching_log_lines": "Fetching log lines" } diff --git a/frontend/public/locales/en-GB/titles.json b/frontend/public/locales/en-GB/titles.json index 53ac325f11..f6ba0b816c 100644 --- a/frontend/public/locales/en-GB/titles.json +++ b/frontend/public/locales/en-GB/titles.json @@ -29,6 +29,7 @@ "NOT_FOUND": "SigNoz | Page Not Found", "LOGS": "SigNoz | Logs", "LOGS_EXPLORER": "SigNoz | Logs Explorer", + "LIVE_LOGS": "SigNoz | Live Logs", "HOME_PAGE": "Open source Observability Platform | SigNoz", "PASSWORD_RESET": "SigNoz | Password Reset", "LIST_LICENSES": "SigNoz | List of Licenses", diff --git a/frontend/public/locales/en/logs.json b/frontend/public/locales/en/logs.json new file mode 100644 index 0000000000..804f66f494 --- /dev/null +++ b/frontend/public/locales/en/logs.json @@ -0,0 +1 @@ +{ "fetching_log_lines": "Fetching log lines" } diff --git a/frontend/public/locales/en/titles.json b/frontend/public/locales/en/titles.json index 53ac325f11..f6ba0b816c 100644 --- a/frontend/public/locales/en/titles.json +++ b/frontend/public/locales/en/titles.json @@ -29,6 +29,7 @@ "NOT_FOUND": "SigNoz | Page Not Found", "LOGS": "SigNoz | Logs", "LOGS_EXPLORER": "SigNoz | Logs Explorer", + "LIVE_LOGS": "SigNoz | Live Logs", "HOME_PAGE": "Open source Observability Platform | SigNoz", "PASSWORD_RESET": "SigNoz | Password Reset", "LIST_LICENSES": "SigNoz | List of Licenses", diff --git a/frontend/src/AppRoutes/pageComponents.ts b/frontend/src/AppRoutes/pageComponents.ts index ba9e4eb617..9b72092342 100644 --- a/frontend/src/AppRoutes/pageComponents.ts +++ b/frontend/src/AppRoutes/pageComponents.ts @@ -110,6 +110,10 @@ export const LogsExplorer = Loadable( () => import(/* webpackChunkName: "Logs Explorer" */ 'pages/LogsExplorer'), ); +export const LiveLogs = Loadable( + () => import(/* webpackChunkName: "Live Logs" */ 'pages/LiveLogs'), +); + export const Login = Loadable( () => import(/* webpackChunkName: "Login" */ 'pages/Login'), ); diff --git a/frontend/src/AppRoutes/routes.ts b/frontend/src/AppRoutes/routes.ts index ecf74b5253..ed19a96d6d 100644 --- a/frontend/src/AppRoutes/routes.ts +++ b/frontend/src/AppRoutes/routes.ts @@ -14,6 +14,7 @@ import { GettingStarted, LicensePage, ListAllALertsPage, + LiveLogs, Login, Logs, LogsExplorer, @@ -234,6 +235,13 @@ const routes: AppRoutes[] = [ key: 'LOGS_EXPLORER', isPrivate: true, }, + { + path: ROUTES.LIVE_LOGS, + exact: true, + component: LiveLogs, + key: 'LIVE_LOGS', + isPrivate: true, + }, { path: ROUTES.LOGIN, exact: true, diff --git a/frontend/src/api/metrics/getQueryRange.ts b/frontend/src/api/metrics/getQueryRange.ts index bc70d19832..bee657d904 100644 --- a/frontend/src/api/metrics/getQueryRange.ts +++ b/frontend/src/api/metrics/getQueryRange.ts @@ -4,11 +4,11 @@ import { AxiosError } from 'axios'; import { ErrorResponse, SuccessResponse } from 'types/api'; import { MetricRangePayloadV3, - MetricsRangeProps, + QueryRangePayload, } from 'types/api/metrics/getQueryRange'; export const getMetricsQueryRange = async ( - props: MetricsRangeProps, + props: QueryRangePayload, ): Promise | ErrorResponse> => { try { const response = await axios.post('/query_range', props); diff --git a/frontend/src/constants/liveTail.ts b/frontend/src/constants/liveTail.ts new file mode 100644 index 0000000000..07813a248b --- /dev/null +++ b/frontend/src/constants/liveTail.ts @@ -0,0 +1,5 @@ +export const LIVE_TAIL_HEARTBEAT_TIMEOUT = 600000; + +export const LIVE_TAIL_GRAPH_INTERVAL = 60000; + +export const MAX_LOGS_LIST_SIZE = 1000; diff --git a/frontend/src/constants/routes.ts b/frontend/src/constants/routes.ts index 65b9fd6477..ada1875c0c 100644 --- a/frontend/src/constants/routes.ts +++ b/frontend/src/constants/routes.ts @@ -29,6 +29,7 @@ const ROUTES = { NOT_FOUND: '/not-found', LOGS: '/logs', LOGS_EXPLORER: '/logs-explorer', + LIVE_LOGS: '/logs-explorer/live', HOME_PAGE: '/', PASSWORD_RESET: '/password-reset', LIST_LICENSES: '/licenses', diff --git a/frontend/src/constants/theme.ts b/frontend/src/constants/theme.ts index 18b7db2b18..be46d0d342 100644 --- a/frontend/src/constants/theme.ts +++ b/frontend/src/constants/theme.ts @@ -52,6 +52,8 @@ const themeColors = { gamboge: '#D89614', bckgGrey: '#1d1d1d', lightBlue: '#177ddc', + buttonSuccessRgb: '73, 170, 25', + red: '#E84749', }; export { themeColors }; diff --git a/frontend/src/container/LiveLogs/BackButton/index.tsx b/frontend/src/container/LiveLogs/BackButton/index.tsx new file mode 100644 index 0000000000..574ba0d637 --- /dev/null +++ b/frontend/src/container/LiveLogs/BackButton/index.tsx @@ -0,0 +1,35 @@ +import { ArrowLeftOutlined } from '@ant-design/icons'; +import { Button } from 'antd'; +import { initialQueriesMap } from 'constants/queryBuilder'; +import ROUTES from 'constants/routes'; +import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; +import { useCallback } from 'react'; +import { useHistory } from 'react-router-dom'; + +function BackButton(): JSX.Element { + const history = useHistory(); + + const { resetQuery } = useQueryBuilder(); + + const handleBack = useCallback(() => { + const compositeQuery = initialQueriesMap.logs; + + const JSONCompositeQuery = encodeURIComponent(JSON.stringify(compositeQuery)); + + const path = `${ROUTES.LOGS_EXPLORER}?${JSONCompositeQuery}`; + + const { queryType, ...queryState } = initialQueriesMap.logs; + + resetQuery(queryState); + + history.push(path); + }, [history, resetQuery]); + + return ( + + ); +} + +export default BackButton; diff --git a/frontend/src/container/LiveLogs/FiltersInput/index.tsx b/frontend/src/container/LiveLogs/FiltersInput/index.tsx new file mode 100644 index 0000000000..6acd638d5f --- /dev/null +++ b/frontend/src/container/LiveLogs/FiltersInput/index.tsx @@ -0,0 +1,78 @@ +import { Col } from 'antd'; +import { initialQueriesMap } from 'constants/queryBuilder'; +import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; +import { useEventSource } from 'providers/EventSource'; +import { useCallback, useMemo } from 'react'; +import { + IBuilderQuery, + Query, + TagFilter, +} from 'types/api/queryBuilder/queryBuilderData'; + +import { getQueryWithoutFilterId } from '../utils'; +import { + ContainerStyled, + FilterSearchInputStyled, + SearchButtonStyled, +} from './styles'; + +function FiltersInput(): JSX.Element { + const { + stagedQuery, + handleSetQueryData, + redirectWithQueryBuilderData, + currentQuery, + } = useQueryBuilder(); + + const { initialLoading, handleSetInitialLoading } = useEventSource(); + + const handleChange = useCallback( + (filters: TagFilter) => { + const listQueryData = stagedQuery?.builder.queryData[0]; + + if (!listQueryData) return; + + const queryData: IBuilderQuery = { + ...listQueryData, + filters, + }; + + handleSetQueryData(0, queryData); + }, + [stagedQuery, handleSetQueryData], + ); + + const query = useMemo(() => { + if (stagedQuery && stagedQuery.builder.queryData.length > 0) { + return stagedQuery?.builder.queryData[0]; + } + + return initialQueriesMap.logs.builder.queryData[0]; + }, [stagedQuery]); + + const handleSearch = useCallback(() => { + if (initialLoading) { + handleSetInitialLoading(false); + } + + const preparedQuery: Query = getQueryWithoutFilterId(currentQuery); + + redirectWithQueryBuilderData(preparedQuery); + }, [ + initialLoading, + currentQuery, + redirectWithQueryBuilderData, + handleSetInitialLoading, + ]); + + return ( + + + + + + + ); +} + +export default FiltersInput; diff --git a/frontend/src/container/LiveLogs/FiltersInput/styles.ts b/frontend/src/container/LiveLogs/FiltersInput/styles.ts new file mode 100644 index 0000000000..ed53f261b0 --- /dev/null +++ b/frontend/src/container/LiveLogs/FiltersInput/styles.ts @@ -0,0 +1,24 @@ +import { Input, Row } from 'antd'; +import { themeColors } from 'constants/theme'; +import QueryBuilderSearch from 'container/QueryBuilder/filters/QueryBuilderSearch'; +import styled from 'styled-components'; + +export const FilterSearchInputStyled = styled(QueryBuilderSearch)` + z-index: 1; + .ant-select-selector { + width: 100%; + border-top-right-radius: 0; + border-bottom-right-radius: 0; + } +`; + +export const ContainerStyled = styled(Row)` + color: ${themeColors.white}; +`; + +export const SearchButtonStyled = styled(Input.Search)` + width: 2rem; + .ant-input { + display: none; + } +`; diff --git a/frontend/src/container/LiveLogs/ListViewPanel/index.tsx b/frontend/src/container/LiveLogs/ListViewPanel/index.tsx new file mode 100644 index 0000000000..360dac36c0 --- /dev/null +++ b/frontend/src/container/LiveLogs/ListViewPanel/index.tsx @@ -0,0 +1,58 @@ +import { Button, Popover, Select, Space } from 'antd'; +import { LOCALSTORAGE } from 'constants/localStorage'; +import { useOptionsMenu } from 'container/OptionsMenu'; +import { + defaultSelectStyle, + logsOptions, + viewModeOptionList, +} from 'pages/Logs/config'; +import PopoverContent from 'pages/Logs/PopoverContent'; +import { useCallback } from 'react'; +import { DataSource, StringOperators } from 'types/common/queryBuilder'; + +function ListViewPanel(): JSX.Element { + const { config } = useOptionsMenu({ + storageKey: LOCALSTORAGE.LOGS_LIST_OPTIONS, + dataSource: DataSource.LOGS, + aggregateOperator: StringOperators.NOOP, + }); + + const isFormatButtonVisible = logsOptions.includes(config.format?.value); + + const renderPopoverContent = useCallback(() => { + if (!config.maxLines) return null; + const linedPerRow = config.maxLines.value as number; + const handleLinesPerRowChange = config.maxLines.onChange as ( + value: unknown, + ) => void; + + return ( + + ); + }, [config]); + + return ( + + + + {isFormatButtonVisible && ( + + + + )} + + ); +} + +export default ListViewPanel; diff --git a/frontend/src/container/LiveLogs/LiveLogsContainer/index.tsx b/frontend/src/container/LiveLogs/LiveLogsContainer/index.tsx new file mode 100644 index 0000000000..1d8b67f394 --- /dev/null +++ b/frontend/src/container/LiveLogs/LiveLogsContainer/index.tsx @@ -0,0 +1,167 @@ +import { Col } from 'antd'; +import Spinner from 'components/Spinner'; +import { MAX_LOGS_LIST_SIZE } from 'constants/liveTail'; +import { PANEL_TYPES } from 'constants/queryBuilder'; +import { themeColors } from 'constants/theme'; +import GoToTop from 'container/GoToTop'; +import FiltersInput from 'container/LiveLogs/FiltersInput'; +import LiveLogsTopNav from 'container/LiveLogsTopNav'; +import { useGetCompositeQueryParam } from 'hooks/queryBuilder/useGetCompositeQueryParam'; +import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; +import useDebouncedFn from 'hooks/useDebouncedFunction'; +import { useEventSourceEvent } from 'hooks/useEventSourceEvent'; +import { useNotifications } from 'hooks/useNotifications'; +import { useEventSource } from 'providers/EventSource'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { prepareQueryRangePayload } from 'store/actions/dashboard/prepareQueryRangePayload'; +import { AppState } from 'store/reducers'; +import { ILog } from 'types/api/logs/log'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; +import { GlobalReducer } from 'types/reducer/globalTime'; + +import { idObject } from '../constants'; +import ListViewPanel from '../ListViewPanel'; +import LiveLogsList from '../LiveLogsList'; +import { prepareQueryByFilter } from '../utils'; +import { ContentWrapper, LiveLogsChart, Wrapper } from './styles'; + +function LiveLogsContainer(): JSX.Element { + const [logs, setLogs] = useState([]); + + const { stagedQuery } = useQueryBuilder(); + + const batchedEventsRef = useRef([]); + + const { notifications } = useNotifications(); + + const { selectedTime: globalSelectedTime } = useSelector< + AppState, + GlobalReducer + >((state) => state.globalTime); + + const { + handleStartOpenConnection, + handleCloseConnection, + initialLoading, + } = useEventSource(); + + const compositeQuery = useGetCompositeQueryParam(); + + const updateLogs = useCallback(() => { + const reversedData = batchedEventsRef.current.reverse(); + setLogs((prevState) => + [...reversedData, ...prevState].slice(0, MAX_LOGS_LIST_SIZE), + ); + + batchedEventsRef.current = []; + }, []); + + const debouncedUpdateLogs = useDebouncedFn(updateLogs, 500); + + const batchLiveLog = useCallback( + (log: ILog): void => { + batchedEventsRef.current.push(log); + + debouncedUpdateLogs(); + }, + [debouncedUpdateLogs], + ); + + const handleGetLiveLogs = useCallback( + (event: MessageEvent) => { + const data: ILog = JSON.parse(event.data); + + batchLiveLog(data); + }, + [batchLiveLog], + ); + + const handleError = useCallback(() => { + notifications.error({ message: 'Sorry, something went wrong' }); + }, [notifications]); + + useEventSourceEvent('message', handleGetLiveLogs); + useEventSourceEvent('error', handleError); + + const getPreparedQuery = useCallback( + (query: Query): Query => { + const firstLogId: string | null = logs.length ? logs[0].id : null; + + const preparedQuery: Query = prepareQueryByFilter( + query, + idObject, + firstLogId, + ); + + return preparedQuery; + }, + [logs], + ); + + const handleStartNewConnection = useCallback(() => { + if (!compositeQuery) return; + + handleCloseConnection(); + + const preparedQuery = getPreparedQuery(compositeQuery); + + const { queryPayload } = prepareQueryRangePayload({ + query: preparedQuery, + graphType: PANEL_TYPES.LIST, + selectedTime: 'GLOBAL_TIME', + globalSelectedInterval: globalSelectedTime, + }); + + const encodedQueryPayload = encodeURIComponent(JSON.stringify(queryPayload)); + + const queryString = `q=${encodedQueryPayload}`; + + handleStartOpenConnection({ queryString }); + }, [ + compositeQuery, + globalSelectedTime, + getPreparedQuery, + handleCloseConnection, + handleStartOpenConnection, + ]); + + useEffect(() => { + if (!compositeQuery || !stagedQuery) return; + + if (compositeQuery.id !== stagedQuery.id || initialLoading) { + handleStartNewConnection(); + } + }, [stagedQuery, initialLoading, compositeQuery, handleStartNewConnection]); + + return ( + + + + + + + {initialLoading ? ( + + + + ) : ( + <> + + + + + + + + + + + )} + + + + ); +} + +export default LiveLogsContainer; diff --git a/frontend/src/container/LiveLogs/LiveLogsContainer/styles.ts b/frontend/src/container/LiveLogs/LiveLogsContainer/styles.ts new file mode 100644 index 0000000000..0f336f2405 --- /dev/null +++ b/frontend/src/container/LiveLogs/LiveLogsContainer/styles.ts @@ -0,0 +1,17 @@ +import { Row } from 'antd'; +import { themeColors } from 'constants/theme'; +import styled from 'styled-components'; + +import LiveLogsListChart from '../LiveLogsListChart'; + +export const LiveLogsChart = styled(LiveLogsListChart)` + margin-bottom: 0.5rem; +`; + +export const ContentWrapper = styled(Row)` + color: rgba(${(themeColors.white, 0.85)}); +`; + +export const Wrapper = styled.div` + padding-bottom: 4rem; +`; diff --git a/frontend/src/container/LiveLogs/LiveLogsList/index.tsx b/frontend/src/container/LiveLogs/LiveLogsList/index.tsx new file mode 100644 index 0000000000..da4414e444 --- /dev/null +++ b/frontend/src/container/LiveLogs/LiveLogsList/index.tsx @@ -0,0 +1,131 @@ +import { Card, Typography } from 'antd'; +import ListLogView from 'components/Logs/ListLogView'; +import RawLogView from 'components/Logs/RawLogView'; +import Spinner from 'components/Spinner'; +import { LOCALSTORAGE } from 'constants/localStorage'; +import { OptionFormatTypes } from 'constants/optionsFormatTypes'; +import InfinityTableView from 'container/LogsExplorerList/InfinityTableView'; +import { InfinityWrapperStyled } from 'container/LogsExplorerList/styles'; +import { convertKeysToColumnFields } from 'container/LogsExplorerList/utils'; +import { Heading } from 'container/LogsTable/styles'; +import { useOptionsMenu } from 'container/OptionsMenu'; +import { contentStyle } from 'container/Trace/Search/config'; +import { useCopyLogLink } from 'hooks/logs/useCopyLogLink'; +import useFontFaceObserver from 'hooks/useFontObserver'; +import { useEventSource } from 'providers/EventSource'; +import { memo, useCallback, useEffect, useMemo, useRef } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Virtuoso, VirtuosoHandle } from 'react-virtuoso'; +// interfaces +import { ILog } from 'types/api/logs/log'; +import { DataSource, StringOperators } from 'types/common/queryBuilder'; + +import { LiveLogsListProps } from './types'; + +function LiveLogsList({ logs }: LiveLogsListProps): JSX.Element { + const ref = useRef(null); + + const { t } = useTranslation(['logs']); + + const { isConnectionError, isConnectionLoading } = useEventSource(); + + const { activeLogId } = useCopyLogLink(); + + const { options } = useOptionsMenu({ + storageKey: LOCALSTORAGE.LOGS_LIST_OPTIONS, + dataSource: DataSource.LOGS, + aggregateOperator: StringOperators.NOOP, + }); + + const activeLogIndex = useMemo( + () => logs.findIndex(({ id }) => id === activeLogId), + [logs, activeLogId], + ); + + useFontFaceObserver( + [ + { + family: 'Fira Code', + weight: '300', + }, + ], + options.format === 'raw', + { + timeout: 5000, + }, + ); + + const selectedFields = convertKeysToColumnFields(options.selectColumns); + + const getItemContent = useCallback( + (_: number, log: ILog): JSX.Element => { + if (options.format === 'raw') { + return ( + + ); + } + + return ( + + ); + }, + [options.format, options.maxLines, selectedFields], + ); + + useEffect(() => { + if (!activeLogId || activeLogIndex < 0) return; + + ref?.current?.scrollToIndex({ + index: activeLogIndex, + align: 'start', + behavior: 'smooth', + }); + }, [activeLogId, activeLogIndex]); + + const isLoadingList = isConnectionLoading && logs.length === 0; + + if (isLoadingList) { + return ; + } + + return ( + <> + {options.format !== OptionFormatTypes.TABLE && ( + + Event + + )} + + {logs.length === 0 && {t('fetching_log_lines')}} + + {!isConnectionError && logs.length !== 0 && ( + + {options.format === 'table' ? ( + + ) : ( + + + + )} + + )} + + ); +} + +export default memo(LiveLogsList); diff --git a/frontend/src/container/LiveLogs/LiveLogsList/types.ts b/frontend/src/container/LiveLogs/LiveLogsList/types.ts new file mode 100644 index 0000000000..ba6871c9d6 --- /dev/null +++ b/frontend/src/container/LiveLogs/LiveLogsList/types.ts @@ -0,0 +1,5 @@ +import { ILog } from 'types/api/logs/log'; + +export type LiveLogsListProps = { + logs: ILog[]; +}; diff --git a/frontend/src/container/LiveLogs/LiveLogsListChart/index.tsx b/frontend/src/container/LiveLogs/LiveLogsListChart/index.tsx new file mode 100644 index 0000000000..3a7fc2e258 --- /dev/null +++ b/frontend/src/container/LiveLogs/LiveLogsListChart/index.tsx @@ -0,0 +1,70 @@ +import { LIVE_TAIL_GRAPH_INTERVAL } from 'constants/liveTail'; +import { PANEL_TYPES } from 'constants/queryBuilder'; +import LogsExplorerChart from 'container/LogsExplorerChart'; +import { useGetExplorerQueryRange } from 'hooks/queryBuilder/useGetExplorerQueryRange'; +import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; +import { useEventSource } from 'providers/EventSource'; +import { useMemo } from 'react'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; +import { QueryData } from 'types/api/widgets/getQuery'; +import { DataSource, LogsAggregatorOperator } from 'types/common/queryBuilder'; + +import { LiveLogsListChartProps } from './types'; + +function LiveLogsListChart({ className }: LiveLogsListChartProps): JSX.Element { + const { stagedQuery } = useQueryBuilder(); + const { isConnectionOpen, isConnectionLoading } = useEventSource(); + + const listChartQuery: Query | null = useMemo(() => { + if (!stagedQuery) return null; + + return { + ...stagedQuery, + builder: { + ...stagedQuery.builder, + queryData: stagedQuery.builder.queryData.map((item) => ({ + ...item, + disabled: false, + aggregateOperator: LogsAggregatorOperator.COUNT, + filters: { + ...item.filters, + items: item.filters.items.filter((item) => item.key?.key !== 'id'), + }, + })), + }, + }; + }, [stagedQuery]); + + const { data, isFetching } = useGetExplorerQueryRange( + listChartQuery, + PANEL_TYPES.TIME_SERIES, + { + enabled: isConnectionOpen, + refetchInterval: LIVE_TAIL_GRAPH_INTERVAL, + keepPreviousData: true, + }, + { dataSource: DataSource.LOGS }, + ); + + const chartData: QueryData[] = useMemo(() => { + if (!data) return []; + + return data.payload.data.result; + }, [data]); + + const isLoading = useMemo( + () => isFetching || (isConnectionLoading && !isConnectionOpen), + [isConnectionLoading, isConnectionOpen, isFetching], + ); + + return ( + + ); +} + +export default LiveLogsListChart; diff --git a/frontend/src/container/LiveLogs/LiveLogsListChart/types.ts b/frontend/src/container/LiveLogs/LiveLogsListChart/types.ts new file mode 100644 index 0000000000..8867304b42 --- /dev/null +++ b/frontend/src/container/LiveLogs/LiveLogsListChart/types.ts @@ -0,0 +1,3 @@ +export type LiveLogsListChartProps = { + className?: string; +}; diff --git a/frontend/src/container/LiveLogs/constants.ts b/frontend/src/container/LiveLogs/constants.ts new file mode 100644 index 0000000000..46d5916a51 --- /dev/null +++ b/frontend/src/container/LiveLogs/constants.ts @@ -0,0 +1,31 @@ +import { + initialQueriesMap, + initialQueryBuilderFormValuesMap, +} from 'constants/queryBuilder'; +import { FILTERS } from 'container/QueryBuilder/filters/OrderByFilter/config'; +import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; +import { LogsAggregatorOperator } from 'types/common/queryBuilder'; + +export const liveLogsCompositeQuery: Query = { + ...initialQueriesMap.logs, + builder: { + ...initialQueriesMap.logs.builder, + queryData: [ + { + ...initialQueryBuilderFormValuesMap.logs, + aggregateOperator: LogsAggregatorOperator.NOOP, + disabled: true, + pageSize: 10, + orderBy: [{ columnName: 'timestamp', order: FILTERS.DESC }], + }, + ], + }, +}; + +export const idObject: BaseAutocompleteData = { + key: 'id', + type: '', + dataType: 'string', + isColumn: true, +}; diff --git a/frontend/src/container/LiveLogs/utils.ts b/frontend/src/container/LiveLogs/utils.ts new file mode 100644 index 0000000000..8e41db49e3 --- /dev/null +++ b/frontend/src/container/LiveLogs/utils.ts @@ -0,0 +1,71 @@ +import { OPERATORS } from 'constants/queryBuilder'; +import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse'; +import { + Query, + TagFilter, + TagFilterItem, +} from 'types/api/queryBuilder/queryBuilderData'; +import { v4 as uuid } from 'uuid'; + +const getIdFilter = (filtersItems: TagFilterItem[]): TagFilterItem | null => + filtersItems.find((item) => item.key?.key === 'id') || null; + +const getFilter = ( + filters: TagFilter, + tagFilter: BaseAutocompleteData, + value: string, +): TagFilter => { + let newItems = filters.items; + + const isExistIdFilter = getIdFilter(newItems); + + if (isExistIdFilter) { + newItems = newItems.map((item) => + item.key?.key === 'id' ? { ...item, value } : item, + ); + } else { + newItems = [ + ...newItems, + { value, key: tagFilter, op: OPERATORS['>'], id: uuid() }, + ]; + } + + return { items: newItems, op: filters.op }; +}; + +export const prepareQueryByFilter = ( + query: Query, + tagFilter: BaseAutocompleteData, + value: string | null, +): Query => { + const preparedQuery: Query = { + ...query, + builder: { + ...query.builder, + queryData: query.builder.queryData.map((item) => ({ + ...item, + filters: value ? getFilter(item.filters, tagFilter, value) : item.filters, + })), + }, + }; + + return preparedQuery; +}; + +export const getQueryWithoutFilterId = (query: Query): Query => { + const preparedQuery: Query = { + ...query, + builder: { + ...query.builder, + queryData: query.builder.queryData.map((item) => ({ + ...item, + filters: { + ...item.filters, + items: item.filters.items.filter((item) => item.key?.key !== 'id'), + }, + })), + }, + }; + + return preparedQuery; +}; diff --git a/frontend/src/container/LiveLogsTopNav/index.tsx b/frontend/src/container/LiveLogsTopNav/index.tsx new file mode 100644 index 0000000000..5f3718d85c --- /dev/null +++ b/frontend/src/container/LiveLogsTopNav/index.tsx @@ -0,0 +1,71 @@ +import { PauseCircleFilled, PlayCircleFilled } from '@ant-design/icons'; +import { Space } from 'antd'; +import BackButton from 'container/LiveLogs/BackButton'; +import { getQueryWithoutFilterId } from 'container/LiveLogs/utils'; +import LocalTopNav from 'container/LocalTopNav'; +import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; +import { useEventSource } from 'providers/EventSource'; +import { memo, useCallback, useMemo } from 'react'; + +import { LiveButtonStyled } from './styles'; + +function LiveLogsTopNav(): JSX.Element { + const { + isConnectionOpen, + isConnectionLoading, + initialLoading, + handleCloseConnection, + handleSetInitialLoading, + } = useEventSource(); + + const { redirectWithQueryBuilderData, currentQuery } = useQueryBuilder(); + + const isPlaying = isConnectionOpen || isConnectionLoading || initialLoading; + + const onLiveButtonClick = useCallback(() => { + if (initialLoading) { + handleSetInitialLoading(false); + } + + if ((!isConnectionOpen && isConnectionLoading) || isConnectionOpen) { + handleCloseConnection(); + } else { + const preparedQuery = getQueryWithoutFilterId(currentQuery); + redirectWithQueryBuilderData(preparedQuery); + } + }, [ + initialLoading, + isConnectionOpen, + isConnectionLoading, + currentQuery, + handleSetInitialLoading, + handleCloseConnection, + redirectWithQueryBuilderData, + ]); + + const liveButton = useMemo( + () => ( + + : } + danger={isPlaying} + onClick={onLiveButtonClick} + type="primary" + > + {isPlaying ? 'Pause' : 'Resume'} + + + + ), + [isPlaying, onLiveButtonClick], + ); + + return ( + + ); +} + +export default memo(LiveLogsTopNav); diff --git a/frontend/src/container/LiveLogsTopNav/styles.ts b/frontend/src/container/LiveLogsTopNav/styles.ts new file mode 100644 index 0000000000..f6c58b9415 --- /dev/null +++ b/frontend/src/container/LiveLogsTopNav/styles.ts @@ -0,0 +1,20 @@ +import { Button, ButtonProps } from 'antd'; +import { themeColors } from 'constants/theme'; +import styled, { css, FlattenSimpleInterpolation } from 'styled-components'; + +export const LiveButtonStyled = styled(Button)` + background-color: rgba(${themeColors.buttonSuccessRgb}, 0.9); + + ${({ danger }): FlattenSimpleInterpolation => + !danger + ? css` + &:hover { + background-color: rgba(${themeColors.buttonSuccessRgb}, 1) !important; + } + + &:active { + background-color: rgba(${themeColors.buttonSuccessRgb}, 0.7) !important; + } + ` + : css``} +`; diff --git a/frontend/src/container/LiveLogsTopNav/types.ts b/frontend/src/container/LiveLogsTopNav/types.ts new file mode 100644 index 0000000000..61b7cad756 --- /dev/null +++ b/frontend/src/container/LiveLogsTopNav/types.ts @@ -0,0 +1,5 @@ +import { Query } from 'types/api/queryBuilder/queryBuilderData'; + +export type LiveLogsTopNavProps = { + getPreparedQuery: (query: Query) => Query; +}; diff --git a/frontend/src/container/LocalTopNav/index.tsx b/frontend/src/container/LocalTopNav/index.tsx new file mode 100644 index 0000000000..bd6b664e93 --- /dev/null +++ b/frontend/src/container/LocalTopNav/index.tsx @@ -0,0 +1,34 @@ +import { Col, Row, Space } from 'antd'; + +import ShowBreadcrumbs from '../TopNav/Breadcrumbs'; +import DateTimeSelector from '../TopNav/DateTimeSelection'; +import { Container } from './styles'; +import { LocalTopNavProps } from './types'; + +function LocalTopNav({ + actions, + renderPermissions, +}: LocalTopNavProps): JSX.Element | null { + return ( + + + + + + + + + {actions} + {renderPermissions?.isDateTimeEnabled && ( +
+ +
+ )} + + + + + ); +} + +export default LocalTopNav; diff --git a/frontend/src/container/LocalTopNav/styles.ts b/frontend/src/container/LocalTopNav/styles.ts new file mode 100644 index 0000000000..feda027d24 --- /dev/null +++ b/frontend/src/container/LocalTopNav/styles.ts @@ -0,0 +1,9 @@ +import { Row } from 'antd'; +import styled from 'styled-components'; + +export const Container = styled(Row)` + &&& { + margin-top: 2rem; + min-height: 8vh; + } +`; diff --git a/frontend/src/container/LocalTopNav/types.ts b/frontend/src/container/LocalTopNav/types.ts new file mode 100644 index 0000000000..64d0ef9414 --- /dev/null +++ b/frontend/src/container/LocalTopNav/types.ts @@ -0,0 +1,6 @@ +import { ReactNode } from 'react'; + +export type LocalTopNavProps = { + actions?: ReactNode; + renderPermissions?: { isDateTimeEnabled: boolean }; +}; diff --git a/frontend/src/container/LogsExplorerChart/LogsExplorerChart.interfaces.ts b/frontend/src/container/LogsExplorerChart/LogsExplorerChart.interfaces.ts index 6df4bacee3..a19a41d778 100644 --- a/frontend/src/container/LogsExplorerChart/LogsExplorerChart.interfaces.ts +++ b/frontend/src/container/LogsExplorerChart/LogsExplorerChart.interfaces.ts @@ -3,4 +3,6 @@ import { QueryData } from 'types/api/widgets/getQuery'; export type LogsExplorerChartProps = { data: QueryData[]; isLoading: boolean; + isLabelEnabled?: boolean; + className?: string; }; diff --git a/frontend/src/container/LogsExplorerChart/index.tsx b/frontend/src/container/LogsExplorerChart/index.tsx index 2e85d9c5de..a64f8eb382 100644 --- a/frontend/src/container/LogsExplorerChart/index.tsx +++ b/frontend/src/container/LogsExplorerChart/index.tsx @@ -1,8 +1,9 @@ import Graph from 'components/Graph'; import Spinner from 'components/Spinner'; +import { themeColors } from 'constants/theme'; import getChartData, { GetChartDataProps } from 'lib/getChartData'; import { colors } from 'lib/getRandomColor'; -import { memo, useMemo } from 'react'; +import { memo, useCallback, useMemo } from 'react'; import { LogsExplorerChartProps } from './LogsExplorerChart.interfaces'; import { CardStyled } from './LogsExplorerChart.styled'; @@ -10,17 +11,22 @@ import { CardStyled } from './LogsExplorerChart.styled'; function LogsExplorerChart({ data, isLoading, + isLabelEnabled = true, + className, }: LogsExplorerChartProps): JSX.Element { - const handleCreateDatasets: Required['createDataset'] = ( - element, - index, - allLabels, - ) => ({ - label: allLabels[index], - data: element, - backgroundColor: colors[index % colors.length] || 'red', - borderColor: colors[index % colors.length] || 'red', - }); + const handleCreateDatasets: Required['createDataset'] = useCallback( + (element, index, allLabels) => ({ + data: element, + backgroundColor: colors[index % colors.length] || themeColors.red, + borderColor: colors[index % colors.length] || themeColors.red, + ...(isLabelEnabled + ? { + label: allLabels[index], + } + : {}), + }), + [isLabelEnabled], + ); const graphData = useMemo( () => @@ -32,11 +38,11 @@ function LogsExplorerChart({ ], createDataset: handleCreateDatasets, }), - [data], + [data, handleCreateDatasets], ); return ( - + {isLoading ? ( ) : ( diff --git a/frontend/src/container/LogsExplorerList/InfinityTableView/index.tsx b/frontend/src/container/LogsExplorerList/InfinityTableView/index.tsx index b5a30e1568..433ce896c2 100644 --- a/frontend/src/container/LogsExplorerList/InfinityTableView/index.tsx +++ b/frontend/src/container/LogsExplorerList/InfinityTableView/index.tsx @@ -10,6 +10,7 @@ import { getDraggedColumns } from 'hooks/useDragColumns/utils'; import { cloneElement, forwardRef, + memo, ReactElement, ReactNode, useCallback, @@ -67,7 +68,6 @@ const InfinityTable = forwardRef( onAddToQuery, } = useActiveLog(); - const { onEndReached } = infitiyTableProps; const { dataSource, columns } = useTableView({ ...tableViewProps, onClickExpand: onSetActiveLog, @@ -158,8 +158,11 @@ const InfinityTable = forwardRef( }} itemContent={itemContent} fixedHeaderContent={tableHeader} - endReached={onEndReached} totalCount={dataSource.length} + // eslint-disable-next-line react/jsx-props-no-spreading + {...(infitiyTableProps?.onEndReached + ? { endReached: infitiyTableProps.onEndReached } + : {})} /> {activeContextLog && ( @@ -179,4 +182,4 @@ const InfinityTable = forwardRef( }, ); -export default InfinityTable; +export default memo(InfinityTable); diff --git a/frontend/src/container/LogsExplorerList/InfinityTableView/types.ts b/frontend/src/container/LogsExplorerList/InfinityTableView/types.ts index fb8eb23170..caf762263e 100644 --- a/frontend/src/container/LogsExplorerList/InfinityTableView/types.ts +++ b/frontend/src/container/LogsExplorerList/InfinityTableView/types.ts @@ -3,7 +3,7 @@ import { UseTableViewProps } from 'components/Logs/TableView/types'; export type InfinityTableProps = { isLoading?: boolean; tableViewProps: Omit; - infitiyTableProps: { + infitiyTableProps?: { onEndReached: (index: number) => void; }; }; diff --git a/frontend/src/container/LogsExplorerViews/index.tsx b/frontend/src/container/LogsExplorerViews/index.tsx index fac0441486..cab522c156 100644 --- a/frontend/src/container/LogsExplorerViews/index.tsx +++ b/frontend/src/container/LogsExplorerViews/index.tsx @@ -39,7 +39,7 @@ import { Query, TagFilter, } from 'types/api/queryBuilder/queryBuilderData'; -import { DataSource, StringOperators } from 'types/common/queryBuilder'; +import { DataSource, LogsAggregatorOperator } from 'types/common/queryBuilder'; import { GlobalReducer } from 'types/reducer/globalTime'; import { generateExportToDashboardLink } from 'utils/dashboard/generateExportToDashboardLink'; @@ -120,7 +120,7 @@ function LogsExplorerViews(): JSX.Element { const modifiedQueryData: IBuilderQuery = { ...listQuery, - aggregateOperator: StringOperators.COUNT, + aggregateOperator: LogsAggregatorOperator.COUNT, }; const modifiedQuery: Query = { diff --git a/frontend/src/container/LogsTopNav/index.tsx b/frontend/src/container/LogsTopNav/index.tsx new file mode 100644 index 0000000000..8eaec2e1f0 --- /dev/null +++ b/frontend/src/container/LogsTopNav/index.tsx @@ -0,0 +1,43 @@ +import { PlayCircleFilled } from '@ant-design/icons'; +import ROUTES from 'constants/routes'; +import { liveLogsCompositeQuery } from 'container/LiveLogs/constants'; +import LocalTopNav from 'container/LocalTopNav'; +import { useCallback, useMemo } from 'react'; +import { useHistory } from 'react-router-dom'; + +import { LiveButtonStyled } from './styles'; + +function LogsTopNav(): JSX.Element { + const history = useHistory(); + + const handleGoLive = useCallback(() => { + const JSONCompositeQuery = encodeURIComponent( + JSON.stringify(liveLogsCompositeQuery), + ); + + const path = `${ROUTES.LIVE_LOGS}?${JSONCompositeQuery}`; + + history.push(path); + }, [history]); + + const liveButton = useMemo( + () => ( + } + onClick={handleGoLive} + type="primary" + > + Go Live + + ), + [handleGoLive], + ); + return ( + + ); +} + +export default LogsTopNav; diff --git a/frontend/src/container/LogsTopNav/styles.ts b/frontend/src/container/LogsTopNav/styles.ts new file mode 100644 index 0000000000..f6c58b9415 --- /dev/null +++ b/frontend/src/container/LogsTopNav/styles.ts @@ -0,0 +1,20 @@ +import { Button, ButtonProps } from 'antd'; +import { themeColors } from 'constants/theme'; +import styled, { css, FlattenSimpleInterpolation } from 'styled-components'; + +export const LiveButtonStyled = styled(Button)` + background-color: rgba(${themeColors.buttonSuccessRgb}, 0.9); + + ${({ danger }): FlattenSimpleInterpolation => + !danger + ? css` + &:hover { + background-color: rgba(${themeColors.buttonSuccessRgb}, 1) !important; + } + + &:active { + background-color: rgba(${themeColors.buttonSuccessRgb}, 0.7) !important; + } + ` + : css``} +`; diff --git a/frontend/src/container/TopNav/NewExplorerCTA/config.ts b/frontend/src/container/NewExplorerCTA/config.ts similarity index 100% rename from frontend/src/container/TopNav/NewExplorerCTA/config.ts rename to frontend/src/container/NewExplorerCTA/config.ts diff --git a/frontend/src/container/TopNav/NewExplorerCTA/index.tsx b/frontend/src/container/NewExplorerCTA/index.tsx similarity index 100% rename from frontend/src/container/TopNav/NewExplorerCTA/index.tsx rename to frontend/src/container/NewExplorerCTA/index.tsx diff --git a/frontend/src/container/OptionsMenu/FormatField/index.tsx b/frontend/src/container/OptionsMenu/FormatField/index.tsx index 2f7dfa7ee4..b4a5ab5eb9 100644 --- a/frontend/src/container/OptionsMenu/FormatField/index.tsx +++ b/frontend/src/container/OptionsMenu/FormatField/index.tsx @@ -1,3 +1,5 @@ +import { RadioChangeEvent } from 'antd'; +import { useCallback } from 'react'; import { useTranslation } from 'react-i18next'; import { FieldTitle } from '../styles'; @@ -7,6 +9,15 @@ import { FormatFieldWrapper, RadioButton, RadioGroup } from './styles'; function FormatField({ config }: FormatFieldProps): JSX.Element | null { const { t } = useTranslation(['trace']); + const onChange = useCallback( + (event: RadioChangeEvent) => { + if (!config) return; + + config.onChange(event.target.value); + }, + [config], + ); + if (!config) return null; return ( @@ -16,7 +27,7 @@ function FormatField({ config }: FormatFieldProps): JSX.Element | null { size="small" buttonStyle="solid" value={config.value} - onChange={config.onChange} + onChange={onChange} > {t('options_menu.raw')} {t('options_menu.default')} diff --git a/frontend/src/container/OptionsMenu/types.ts b/frontend/src/container/OptionsMenu/types.ts index a4bbdc8641..57b81364d6 100644 --- a/frontend/src/container/OptionsMenu/types.ts +++ b/frontend/src/container/OptionsMenu/types.ts @@ -14,7 +14,9 @@ export interface InitialOptions } export type OptionsMenuConfig = { - format?: Pick; + format?: Pick & { + onChange: (value: LogViewMode) => void; + }; maxLines?: Pick; addColumn?: Pick< SelectProps, diff --git a/frontend/src/container/OptionsMenu/useOptionsMenu.ts b/frontend/src/container/OptionsMenu/useOptionsMenu.ts index cd94cf1b80..be2ae00b37 100644 --- a/frontend/src/container/OptionsMenu/useOptionsMenu.ts +++ b/frontend/src/container/OptionsMenu/useOptionsMenu.ts @@ -1,8 +1,8 @@ -import { RadioChangeEvent } from 'antd'; import getFromLocalstorage from 'api/browser/localstorage/get'; import setToLocalstorage from 'api/browser/localstorage/set'; import { getAggregateKeys } from 'api/queryBuilder/getAttributeKeys'; import { LOCALSTORAGE } from 'constants/localStorage'; +import { LogViewMode } from 'container/LogsTable'; import { useGetAggregateKeys } from 'hooks/queryBuilder/useGetAggregateKeys'; import useDebounce from 'hooks/useDebounce'; import { useNotifications } from 'hooks/useNotifications'; @@ -213,10 +213,10 @@ const useOptionsMenu = ({ ); const handleFormatChange = useCallback( - (event: RadioChangeEvent) => { + (value: LogViewMode) => { const optionsData: OptionsQuery = { ...optionsQueryData, - format: event.target.value, + format: value, }; handleRedirectWithOptionsData(optionsData); diff --git a/frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx b/frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx index fb2770d0c4..243441fa9b 100644 --- a/frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx +++ b/frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx @@ -35,6 +35,7 @@ function QueryBuilderSearch({ query, onChange, whereClauseConfig, + className, }: QueryBuilderSearchProps): JSX.Element { const { updateTag, @@ -163,6 +164,7 @@ function QueryBuilderSearch({ placeholder={PLACEHOLDER} value={queryTags} searchValue={searchValue} + className={className} disabled={isMetricsDataSource && !query.aggregateAttribute.key} style={selectStyle} onSearch={handleSearch} @@ -186,10 +188,12 @@ interface QueryBuilderSearchProps { query: IBuilderQuery; onChange: (value: TagFilter) => void; whereClauseConfig?: WhereClauseConfig; + className?: string; } QueryBuilderSearch.defaultProps = { whereClauseConfig: undefined, + className: '', }; export interface CustomTagProps { diff --git a/frontend/src/container/TopNav/Breadcrumbs/index.tsx b/frontend/src/container/TopNav/Breadcrumbs/index.tsx index ca127b388f..6204c8e259 100644 --- a/frontend/src/container/TopNav/Breadcrumbs/index.tsx +++ b/frontend/src/container/TopNav/Breadcrumbs/index.tsx @@ -21,6 +21,7 @@ const breadcrumbNameMap = { [ROUTES.ALL_DASHBOARD]: 'Dashboard', [ROUTES.LOGS]: 'Logs', [ROUTES.LOGS_EXPLORER]: 'Logs Explorer', + [ROUTES.LIVE_LOGS]: 'Live View', [ROUTES.PIPELINES]: 'Pipelines', }; diff --git a/frontend/src/container/TopNav/DateTimeSelection/config.ts b/frontend/src/container/TopNav/DateTimeSelection/config.ts index ef31201da7..261f30cb8d 100644 --- a/frontend/src/container/TopNav/DateTimeSelection/config.ts +++ b/frontend/src/container/TopNav/DateTimeSelection/config.ts @@ -84,3 +84,5 @@ export const routesToSkip = [ ROUTES.LIST_ALL_ALERT, ROUTES.PIPELINES, ]; + +export const routesToDisable = [ROUTES.LOGS_EXPLORER, ROUTES.LIVE_LOGS]; diff --git a/frontend/src/container/TopNav/index.tsx b/frontend/src/container/TopNav/index.tsx index a481ce77d4..e5f1d6ade3 100644 --- a/frontend/src/container/TopNav/index.tsx +++ b/frontend/src/container/TopNav/index.tsx @@ -3,10 +3,10 @@ import ROUTES from 'constants/routes'; import { useMemo } from 'react'; import { matchPath, useHistory } from 'react-router-dom'; +import NewExplorerCTA from '../NewExplorerCTA'; import ShowBreadcrumbs from './Breadcrumbs'; import DateTimeSelector from './DateTimeSelection'; -import { routesToSkip } from './DateTimeSelection/config'; -import NewExplorerCTA from './NewExplorerCTA'; +import { routesToDisable, routesToSkip } from './DateTimeSelection/config'; import { Container } from './styles'; function TopNav(): JSX.Element | null { @@ -20,12 +20,20 @@ function TopNav(): JSX.Element | null { [location.pathname], ); + const isDisabled = useMemo( + () => + routesToDisable.some((route) => + matchPath(location.pathname, { path: route, exact: true }), + ), + [location.pathname], + ); + const isSignUpPage = useMemo( () => matchPath(location.pathname, { path: ROUTES.SIGN_UP, exact: true }), [location.pathname], ); - if (isSignUpPage) { + if (isSignUpPage || isDisabled) { return null; } @@ -40,7 +48,6 @@ function TopNav(): JSX.Element | null { -
diff --git a/frontend/src/hooks/queryBuilder/useShareBuilderUrl.ts b/frontend/src/hooks/queryBuilder/useShareBuilderUrl.ts index b3ccc230e9..d79f6531ee 100644 --- a/frontend/src/hooks/queryBuilder/useShareBuilderUrl.ts +++ b/frontend/src/hooks/queryBuilder/useShareBuilderUrl.ts @@ -8,7 +8,7 @@ import { useQueryBuilder } from './useQueryBuilder'; export type UseShareBuilderUrlParams = { defaultValue: Query }; export const useShareBuilderUrl = (defaultQuery: Query): void => { - const { redirectWithQueryBuilderData, resetStagedQuery } = useQueryBuilder(); + const { redirectWithQueryBuilderData, resetQuery } = useQueryBuilder(); const urlQuery = useUrlQuery(); const compositeQuery = useGetCompositeQueryParam(); @@ -21,8 +21,8 @@ export const useShareBuilderUrl = (defaultQuery: Query): void => { useEffect( () => (): void => { - resetStagedQuery(); + resetQuery(); }, - [resetStagedQuery], + [resetQuery], ); }; diff --git a/frontend/src/hooks/useEventSourceEvent/index.ts b/frontend/src/hooks/useEventSourceEvent/index.ts index 2194d6255e..79e86a9b7e 100644 --- a/frontend/src/hooks/useEventSourceEvent/index.ts +++ b/frontend/src/hooks/useEventSourceEvent/index.ts @@ -2,20 +2,29 @@ import { EventListener, EventSourceEventMap } from 'event-source-polyfill'; import { useEventSource } from 'providers/EventSource'; import { useEffect } from 'react'; -export const useEventSourceEvent = ( - eventName: keyof EventSourceEventMap, - listener: EventListener, +type EventMap = { + message: MessageEvent; + open: Event; + error: Event; +}; + +export const useEventSourceEvent = ( + eventName: T, + listener: (event: EventMap[T]) => void, ): void => { const { eventSourceInstance } = useEventSource(); useEffect(() => { if (eventSourceInstance) { - eventSourceInstance.addEventListener(eventName, listener); + eventSourceInstance.addEventListener(eventName, listener as EventListener); } return (): void => { if (eventSourceInstance) { - eventSourceInstance.removeEventListener(eventName, listener); + eventSourceInstance.removeEventListener( + eventName, + listener as EventListener, + ); } }; }, [eventName, eventSourceInstance, listener]); diff --git a/frontend/src/pages/LiveLogs/index.tsx b/frontend/src/pages/LiveLogs/index.tsx new file mode 100644 index 0000000000..79a60a1222 --- /dev/null +++ b/frontend/src/pages/LiveLogs/index.tsx @@ -0,0 +1,25 @@ +import { PANEL_TYPES } from 'constants/queryBuilder'; +import { liveLogsCompositeQuery } from 'container/LiveLogs/constants'; +import LiveLogsContainer from 'container/LiveLogs/LiveLogsContainer'; +import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; +import { useShareBuilderUrl } from 'hooks/queryBuilder/useShareBuilderUrl'; +import { EventSourceProvider } from 'providers/EventSource'; +import { useEffect } from 'react'; +import { DataSource } from 'types/common/queryBuilder'; + +function LiveLogs(): JSX.Element { + useShareBuilderUrl(liveLogsCompositeQuery); + const { handleSetConfig } = useQueryBuilder(); + + useEffect(() => { + handleSetConfig(PANEL_TYPES.LIST, DataSource.LOGS); + }, [handleSetConfig]); + + return ( + + + + ); +} + +export default LiveLogs; diff --git a/frontend/src/pages/LogsExplorer/index.tsx b/frontend/src/pages/LogsExplorer/index.tsx index 229ab2f6a8..85514dfa5b 100644 --- a/frontend/src/pages/LogsExplorer/index.tsx +++ b/frontend/src/pages/LogsExplorer/index.tsx @@ -2,24 +2,28 @@ import { Col, Row } from 'antd'; import ExplorerCard from 'components/ExplorerCard'; import LogExplorerQuerySection from 'container/LogExplorerQuerySection'; import LogsExplorerViews from 'container/LogsExplorerViews'; +import LogsTopNav from 'container/LogsTopNav'; // ** Styles import { WrapperStyled } from './styles'; function LogsExplorer(): JSX.Element { return ( - - - - - - - - - - - - + <> + + + + + + + + + + + + + + ); } diff --git a/frontend/src/providers/EventSource.tsx b/frontend/src/providers/EventSource.tsx index 972438db77..89c962a778 100644 --- a/frontend/src/providers/EventSource.tsx +++ b/frontend/src/providers/EventSource.tsx @@ -1,11 +1,13 @@ import { apiV3 } from 'api/apiV1'; import { ENVIRONMENT } from 'constants/env'; +import { LIVE_TAIL_HEARTBEAT_TIMEOUT } from 'constants/liveTail'; import { EventListener, EventSourcePolyfill } from 'event-source-polyfill'; import { createContext, PropsWithChildren, useCallback, useContext, + useEffect, useMemo, useRef, useState, @@ -18,18 +20,25 @@ interface IEventSourceContext { eventSourceInstance: EventSourcePolyfill | null; isConnectionOpen: boolean; isConnectionLoading: boolean; - isConnectionError: string; - handleStartOpenConnection: (url?: string) => void; + isConnectionError: boolean; + initialLoading: boolean; + handleStartOpenConnection: (urlProps: { + url?: string; + queryString: string; + }) => void; handleCloseConnection: () => void; + handleSetInitialLoading: (value: boolean) => void; } const EventSourceContext = createContext({ eventSourceInstance: null, isConnectionOpen: false, isConnectionLoading: false, - isConnectionError: '', + initialLoading: true, + isConnectionError: false, handleStartOpenConnection: () => {}, handleCloseConnection: () => {}, + handleSetInitialLoading: () => {}, }); export function EventSourceProvider({ @@ -37,72 +46,101 @@ export function EventSourceProvider({ }: PropsWithChildren): JSX.Element { const [isConnectionOpen, setIsConnectionOpen] = useState(false); const [isConnectionLoading, setIsConnectionLoading] = useState(false); - const [isConnectionError, setIsConnectionError] = useState(''); + const [isConnectionError, setIsConnectionError] = useState(false); + + const [initialLoading, setInitialLoading] = useState(true); const { user } = useSelector((state) => state.app); const eventSourceRef = useRef(null); - const handleCloseConnection = useCallback(() => { - if (!eventSourceRef.current) return; - - eventSourceRef.current.close(); - setIsConnectionOpen(false); - setIsConnectionLoading(false); + const handleSetInitialLoading = useCallback((value: boolean) => { + setInitialLoading(value); }, []); const handleOpenConnection: EventListener = useCallback(() => { setIsConnectionLoading(false); setIsConnectionOpen(true); + setInitialLoading(false); }, []); const handleErrorConnection: EventListener = useCallback(() => { + setIsConnectionOpen(false); + setIsConnectionLoading(false); + setIsConnectionError(true); + setInitialLoading(false); + if (!eventSourceRef.current) return; - handleCloseConnection(); + eventSourceRef.current.close(); + }, []); + const destroyEventSourceSession = useCallback(() => { + if (!eventSourceRef.current) return; + + eventSourceRef.current.close(); eventSourceRef.current.removeEventListener('error', handleErrorConnection); eventSourceRef.current.removeEventListener('open', handleOpenConnection); - }, [handleCloseConnection, handleOpenConnection]); + }, [handleErrorConnection, handleOpenConnection]); + + const handleCloseConnection = useCallback(() => { + setIsConnectionOpen(false); + setIsConnectionLoading(false); + setIsConnectionError(false); + + destroyEventSourceSession(); + }, [destroyEventSourceSession]); const handleStartOpenConnection = useCallback( - (url?: string) => { - const eventSourceUrl = url || `${ENVIRONMENT.baseURL}${apiV3}logs/livetail`; + (urlProps: { url?: string; queryString: string }): void => { + const { url, queryString } = urlProps; - const TIMEOUT_IN_MS = 10 * 60 * 1000; + const eventSourceUrl = url + ? `${url}/?${queryString}` + : `${ENVIRONMENT.baseURL}${apiV3}logs/livetail?${queryString}`; eventSourceRef.current = new EventSourcePolyfill(eventSourceUrl, { headers: { Authorization: `Bearer ${user?.accessJwt}`, }, - heartbeatTimeout: TIMEOUT_IN_MS, + heartbeatTimeout: LIVE_TAIL_HEARTBEAT_TIMEOUT, }); setIsConnectionLoading(true); - setIsConnectionError(''); + setIsConnectionError(false); eventSourceRef.current.addEventListener('error', handleErrorConnection); - eventSourceRef.current.addEventListener('open', handleOpenConnection); }, - [handleErrorConnection, handleOpenConnection, user?.accessJwt], + [user, handleErrorConnection, handleOpenConnection], ); - const contextValue = useMemo( + useEffect( + () => (): void => { + handleCloseConnection(); + }, + [handleCloseConnection], + ); + + const contextValue: IEventSourceContext = useMemo( () => ({ eventSourceInstance: eventSourceRef.current, isConnectionError, isConnectionLoading, isConnectionOpen, + initialLoading, handleStartOpenConnection, handleCloseConnection, + handleSetInitialLoading, }), [ isConnectionError, isConnectionLoading, isConnectionOpen, + initialLoading, handleStartOpenConnection, handleCloseConnection, + handleSetInitialLoading, ], ); diff --git a/frontend/src/providers/QueryBuilder.tsx b/frontend/src/providers/QueryBuilder.tsx index 7c1f1d9792..41d9d8eec4 100644 --- a/frontend/src/providers/QueryBuilder.tsx +++ b/frontend/src/providers/QueryBuilder.tsx @@ -67,7 +67,7 @@ export const QueryBuilderContext = createContext({ addNewQueryItem: () => {}, redirectWithQueryBuilderData: () => {}, handleRunQuery: () => {}, - resetStagedQuery: () => {}, + resetQuery: () => {}, updateAllQueriesOperators: () => initialQueriesMap.metrics, updateQueriesData: () => initialQueriesMap.metrics, initQueryBuilderData: () => {}, @@ -526,8 +526,12 @@ export function QueryBuilderProvider({ }); }, [currentQuery, queryType, maxTime, minTime, redirectWithQueryBuilderData]); - const resetStagedQuery = useCallback(() => { + const resetQuery = useCallback((newCurrentQuery?: QueryState) => { setStagedQuery(null); + + if (newCurrentQuery) { + setCurrentQuery(newCurrentQuery); + } }, []); useEffect(() => { @@ -595,7 +599,7 @@ export function QueryBuilderProvider({ addNewQueryItem, redirectWithQueryBuilderData, handleRunQuery, - resetStagedQuery, + resetQuery, updateAllQueriesOperators, updateQueriesData, initQueryBuilderData, @@ -618,7 +622,7 @@ export function QueryBuilderProvider({ addNewQueryItem, redirectWithQueryBuilderData, handleRunQuery, - resetStagedQuery, + resetQuery, updateAllQueriesOperators, updateQueriesData, initQueryBuilderData, diff --git a/frontend/src/store/actions/dashboard/getQueryResults.ts b/frontend/src/store/actions/dashboard/getQueryResults.ts index a132fc8e78..2e6beae483 100644 --- a/frontend/src/store/actions/dashboard/getQueryResults.ts +++ b/frontend/src/store/actions/dashboard/getQueryResults.ts @@ -3,106 +3,24 @@ // @ts-nocheck import { getMetricsQueryRange } from 'api/metrics/getQueryRange'; -import { GRAPH_TYPES } from 'container/NewDashboard/ComponentsSlider'; import { timePreferenceType } from 'container/NewWidget/RightContainer/timeItems'; import { Time } from 'container/TopNav/DateTimeSelection/config'; -import getStartEndRangeTime from 'lib/getStartEndRangeTime'; -import getStep from 'lib/getStep'; import { convertNewDataToOld } from 'lib/newQueryBuilder/convertNewDataToOld'; -import { mapQueryDataToApi } from 'lib/newQueryBuilder/queryBuilderMappers/mapQueryDataToApi'; import { isEmpty } from 'lodash-es'; -import store from 'store'; import { SuccessResponse } from 'types/api'; import { MetricRangePayloadProps } from 'types/api/metrics/getQueryRange'; import { Query } from 'types/api/queryBuilder/queryBuilderData'; -import { EQueryType } from 'types/common/dashboard'; import { Pagination } from 'hooks/queryPagination'; +import { PANEL_TYPES } from 'constants/queryBuilder'; +import { prepareQueryRangePayload } from './prepareQueryRangePayload'; -export async function GetMetricQueryRange({ - query, - globalSelectedInterval, - graphType, - selectedTime, - tableParams, - variables = {}, - params = {}, -}: GetQueryResultsProps): Promise> { - const queryData = query[query.queryType]; - let legendMap: Record = {}; +export async function GetMetricQueryRange( + props: GetQueryResultsProps, +): Promise> { + const { legendMap, queryPayload } = prepareQueryRangePayload(props); - const QueryPayload = { - compositeQuery: { - queryType: query.queryType, - panelType: graphType, - unit: query?.unit, - }, - }; + const response = await getMetricsQueryRange(queryPayload); - switch (query.queryType) { - case EQueryType.QUERY_BUILDER: { - const { queryData: data, queryFormulas } = query.builder; - const currentQueryData = mapQueryDataToApi(data, 'queryName', tableParams); - const currentFormulas = mapQueryDataToApi(queryFormulas, 'queryName'); - - const builderQueries = { - ...currentQueryData.data, - ...currentFormulas.data, - }; - legendMap = { - ...currentQueryData.newLegendMap, - ...currentFormulas.newLegendMap, - }; - - QueryPayload.compositeQuery.builderQueries = builderQueries; - break; - } - case EQueryType.CLICKHOUSE: { - const chQueries = {}; - queryData.map((query) => { - if (!query.query) return; - chQueries[query.name] = { - query: query.query, - disabled: query.disabled, - }; - legendMap[query.name] = query.legend; - }); - QueryPayload.compositeQuery.chQueries = chQueries; - break; - } - case EQueryType.PROM: { - const promQueries = {}; - queryData.map((query) => { - if (!query.query) return; - promQueries[query.name] = { - query: query.query, - disabled: query.disabled, - }; - legendMap[query.name] = query.legend; - }); - QueryPayload.compositeQuery.promQueries = promQueries; - break; - } - default: - return; - } - - const { start, end } = getStartEndRangeTime({ - type: selectedTime, - interval: globalSelectedInterval, - }); - - const response = await getMetricsQueryRange({ - start: parseInt(start, 10) * 1e3, - end: parseInt(end, 10) * 1e3, - step: getStep({ - start: store.getState().globalTime.minTime, - end: store.getState().globalTime.maxTime, - inputFormat: 'ns', - }), - variables, - ...QueryPayload, - ...params, - }); if (response.statusCode >= 400) { throw new Error( `API responded with ${response.statusCode} - ${response.error}`, @@ -139,7 +57,7 @@ export async function GetMetricQueryRange({ export interface GetQueryResultsProps { query: Query; - graphType: GRAPH_TYPES; + graphType: PANEL_TYPES; selectedTime: timePreferenceType; globalSelectedInterval: Time; variables?: Record; diff --git a/frontend/src/store/actions/dashboard/prepareQueryRangePayload.ts b/frontend/src/store/actions/dashboard/prepareQueryRangePayload.ts new file mode 100644 index 0000000000..f8a41dcd97 --- /dev/null +++ b/frontend/src/store/actions/dashboard/prepareQueryRangePayload.ts @@ -0,0 +1,103 @@ +import getStartEndRangeTime from 'lib/getStartEndRangeTime'; +import getStep from 'lib/getStep'; +import { mapQueryDataToApi } from 'lib/newQueryBuilder/queryBuilderMappers/mapQueryDataToApi'; +import store from 'store'; +import { QueryRangePayload } from 'types/api/metrics/getQueryRange'; +import { EQueryType } from 'types/common/dashboard'; + +import { GetQueryResultsProps } from './getQueryResults'; + +type PrepareQueryRangePayload = { + queryPayload: QueryRangePayload; + legendMap: Record; +}; + +export const prepareQueryRangePayload = ({ + query, + globalSelectedInterval, + graphType, + selectedTime, + tableParams, + variables = {}, + params = {}, +}: GetQueryResultsProps): PrepareQueryRangePayload => { + let legendMap: Record = {}; + + const compositeQuery: QueryRangePayload['compositeQuery'] = { + queryType: query.queryType, + panelType: graphType, + }; + + switch (query.queryType) { + case EQueryType.QUERY_BUILDER: { + const { queryData: data, queryFormulas } = query.builder; + const currentQueryData = mapQueryDataToApi(data, 'queryName', tableParams); + const currentFormulas = mapQueryDataToApi(queryFormulas, 'queryName'); + + const builderQueries = { + ...currentQueryData.data, + ...currentFormulas.data, + }; + legendMap = { + ...currentQueryData.newLegendMap, + ...currentFormulas.newLegendMap, + }; + + compositeQuery.builderQueries = builderQueries; + break; + } + case EQueryType.CLICKHOUSE: { + const chQueries = query[query.queryType].reduce((acc, query) => { + if (!query.query) return acc; + + acc[query.name] = query; + + legendMap[query.name] = query.legend; + + return acc; + }, {} as NonNullable); + + compositeQuery.chQueries = chQueries; + + break; + } + case EQueryType.PROM: { + // eslint-disable-next-line sonarjs/no-identical-functions + const promQueries = query[query.queryType].reduce((acc, query) => { + if (!query.query) return acc; + + acc[query.name] = query; + + legendMap[query.name] = query.legend; + + return acc; + }, {} as NonNullable); + + compositeQuery.promQueries = promQueries; + break; + } + + default: + break; + } + + const { start, end } = getStartEndRangeTime({ + type: selectedTime, + interval: globalSelectedInterval, + }); + + const queryPayload: QueryRangePayload = { + start: parseInt(start, 10) * 1e3, + end: parseInt(end, 10) * 1e3, + step: getStep({ + start: store.getState().globalTime.minTime, + end: store.getState().globalTime.maxTime, + inputFormat: 'ns', + }), + variables, + compositeQuery, + ...params, + }; + + return { legendMap, queryPayload }; +}; diff --git a/frontend/src/types/api/metrics/getQueryRange.ts b/frontend/src/types/api/metrics/getQueryRange.ts index 5dd80a451f..b5dcba2d77 100644 --- a/frontend/src/types/api/metrics/getQueryRange.ts +++ b/frontend/src/types/api/metrics/getQueryRange.ts @@ -1,6 +1,30 @@ +import { PANEL_TYPES } from 'constants/queryBuilder'; +import { EQueryType } from 'types/common/dashboard'; + +import { + IBuilderFormula, + IBuilderQuery, + IClickHouseQuery, + IPromQLQuery, +} from '../queryBuilder/queryBuilderData'; import { QueryData, QueryDataV3 } from '../widgets/getQuery'; -export type MetricsRangeProps = never; +export type QueryRangePayload = { + compositeQuery: { + builderQueries?: { + [x: string]: IBuilderQuery | IBuilderFormula; + }; + chQueries?: Record; + promQueries?: Record; + queryType: EQueryType; + panelType: PANEL_TYPES; + }; + end: number; + start: number; + step: number; + variables?: Record; + [param: string]: unknown; +}; export interface MetricRangePayloadProps { data: { result: QueryData[]; diff --git a/frontend/src/types/api/queryBuilder/queryAutocompleteResponse.ts b/frontend/src/types/api/queryBuilder/queryAutocompleteResponse.ts index 2ce23c8b9b..24e062d9ab 100644 --- a/frontend/src/types/api/queryBuilder/queryAutocompleteResponse.ts +++ b/frontend/src/types/api/queryBuilder/queryAutocompleteResponse.ts @@ -9,7 +9,7 @@ export interface BaseAutocompleteData { dataType: DataType; isColumn: boolean; key: string; - type: AutocompleteType; + type: AutocompleteType | string | null; } export interface IQueryAutocompleteResponse { diff --git a/frontend/src/types/common/queryBuilder.ts b/frontend/src/types/common/queryBuilder.ts index 3d74f30fdc..141f265967 100644 --- a/frontend/src/types/common/queryBuilder.ts +++ b/frontend/src/types/common/queryBuilder.ts @@ -6,6 +6,7 @@ import { IClickHouseQuery, IPromQLQuery, Query, + QueryState, } from 'types/api/queryBuilder/queryBuilderData'; import { EQueryType } from './dashboard'; @@ -187,8 +188,8 @@ export type QueryBuilderContextType = { searchParams?: Record, ) => void; handleRunQuery: () => void; - resetStagedQuery: () => void; handleOnUnitsChange: (units: Format['id']) => void; + resetQuery: (newCurrentQuery?: QueryState) => void; updateAllQueriesOperators: ( queryData: Query, panelType: PANEL_TYPES, diff --git a/frontend/src/utils/permission/index.ts b/frontend/src/utils/permission/index.ts index 7beb10c385..2e0f233805 100644 --- a/frontend/src/utils/permission/index.ts +++ b/frontend/src/utils/permission/index.ts @@ -73,6 +73,7 @@ export const routePermission: Record = { VERSION: ['ADMIN', 'EDITOR', 'VIEWER'], LOGS: ['ADMIN', 'EDITOR', 'VIEWER'], LOGS_EXPLORER: ['ADMIN', 'EDITOR', 'VIEWER'], + LIVE_LOGS: ['ADMIN', 'EDITOR', 'VIEWER'], LIST_LICENSES: ['ADMIN'], LOGS_INDEX_FIELDS: ['ADMIN', 'EDITOR', 'VIEWER'], LOGS_PIPELINE: ['ADMIN', 'EDITOR', 'VIEWER'], From 7e297dcb75c1d975455bc68759245a36fc779242 Mon Sep 17 00:00:00 2001 From: Yevhen Shevchenko <90138953+yeshev@users.noreply.github.com> Date: Tue, 29 Aug 2023 17:09:40 +0300 Subject: [PATCH 13/21] feat: use prefetched query in live (#3450) * feat: create live logs page and custom top nav * feat: add live logs where clause * fix: success button color * fix: turn back color * fix: undefined scenario * feat: get live data * fix: change color, change number format * feat: add live logs list * feat: hide view if error, clear logs * feat: add condition for disable initial loading * fix: double request * fix: render id in the where clause * fix: render where clause and live list * fix: last log padding * fix: list data loading * fix: no logs text * fix: logs list size * fix: small issues * feat: use prefetched query in live * chore: useMemo is updated * feat: add live logs list (#3341) * feat: add live logs list * feat: hide view if error, clear logs * feat: add condition for disable initial loading * fix: double request * fix: render id in the where clause * fix: render where clause and live list * fix: last log padding * fix: list data loading * fix: no logs text * fix: logs list size * fix: small issues * fix: render view with memo --------- Co-authored-by: Palash Gupta * chore: alignment is updated * fix: action column size --------- Co-authored-by: Palash Gupta --- .../container/LiveLogs/BackButton/index.tsx | 36 +++++-- .../LiveLogs/LiveLogsContainer/index.tsx | 98 +++++++++++++------ .../container/LiveLogs/LiveLogsList/index.tsx | 4 +- .../LiveLogs/LiveLogsListChart/index.tsx | 18 ++-- .../LiveLogs/LiveLogsListChart/types.ts | 3 + frontend/src/container/LiveLogs/constants.ts | 49 +++++++--- frontend/src/container/LiveLogs/types.ts | 6 ++ .../src/container/LiveLogsTopNav/types.ts | 5 - frontend/src/container/LocalTopNav/index.tsx | 2 +- .../container/LogDetailedView/TableView.tsx | 2 +- frontend/src/container/LogsTopNav/index.tsx | 62 ++++++++++-- .../hooks/queryBuilder/useShareBuilderUrl.ts | 9 +- 12 files changed, 207 insertions(+), 87 deletions(-) create mode 100644 frontend/src/container/LiveLogs/types.ts delete mode 100644 frontend/src/container/LiveLogsTopNav/types.ts diff --git a/frontend/src/container/LiveLogs/BackButton/index.tsx b/frontend/src/container/LiveLogs/BackButton/index.tsx index 574ba0d637..8386a0208d 100644 --- a/frontend/src/container/LiveLogs/BackButton/index.tsx +++ b/frontend/src/container/LiveLogs/BackButton/index.tsx @@ -1,29 +1,49 @@ import { ArrowLeftOutlined } from '@ant-design/icons'; import { Button } from 'antd'; -import { initialQueriesMap } from 'constants/queryBuilder'; +import { + initialQueryBuilderFormValuesMap, + PANEL_TYPES, +} from 'constants/queryBuilder'; +import { queryParamNamesMap } from 'constants/queryBuilderQueryNames'; import ROUTES from 'constants/routes'; +import { useGetCompositeQueryParam } from 'hooks/queryBuilder/useGetCompositeQueryParam'; import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; import { useCallback } from 'react'; import { useHistory } from 'react-router-dom'; +import { DataSource } from 'types/common/queryBuilder'; + +import { constructCompositeQuery } from '../constants'; function BackButton(): JSX.Element { const history = useHistory(); - const { resetQuery } = useQueryBuilder(); + const { updateAllQueriesOperators, resetQuery } = useQueryBuilder(); + + const compositeQuery = useGetCompositeQueryParam(); const handleBack = useCallback(() => { - const compositeQuery = initialQueriesMap.logs; + if (!compositeQuery) return; - const JSONCompositeQuery = encodeURIComponent(JSON.stringify(compositeQuery)); + const nextCompositeQuery = constructCompositeQuery({ + query: compositeQuery, + initialQueryData: initialQueryBuilderFormValuesMap.logs, + customQueryData: { disabled: false }, + }); - const path = `${ROUTES.LOGS_EXPLORER}?${JSONCompositeQuery}`; + const updatedQuery = updateAllQueriesOperators( + nextCompositeQuery, + PANEL_TYPES.LIST, + DataSource.LOGS, + ); - const { queryType, ...queryState } = initialQueriesMap.logs; + resetQuery(updatedQuery); - resetQuery(queryState); + const JSONCompositeQuery = encodeURIComponent(JSON.stringify(updatedQuery)); + + const path = `${ROUTES.LOGS_EXPLORER}?${queryParamNamesMap.compositeQuery}=${JSONCompositeQuery}`; history.push(path); - }, [history, resetQuery]); + }, [history, compositeQuery, resetQuery, updateAllQueriesOperators]); return (