feat: frontend: log pipelines preview (#3706)

* feat: add pipeline preview API

* chore: separate PipelineActions and ProcessorActions components

* feat: add pipeline preview action

* chore: extract useSampleLogs hook and move SampleLogs to filter preview components

* chore: extract SampleLogsResponseDisplay for reuse

* feat: bring together pipeline preview modal content

* chore: generalize SampleLogsResponse to LogsResponse

* feat: finish wiring up pipeline preview flow

* chore: separate response models for useSampleLogs and usePipelinePreview

* chore: require explicit action for simulation after changing logs sample search interval

* feat: error and empty state for pipeline simulation result

* chore: look for error in sample logs response data too

* chore: remove tests for deleted component & update snapshot for PipelineAction tests

* chore: minor cleanup

* chore: address feedback: move timestamp normalization out of api file

* chore: address feedback: use axios directly in pipeline preview API call

* chore: address feedback: use REACT_QUERY_KEY constant for useQuery key

* chore: minor cleanup

---------

Co-authored-by: Palash Gupta <palashgdev@gmail.com>
This commit is contained in:
Raj Kamal Singh 2023-10-12 17:11:23 +05:30 committed by GitHub
parent 7fa50070ce
commit 2be3d35952
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 463 additions and 169 deletions

View File

@ -0,0 +1,21 @@
import axios from 'api';
import { ILog } from 'types/api/logs/log';
import { PipelineData } from 'types/api/pipeline/def';
export interface PipelineSimulationRequest {
logs: ILog[];
pipelines: PipelineData[];
}
export interface PipelineSimulationResponse {
logs: ILog[];
}
const simulatePipelineProcessing = async (
requestBody: PipelineSimulationRequest,
): Promise<PipelineSimulationResponse> =>
axios
.post('/logs/pipelines/preview', requestBody)
.then((res) => res.data.data);
export default simulatePipelineProcessing;

View File

@ -6,4 +6,5 @@ export const REACT_QUERY_KEY = {
DASHBOARD_BY_ID: 'DASHBOARD_BY_ID', DASHBOARD_BY_ID: 'DASHBOARD_BY_ID',
GET_FEATURES_FLAGS: 'GET_FEATURES_FLAGS', GET_FEATURES_FLAGS: 'GET_FEATURES_FLAGS',
DELETE_DASHBOARD: 'DELETE_DASHBOARD', DELETE_DASHBOARD: 'DELETE_DASHBOARD',
LOGS_PIPELINE_PREVIEW: 'LOGS_PIPELINE_PREVIEW',
}; };

View File

@ -18,7 +18,7 @@ import { AlertMessage } from '.';
import { processorColumns } from './config'; import { processorColumns } from './config';
import { FooterButton, StyledTable } from './styles'; import { FooterButton, StyledTable } from './styles';
import DragAction from './TableComponents/DragAction'; import DragAction from './TableComponents/DragAction';
import PipelineActions from './TableComponents/PipelineActions'; import ProcessorActions from './TableComponents/ProcessorActions';
import { import {
getEditedDataSource, getEditedDataSource,
getProcessorUpdatedRow, getProcessorUpdatedRow,
@ -112,8 +112,7 @@ function PipelineExpandView({
dataIndex: 'action', dataIndex: 'action',
key: 'action', key: 'action',
render: (_value, record): JSX.Element => ( render: (_value, record): JSX.Element => (
<PipelineActions <ProcessorActions
isPipelineAction={false}
editAction={processorEditAction(record)} editAction={processorEditAction(record)}
deleteAction={processorDeleteAction(record)} deleteAction={processorDeleteAction(record)}
/> />

View File

@ -29,7 +29,7 @@ function LogsFilterPreview({ filter }: LogsFilterPreviewProps): JSX.Element {
{isEmptyFilter ? ( {isEmptyFilter ? (
<div>Please select a filter</div> <div>Please select a filter</div>
) : ( ) : (
<SampleLogs filter={filter} timeInterval={previewTimeInterval} /> <SampleLogs filter={filter} timeInterval={previewTimeInterval} count={5} />
)} )}
</div> </div>
</div> </div>

View File

@ -12,7 +12,7 @@
align-items: center; align-items: center;
width: 100%; width: 100%;
height: 8rem; height: 12em;
overflow: hidden; overflow: hidden;
border: 1px solid rgba(253, 253, 253, 0.12); border: 1px solid rgba(253, 253, 253, 0.12);
} }

View File

@ -0,0 +1,39 @@
import { Button } from 'antd';
import { useState } from 'react';
import { ILog } from 'types/api/logs/log';
import { PipelineData } from 'types/api/pipeline/def';
import PipelineSimulationResult from './PipelineSimulationResult';
function LogsProcessingSimulator({
inputLogs,
pipeline,
}: LogsProcessingSimulatorProps): JSX.Element {
const [simulationInput, setSimulationInput] = useState<ILog[] | null>(null);
const simulate = (): void => setSimulationInput(inputLogs);
if (simulationInput !== inputLogs) {
return (
<div>
<Button
disabled={(inputLogs?.length || 0) < 1}
type="primary"
onClick={simulate}
>
Simulate Processing
</Button>
</div>
);
}
return (
<PipelineSimulationResult pipeline={pipeline} inputLogs={simulationInput} />
);
}
export interface LogsProcessingSimulatorProps {
inputLogs: ILog[];
pipeline: PipelineData;
}
export default LogsProcessingSimulator;

View File

@ -0,0 +1,43 @@
import './styles.scss';
import { ILog } from 'types/api/logs/log';
import { PipelineData } from 'types/api/pipeline/def';
import LogsList from '../../../components/LogsList';
import usePipelinePreview from '../../../hooks/usePipelinePreview';
function PipelineSimulationResult({
inputLogs,
pipeline,
}: PipelineSimulationResultProps): JSX.Element {
const { isLoading, outputLogs, isError, errorMsg } = usePipelinePreview({
pipeline,
inputLogs,
});
if (isError) {
return (
<div className="pipeline-simulation-error">
<div>There was an error</div>
<div>{errorMsg}</div>
</div>
);
}
if (isLoading) {
return <div>Loading...</div>;
}
if (outputLogs.length < 1) {
return <div>No logs found</div>;
}
return <LogsList logs={outputLogs} />;
}
export interface PipelineSimulationResultProps {
inputLogs: ILog[];
pipeline: PipelineData;
}
export default PipelineSimulationResult;

View File

@ -0,0 +1,3 @@
.pipeline-simulation-error {
text-align: center;
}

View File

@ -0,0 +1,55 @@
import './styles.scss';
import { RelativeDurationOptions } from 'container/TopNav/DateTimeSelection/config';
import { useState } from 'react';
import { PipelineData } from 'types/api/pipeline/def';
import PreviewIntervalSelector from '../components/PreviewIntervalSelector';
import SampleLogsResponseDisplay from '../components/SampleLogs/SampleLogsResponseDisplay';
import useSampleLogs from '../hooks/useSampleLogs';
import LogsProcessingSimulator from './components/LogsProcessingSimulator';
function PipelineProcessingPreview({
pipeline,
}: PipelineProcessingPreviewProps): JSX.Element {
const last1HourInterval = RelativeDurationOptions[3].value;
const [logsSampleQueryInterval, setLogsSampleQueryInterval] = useState(
last1HourInterval,
);
const sampleLogsResponse = useSampleLogs({
filter: pipeline.filter,
timeInterval: logsSampleQueryInterval,
count: 5,
});
const { logs: sampleLogs } = sampleLogsResponse;
return (
<div>
<div className="pipeline-preview-section-header">
<div>Sample logs</div>
<PreviewIntervalSelector
previewFilter={pipeline.filter}
value={logsSampleQueryInterval}
onChange={setLogsSampleQueryInterval}
/>
</div>
<div className="pipeline-preview-logs-container">
<SampleLogsResponseDisplay response={sampleLogsResponse} />
</div>
<div className="pipeline-preview-section-header">
<div>Processed Output</div>
</div>
<div className="pipeline-preview-logs-container">
<LogsProcessingSimulator inputLogs={sampleLogs} pipeline={pipeline} />
</div>
</div>
);
}
export interface PipelineProcessingPreviewProps {
pipeline: PipelineData;
}
export default PipelineProcessingPreview;

View File

@ -0,0 +1,19 @@
.pipeline-preview-section-header {
display: flex;
justify-content: space-between;
align-items: center;
padding-bottom: 0.4rem;
margin: 1.2rem 0 0.4rem 0;
}
.pipeline-preview-logs-container {
position: relative;
display: flex;
justify-content: center;
align-items: center;
width: 100%;
height: 12em;
overflow: hidden;
border: 1px solid rgba(253, 253, 253, 0.12);
}

View File

@ -0,0 +1,32 @@
import { SampleLogsResponse } from '../../hooks/useSampleLogs';
import LogsList from '../LogsList';
function SampleLogsResponseDisplay({
response,
}: SampleLogsResponseDisplayProps): JSX.Element {
const { isLoading, isError, logs } = response;
if (isError) {
return (
<div className="sample-logs-notice-container">
An error occured while querying sample logs
</div>
);
}
if (isLoading) {
return <div className="sample-logs-notice-container">Loading...</div>;
}
if (logs.length < 1) {
return <div className="sample-logs-notice-container">No logs found</div>;
}
return <LogsList logs={logs} />;
}
export interface SampleLogsResponseDisplayProps {
response: SampleLogsResponse;
}
export default SampleLogsResponseDisplay;

View File

@ -1,76 +1,16 @@
import './styles.scss'; import useSampleLogs, { SampleLogsRequest } from '../../hooks/useSampleLogs';
import LogsResponseDisplay from './SampleLogsResponseDisplay';
import { function SampleLogs(props: SampleLogsRequest): JSX.Element {
initialFilters, const sampleLogsResponse = useSampleLogs(props);
initialQueriesMap,
PANEL_TYPES,
} from 'constants/queryBuilder';
import { Time } from 'container/TopNav/DateTimeSelection/config';
import { useGetQueryRange } from 'hooks/queryBuilder/useGetQueryRange';
import cloneDeep from 'lodash-es/cloneDeep';
import { useMemo } from 'react';
import { ILog } from 'types/api/logs/log';
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
import { LogsAggregatorOperator } from 'types/common/queryBuilder';
import LogsList from '../LogsList'; if ((props?.filter?.items?.length || 0) < 1) {
function SampleLogs({ filter, timeInterval }: SampleLogsProps): JSX.Element {
const sampleLogsQuery = useMemo(() => {
const q = cloneDeep(initialQueriesMap.logs);
q.builder.queryData[0] = {
...q.builder.queryData[0],
filters: filter || initialFilters,
aggregateOperator: LogsAggregatorOperator.NOOP,
orderBy: [{ columnName: 'timestamp', order: 'desc' }],
limit: 5,
};
return q;
}, [filter]);
const sampleLogsResponse = useGetQueryRange({
graphType: PANEL_TYPES.LIST,
query: sampleLogsQuery,
selectedTime: 'GLOBAL_TIME',
globalSelectedInterval: timeInterval,
});
if (sampleLogsResponse?.isError) {
return (
<div className="sample-logs-notice-container">
could not fetch logs for filter
</div>
);
}
if (sampleLogsResponse?.isFetching) {
return <div className="sample-logs-notice-container">Loading...</div>;
}
if ((filter?.items?.length || 0) < 1) {
return ( return (
<div className="sample-logs-notice-container">Please select a filter</div> <div className="sample-logs-notice-container">Please select a filter</div>
); );
} }
const logsList = return <LogsResponseDisplay response={sampleLogsResponse} />;
sampleLogsResponse?.data?.payload?.data?.newResult?.data?.result[0]?.list ||
[];
if (logsList.length < 1) {
return <div className="sample-logs-notice-container">No logs found</div>;
}
const logs: ILog[] = logsList.map((item) => ({
...item.data,
timestamp: item.timestamp,
}));
return <LogsList logs={logs} />;
}
interface SampleLogsProps {
filter: TagFilter;
timeInterval: Time;
} }
export default SampleLogs; export default SampleLogs;

View File

@ -0,0 +1,53 @@
import simulatePipelineProcessing, {
PipelineSimulationResponse,
} from 'api/pipeline/preview';
import { AxiosError } from 'axios';
import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
import { useQuery } from 'react-query';
import { ILog } from 'types/api/logs/log';
import { PipelineData } from 'types/api/pipeline/def';
export interface PipelinePreviewRequest {
pipeline: PipelineData;
inputLogs: ILog[];
}
export interface PipelinePreviewResponse {
isLoading: boolean;
outputLogs: ILog[];
isError: boolean;
errorMsg: string;
}
const usePipelinePreview = ({
pipeline,
inputLogs,
}: PipelinePreviewRequest): PipelinePreviewResponse => {
// Ensure log timestamps are numbers for pipeline preview API request
// ILog allows both number and string while the API needs a number
const simulationInput = inputLogs.map((l) => ({
...l,
timestamp: new Date(l.timestamp).getTime(),
}));
const response = useQuery<PipelineSimulationResponse, AxiosError>({
queryFn: async () =>
simulatePipelineProcessing({
logs: simulationInput,
pipelines: [pipeline],
}),
queryKey: [REACT_QUERY_KEY.LOGS_PIPELINE_PREVIEW, pipeline, inputLogs],
retry: false,
});
const { isFetching, isError, data, error } = response;
return {
isLoading: isFetching,
outputLogs: data?.logs || [],
isError,
errorMsg: error?.response?.data?.error || '',
};
};
export default usePipelinePreview;

View File

@ -0,0 +1,69 @@
import {
initialFilters,
initialQueriesMap,
PANEL_TYPES,
} from 'constants/queryBuilder';
import { Time } from 'container/TopNav/DateTimeSelection/config';
import { useGetQueryRange } from 'hooks/queryBuilder/useGetQueryRange';
import cloneDeep from 'lodash-es/cloneDeep';
import { useMemo } from 'react';
import { ILog } from 'types/api/logs/log';
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
import { LogsAggregatorOperator } from 'types/common/queryBuilder';
export interface SampleLogsRequest {
filter: TagFilter;
timeInterval: Time;
count: number;
}
export interface SampleLogsResponse {
isLoading: boolean;
logs: ILog[];
isError: boolean;
}
const DEFAULT_SAMPLE_LOGS_COUNT = 5;
const useSampleLogs = ({
filter,
timeInterval,
count,
}: SampleLogsRequest): SampleLogsResponse => {
const query = useMemo(() => {
const q = cloneDeep(initialQueriesMap.logs);
q.builder.queryData[0] = {
...q.builder.queryData[0],
filters: filter || initialFilters,
aggregateOperator: LogsAggregatorOperator.NOOP,
orderBy: [{ columnName: 'timestamp', order: 'desc' }],
limit: count || DEFAULT_SAMPLE_LOGS_COUNT,
};
return q;
}, [count, filter]);
const response = useGetQueryRange({
graphType: PANEL_TYPES.LIST,
query,
selectedTime: 'GLOBAL_TIME',
globalSelectedInterval: timeInterval,
});
const { isFetching: isLoading, data } = response;
const errorMsg = data?.error || '';
const isError = response.isError || Boolean(errorMsg);
let logs: ILog[] = [];
if (!(isLoading || isError)) {
const logsList = data?.payload?.data?.newResult?.data?.result[0]?.list || [];
logs = logsList.map((item) => ({
...item.data,
timestamp: item.timestamp,
}));
}
return { isLoading, logs, isError };
};
export default useSampleLogs;

View File

@ -1,28 +0,0 @@
import { IconListStyle } from '../styles';
import DeleteAction from './TableActions/DeleteAction';
import EditAction from './TableActions/EditAction';
// import ViewAction from './TableActions/ViewAction';
function PipelineActions({
isPipelineAction,
editAction,
deleteAction,
}: PipelineActionsProps): JSX.Element {
return (
<IconListStyle>
<EditAction editAction={editAction} isPipelineAction={isPipelineAction} />
{/* <ViewAction isPipelineAction={isPipelineAction} /> */}
<DeleteAction
deleteAction={deleteAction}
isPipelineAction={isPipelineAction}
/>
</IconListStyle>
);
}
export interface PipelineActionsProps {
isPipelineAction: boolean;
editAction: VoidFunction;
deleteAction: VoidFunction;
}
export default PipelineActions;

View File

@ -0,0 +1,44 @@
import { EyeFilled } from '@ant-design/icons';
import { Divider, Modal } from 'antd';
import PipelineProcessingPreview from 'container/PipelinePage/PipelineListsView/Preview/PipelineProcessingPreview';
import { useState } from 'react';
import { PipelineData } from 'types/api/pipeline/def';
import { iconStyle } from '../../../config';
function PreviewAction({ pipeline }: PreviewActionProps): JSX.Element | null {
const [previewKey, setPreviewKey] = useState<string | null>(null);
const isModalOpen = Boolean(previewKey);
const openModal = (): void => setPreviewKey(String(Math.random()));
const closeModal = (): void => setPreviewKey(null);
// Can only preview pipelines with some processors in them
if ((pipeline?.config?.length || 0) < 1) {
return null;
}
return (
<>
<EyeFilled style={iconStyle} onClick={openModal} />
<Modal
open={isModalOpen}
onCancel={closeModal}
centered
width={800}
footer={null}
title={`Logs processing preview for ${pipeline.name}`}
>
<Divider />
{isModalOpen && (
<PipelineProcessingPreview pipeline={pipeline} key={previewKey} />
)}
</Modal>
</>
);
}
export interface PreviewActionProps {
pipeline: PipelineData;
}
export default PreviewAction;

View File

@ -0,0 +1,27 @@
import { PipelineData } from 'types/api/pipeline/def';
import { IconListStyle } from '../../styles';
import DeleteAction from '../TableActions/DeleteAction';
import EditAction from '../TableActions/EditAction';
import PreviewAction from './components/PreviewAction';
function PipelineActions({
pipeline,
editAction,
deleteAction,
}: PipelineActionsProps): JSX.Element {
return (
<IconListStyle>
<PreviewAction pipeline={pipeline} />
<EditAction editAction={editAction} isPipelineAction />
<DeleteAction deleteAction={deleteAction} isPipelineAction />
</IconListStyle>
);
}
export interface PipelineActionsProps {
pipeline: PipelineData;
editAction: VoidFunction;
deleteAction: VoidFunction;
}
export default PipelineActions;

View File

@ -0,0 +1,21 @@
import { IconListStyle } from '../styles';
import DeleteAction from './TableActions/DeleteAction';
import EditAction from './TableActions/EditAction';
function ProcessorActions({
editAction,
deleteAction,
}: ProcessorActionsProps): JSX.Element {
return (
<IconListStyle>
<EditAction editAction={editAction} isPipelineAction={false} />
<DeleteAction deleteAction={deleteAction} isPipelineAction={false} />
</IconListStyle>
);
}
export interface ProcessorActionsProps {
editAction: VoidFunction;
deleteAction: VoidFunction;
}
export default ProcessorActions;

View File

@ -1,19 +0,0 @@
import { CopyFilled, EyeFilled } from '@ant-design/icons';
import { iconStyle, smallIconStyle } from '../../config';
function ViewAction({ isPipelineAction }: ViewActionProps): JSX.Element {
if (isPipelineAction) {
return <EyeFilled style={iconStyle} />;
}
return (
<span key="view-action">
<CopyFilled style={smallIconStyle} />
</span>
);
}
export interface ViewActionProps {
isPipelineAction: boolean;
}
export default ViewAction;

View File

@ -172,7 +172,7 @@ function PipelineListsView({
align: 'center', align: 'center',
render: (_value, record): JSX.Element => ( render: (_value, record): JSX.Element => (
<PipelineActions <PipelineActions
isPipelineAction pipeline={record}
editAction={pipelineEditAction(record)} editAction={pipelineEditAction(record)}
deleteAction={pipelineDeleteAction(record)} deleteAction={pipelineDeleteAction(record)}
/> />

View File

@ -1,11 +1,13 @@
import { render } from '@testing-library/react'; import { render } from '@testing-library/react';
import PipelineActions from 'container/PipelinePage/PipelineListsView/TableComponents/PipelineActions';
import { I18nextProvider } from 'react-i18next'; import { I18nextProvider } from 'react-i18next';
import { Provider } from 'react-redux'; import { Provider } from 'react-redux';
import { MemoryRouter } from 'react-router-dom'; import { MemoryRouter } from 'react-router-dom';
import i18n from 'ReactI18'; import i18n from 'ReactI18';
import store from 'store'; import store from 'store';
import { pipelineMockData } from '../mocks/pipeline';
import PipelineActions from '../PipelineListsView/TableComponents/PipelineActions';
describe('PipelinePage container test', () => { describe('PipelinePage container test', () => {
it('should render PipelineActions section', () => { it('should render PipelineActions section', () => {
const { asFragment } = render( const { asFragment } = render(
@ -13,7 +15,7 @@ describe('PipelinePage container test', () => {
<Provider store={store}> <Provider store={store}>
<I18nextProvider i18n={i18n}> <I18nextProvider i18n={i18n}>
<PipelineActions <PipelineActions
isPipelineAction pipeline={pipelineMockData[0]}
editAction={jest.fn()} editAction={jest.fn()}
deleteAction={jest.fn()} deleteAction={jest.fn()}
/> />

View File

@ -1,22 +0,0 @@
import { render } from '@testing-library/react';
import ViewAction from 'container/PipelinePage/PipelineListsView/TableComponents/TableActions/ViewAction';
import { I18nextProvider } from 'react-i18next';
import { Provider } from 'react-redux';
import { MemoryRouter } from 'react-router-dom';
import i18n from 'ReactI18';
import store from 'store';
describe('PipelinePage container test', () => {
it('should render ViewAction section', () => {
const { asFragment } = render(
<MemoryRouter>
<Provider store={store}>
<I18nextProvider i18n={i18n}>
<ViewAction isPipelineAction />
</I18nextProvider>
</Provider>
</MemoryRouter>,
);
expect(asFragment()).toMatchSnapshot();
});
});

View File

@ -17,6 +17,27 @@ exports[`PipelinePage container test should render PipelineActions section 1`] =
<div <div
class="c0" class="c0"
> >
<span
aria-label="eye"
class="anticon anticon-eye"
role="img"
style="font-size: 1.5rem;"
tabindex="-1"
>
<svg
aria-hidden="true"
data-icon="eye"
fill="currentColor"
focusable="false"
height="1em"
viewBox="64 64 896 896"
width="1em"
>
<path
d="M396 512a112 112 0 10224 0 112 112 0 10-224 0zm546.2-25.8C847.4 286.5 704.1 186 512 186c-192.2 0-335.4 100.5-430.2 300.3a60.3 60.3 0 000 51.5C176.6 737.5 319.9 838 512 838c192.2 0 335.4-100.5 430.2-300.3 7.7-16.2 7.7-35 0-51.5zM508 688c-97.2 0-176-78.8-176-176s78.8-176 176-176 176 78.8 176 176-78.8 176-176 176z"
/>
</svg>
</span>
<span <span
aria-label="edit" aria-label="edit"
class="anticon anticon-edit" class="anticon anticon-edit"

View File

@ -1,26 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`PipelinePage container test should render ViewAction section 1`] = `
<DocumentFragment>
<span
aria-label="eye"
class="anticon anticon-eye"
role="img"
style="font-size: 1.5rem;"
>
<svg
aria-hidden="true"
data-icon="eye"
fill="currentColor"
focusable="false"
height="1em"
viewBox="64 64 896 896"
width="1em"
>
<path
d="M396 512a112 112 0 10224 0 112 112 0 10-224 0zm546.2-25.8C847.4 286.5 704.1 186 512 186c-192.2 0-335.4 100.5-430.2 300.3a60.3 60.3 0 000 51.5C176.6 737.5 319.9 838 512 838c192.2 0 335.4-100.5 430.2-300.3 7.7-16.2 7.7-35 0-51.5zM508 688c-97.2 0-176-78.8-176-176s78.8-176 176-176 176 78.8 176 176-78.8 176-176 176z"
/>
</svg>
</span>
</DocumentFragment>
`;