From 091d769ad86b4a31f5dfac68314c9cccde9732e4 Mon Sep 17 00:00:00 2001 From: Ankit Anand <83692067+ankit01-oss@users.noreply.github.com> Date: Wed, 6 Jul 2022 11:45:42 +0530 Subject: [PATCH 01/30] Updated Typo (#1362) --- pkg/query-service/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/README.md b/pkg/query-service/README.md index 72b69afc9a..bac8851855 100644 --- a/pkg/query-service/README.md +++ b/pkg/query-service/README.md @@ -1,6 +1,6 @@ # Query Service -Query service is the interface between forntend and databases. It is written in **Golang**. It will have modules for all supported databases. Query service is responsible to: +Query service is the interface between frontend and databases. It is written in **Golang**. It will have modules for all supported databases. Query service is responsible to: - parse the request from Frontend - create relevant Clickhouse queries (and all other supported database queries) - parse response from databases and handle error if any @@ -29,4 +29,4 @@ ClickHouseUrl=tcp://localhost:9001 STORAGE=clickhouse build/query-service ``` #### Docker Images -The docker images of query-service is available at https://hub.docker.com/r/signoz/query-service \ No newline at end of file +The docker images of query-service is available at https://hub.docker.com/r/signoz/query-service From da368ab5e8e118569417c541e55b4e414f756775 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 6 Jul 2022 15:49:27 +0530 Subject: [PATCH 02/30] feat: add support for not regex (#1328) * feat: add support for not regex Co-authored-by: Vishal Sharma --- pkg/query-service/app/clickhouseReader/reader.go | 5 +++++ pkg/query-service/app/http_handler.go | 8 ++++---- pkg/query-service/app/metrics/query_builder.go | 10 +++++----- .../app/metrics/query_builder_test.go | 14 ++++++++------ pkg/query-service/model/queryParams.go | 15 +++++---------- pkg/query-service/utils/time.go | 14 ++++++++++++++ 6 files changed, 41 insertions(+), 25 deletions(-) create mode 100644 pkg/query-service/utils/time.go diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 8a6e2e6d36..596354433e 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -50,6 +50,7 @@ import ( "go.signoz.io/query-service/constants" am "go.signoz.io/query-service/integrations/alertManager" "go.signoz.io/query-service/model" + "go.signoz.io/query-service/utils" "go.uber.org/zap" ) @@ -2829,6 +2830,10 @@ func (r *ClickHouseReader) GetMetricAutocompleteMetricNames(ctx context.Context, // GetMetricResult runs the query and returns list of time series func (r *ClickHouseReader) GetMetricResult(ctx context.Context, query string) ([]*model.Series, error) { + defer utils.Elapsed("GetMetricResult")() + + zap.S().Infof("Executing metric result query: %s", query) + rows, err := r.db.Query(ctx, query) if err != nil { diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 51ff99a8a9..4e923af79c 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -180,10 +180,10 @@ func writeHttpResponse(w http.ResponseWriter, data interface{}) { } func (aH *APIHandler) RegisterMetricsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v2/metrics").Subrouter() - subRouter.HandleFunc("/query_range", aH.queryRangeMetricsV2).Methods(http.MethodPost) - subRouter.HandleFunc("/autocomplete/list", aH.metricAutocompleteMetricName).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/tagKey", aH.metricAutocompleteTagKey).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/tagValue", aH.metricAutocompleteTagValue).Methods(http.MethodGet) + subRouter.HandleFunc("/query_range", ViewAccess(aH.queryRangeMetricsV2)).Methods(http.MethodPost) + subRouter.HandleFunc("/autocomplete/list", ViewAccess(aH.metricAutocompleteMetricName)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/tagKey", ViewAccess(aH.metricAutocompleteTagKey)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/tagValue", ViewAccess(aH.metricAutocompleteTagValue)).Methods(http.MethodGet) } func (aH *APIHandler) respond(w http.ResponseWriter, data interface{}) { diff --git a/pkg/query-service/app/metrics/query_builder.go b/pkg/query-service/app/metrics/query_builder.go index bf1896af2e..26f57261b9 100644 --- a/pkg/query-service/app/metrics/query_builder.go +++ b/pkg/query-service/app/metrics/query_builder.go @@ -91,10 +91,9 @@ func BuildMetricsTimeSeriesFilterQuery(fs *model.FilterSet, groupTags []string, if fs != nil && len(fs.Items) != 0 { for _, item := range fs.Items { toFormat := item.Value + op := strings.ToLower(strings.TrimSpace(item.Operator)) // if the received value is an array for like/match op, just take the first value - if strings.ToLower(item.Operation) == "like" || - strings.ToLower(item.Operation) == "match" || - strings.ToLower(item.Operation) == "nlike" { + if op == "like" || op == "match" || op == "nlike" || op == "nmatch" { x, ok := item.Value.([]interface{}) if ok { if len(x) == 0 { @@ -104,7 +103,7 @@ func BuildMetricsTimeSeriesFilterQuery(fs *model.FilterSet, groupTags []string, } } fmtVal := formattedValue(toFormat) - switch op := strings.ToLower(item.Operation); op { + switch op { case "eq": conditions = append(conditions, fmt.Sprintf("labels_object.%s = %s", item.Key, fmtVal)) case "neq": @@ -119,6 +118,8 @@ func BuildMetricsTimeSeriesFilterQuery(fs *model.FilterSet, groupTags []string, conditions = append(conditions, fmt.Sprintf("notLike(labels_object.%s, %s)", item.Key, fmtVal)) case "match": conditions = append(conditions, fmt.Sprintf("match(labels_object.%s, %s)", item.Key, fmtVal)) + case "nmatch": + conditions = append(conditions, fmt.Sprintf("not match(labels_object.%s, %s)", item.Key, fmtVal)) default: return "", fmt.Errorf("unsupported operation") } @@ -416,6 +417,5 @@ func PrepareBuilderMetricQueries(qp *model.QueryRangeParamsV2, tableName string) if len(errs) != 0 { return &RunQueries{Err: fmt.Errorf("errors with formulas: %s", FormatErrs(errs, "\n"))} } - fmt.Println(namedQueries) return &RunQueries{Queries: namedQueries} } diff --git a/pkg/query-service/app/metrics/query_builder_test.go b/pkg/query-service/app/metrics/query_builder_test.go index 4530a01a79..f6f9944605 100644 --- a/pkg/query-service/app/metrics/query_builder_test.go +++ b/pkg/query-service/app/metrics/query_builder_test.go @@ -42,8 +42,9 @@ func TestBuildQueryWithFilters(t *testing.T) { "a": { QueryName: "a", MetricName: "name", - TagFilters: &model.FilterSet{Operation: "AND", Items: []model.FilterItem{ - {Key: "a", Value: "b", Operation: "neq"}, + TagFilters: &model.FilterSet{Operator: "AND", Items: []model.FilterItem{ + {Key: "a", Value: "b", Operator: "neq"}, + {Key: "code", Value: "ERROR_*", Operator: "nmatch"}, }}, AggregateOperator: model.RATE_MAX, Expression: "a", @@ -56,6 +57,7 @@ func TestBuildQueryWithFilters(t *testing.T) { So(queries["a"], ShouldContainSubstring, "WHERE metric_name = 'name' AND labels_object.a != 'b'") So(queries["a"], ShouldContainSubstring, "runningDifference(value)/runningDifference(ts)") + So(queries["a"], ShouldContainSubstring, "not match(labels_object.code, 'ERROR_*')") }) } @@ -70,8 +72,8 @@ func TestBuildQueryWithMultipleQueries(t *testing.T) { "a": { QueryName: "a", MetricName: "name", - TagFilters: &model.FilterSet{Operation: "AND", Items: []model.FilterItem{ - {Key: "in", Value: []interface{}{"a", "b", "c"}, Operation: "in"}, + TagFilters: &model.FilterSet{Operator: "AND", Items: []model.FilterItem{ + {Key: "in", Value: []interface{}{"a", "b", "c"}, Operator: "in"}, }}, AggregateOperator: model.RATE_AVG, Expression: "a", @@ -103,8 +105,8 @@ func TestBuildQueryWithMultipleQueriesAndFormula(t *testing.T) { "a": { QueryName: "a", MetricName: "name", - TagFilters: &model.FilterSet{Operation: "AND", Items: []model.FilterItem{ - {Key: "in", Value: []interface{}{"a", "b", "c"}, Operation: "in"}, + TagFilters: &model.FilterSet{Operator: "AND", Items: []model.FilterItem{ + {Key: "in", Value: []interface{}{"a", "b", "c"}, Operator: "in"}, }}, AggregateOperator: model.RATE_MAX, Expression: "a", diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index d1763a0440..69509849d2 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -294,17 +294,12 @@ type GetErrorParams struct { } type FilterItem struct { - Key string `json:"key"` - Value interface{} `json:"value"` - Operation string `json:"op"` + Key string `json:"key"` + Value interface{} `json:"value"` + Operator string `json:"op"` } type FilterSet struct { - Operation string `json:"op,omitempty"` - Items []FilterItem `json:"items"` -} - -type RemoveTTLParams struct { - Type string - RemoveAllTTL bool + Operator string `json:"op,omitempty"` + Items []FilterItem `json:"items"` } diff --git a/pkg/query-service/utils/time.go b/pkg/query-service/utils/time.go new file mode 100644 index 0000000000..90e132aff2 --- /dev/null +++ b/pkg/query-service/utils/time.go @@ -0,0 +1,14 @@ +package utils + +import ( + "time" + + "go.uber.org/zap" +) + +func Elapsed(funcName string) func() { + start := time.Now() + return func() { + zap.S().Infof("%s took %v\n", funcName, time.Since(start)) + } +} From 80c80b2180b8e69342526c3abed00052ad09d1b9 Mon Sep 17 00:00:00 2001 From: Pranshu Chittora Date: Fri, 8 Jul 2022 16:18:08 +0530 Subject: [PATCH 03/30] feat: missing spans handling by returning a forest of trees (#1365) * feat: spanToTree 2.0 * feat: spanToTree EPIFI data * feat: missing spans multiple trees * chore: migrated to popoverss Co-authored-by: Palash --- .../src/container/GantChart/Trace/index.tsx | 9 +- .../src/container/GantChart/Trace/styles.ts | 7 +- frontend/src/container/GantChart/index.tsx | 58 ++-- frontend/src/container/GantChart/styles.ts | 1 + frontend/src/container/GantChart/utils.ts | 42 ++- .../container/TraceDetail/Missingtrace.tsx | 42 +++ .../TraceDetail/SelectedSpanDetails/index.tsx | 1 + frontend/src/container/TraceDetail/index.tsx | 153 ++++++++--- frontend/src/container/TraceDetail/styles.ts | 38 ++- frontend/src/container/TraceDetail/utils.ts | 20 +- .../__tests__/TraceFlameGraph.test.tsx | 1 + .../src/container/TraceFlameGraph/index.tsx | 6 +- frontend/src/types/api/trace/getTraceItem.ts | 12 +- frontend/src/utils/getSpanTreeMetadata.ts | 26 +- frontend/src/utils/spanToTree.ts | 252 +++++++++--------- 15 files changed, 441 insertions(+), 227 deletions(-) create mode 100644 frontend/src/container/TraceDetail/Missingtrace.tsx diff --git a/frontend/src/container/GantChart/Trace/index.tsx b/frontend/src/container/GantChart/Trace/index.tsx index db607092eb..d6982e04e6 100644 --- a/frontend/src/container/GantChart/Trace/index.tsx +++ b/frontend/src/container/GantChart/Trace/index.tsx @@ -39,6 +39,7 @@ function Trace(props: TraceProps): JSX.Element { isExpandAll, intervalUnit, children, + isMissing, } = props; const { isDarkMode } = useThemeMode(); @@ -125,7 +126,7 @@ function Trace(props: TraceProps): JSX.Element { isDarkMode={isDarkMode} /> - + @@ -174,6 +175,7 @@ function Trace(props: TraceProps): JSX.Element { activeSpanPath={activeSpanPath} isExpandAll={isExpandAll} intervalUnit={intervalUnit} + isMissing={child.isMissing} /> ))} @@ -182,6 +184,10 @@ function Trace(props: TraceProps): JSX.Element { ); } +Trace.defaultProps = { + isMissing: false, +}; + interface ITraceGlobal { globalSpread: ITraceMetaData['spread']; globalStart: ITraceMetaData['globalStart']; @@ -196,6 +202,7 @@ interface TraceProps extends ITraceTree, ITraceGlobal { activeSpanPath: string[]; isExpandAll: boolean; intervalUnit: IIntervalUnit; + isMissing?: boolean; } export default Trace; diff --git a/frontend/src/container/GantChart/Trace/styles.ts b/frontend/src/container/GantChart/Trace/styles.ts index ccf139d6c2..7710e77b5b 100644 --- a/frontend/src/container/GantChart/Trace/styles.ts +++ b/frontend/src/container/GantChart/Trace/styles.ts @@ -1,3 +1,4 @@ +import { volcano } from '@ant-design/colors'; import styled, { css, DefaultTheme, @@ -15,7 +16,6 @@ export const Wrapper = styled.ul` padding-top: 0.5rem; position: relative; z-index: 1; - ul { border-left: ${({ isOnlyChild }): StyledCSS => isOnlyChild && 'none'} !important; @@ -36,10 +36,13 @@ export const Wrapper = styled.ul` } `; -export const CardContainer = styled.li` +export const CardContainer = styled.li<{ isMissing?: boolean }>` display: flex; width: 100%; cursor: pointer; + border-radius: 0.25rem; + ${({ isMissing }): string => + isMissing ? `border: 1px dashed ${volcano[6]};` : ''} `; interface Props { diff --git a/frontend/src/container/GantChart/index.tsx b/frontend/src/container/GantChart/index.tsx index a25f4af228..dbe707c2d7 100644 --- a/frontend/src/container/GantChart/index.tsx +++ b/frontend/src/container/GantChart/index.tsx @@ -3,7 +3,7 @@ import { IIntervalUnit } from 'container/TraceDetail/utils'; import React, { useEffect, useState } from 'react'; import { ITraceTree } from 'types/api/trace/getTraceItem'; -import { CardContainer, CardWrapper, CollapseButton, Wrapper } from './styles'; +import { CardContainer, CardWrapper, CollapseButton } from './styles'; import Trace from './Trace'; import { getSpanPath } from './utils'; @@ -36,35 +36,33 @@ function GanttChart(props: GanttChartProps): JSX.Element { setIsExpandAll((prev) => !prev); }; return ( - - - - {isExpandAll ? : } - - - - - - + + + {isExpandAll ? : } + + + + + ); } diff --git a/frontend/src/container/GantChart/styles.ts b/frontend/src/container/GantChart/styles.ts index 4d523c4998..6f05611599 100644 --- a/frontend/src/container/GantChart/styles.ts +++ b/frontend/src/container/GantChart/styles.ts @@ -38,6 +38,7 @@ export const CardWrapper = styled.div` export const CardContainer = styled.li` display: flex; width: 100%; + position: relative; `; export const CollapseButton = styled.div` diff --git a/frontend/src/container/GantChart/utils.ts b/frontend/src/container/GantChart/utils.ts index d229af9839..c91564b3e2 100644 --- a/frontend/src/container/GantChart/utils.ts +++ b/frontend/src/container/GantChart/utils.ts @@ -1,4 +1,5 @@ -import { ITraceTree } from 'types/api/trace/getTraceItem'; +import { set } from 'lodash-es'; +import { ITraceForest, ITraceTree } from 'types/api/trace/getTraceItem'; interface GetTraceMetaData { globalStart: number; @@ -65,25 +66,48 @@ export function getTopLeftFromBody( export const getNodeById = ( searchingId: string, - treeData: ITraceTree, -): ITraceTree | undefined => { - let foundNode: ITraceTree | undefined; - const traverse = (treeNode: ITraceTree, level = 0): void => { + treesData: ITraceForest | undefined, +): ITraceForest => { + const newtreeData: ITraceForest = {} as ITraceForest; + + const traverse = ( + treeNode: ITraceTree, + setCallBack: (arg0: ITraceTree) => void, + level = 0, + ): void => { if (!treeNode) { return; } if (searchingId === treeNode.id) { - foundNode = treeNode; + setCallBack(treeNode); } treeNode.children.forEach((childNode) => { - traverse(childNode, level + 1); + traverse(childNode, setCallBack, level + 1); }); }; - traverse(treeData, 1); - return foundNode; + const spanTreeSetCallback = ( + path: (keyof ITraceForest)[], + value: ITraceTree, + ): ITraceForest => set(newtreeData, path, [value]); + + if (treesData?.spanTree) + treesData.spanTree.forEach((tree) => { + traverse(tree, (value) => spanTreeSetCallback(['spanTree'], value), 1); + }); + + if (treesData?.missingSpanTree) + treesData.missingSpanTree.forEach((tree) => { + traverse( + tree, + (value) => spanTreeSetCallback(['missingSpanTree'], value), + 1, + ); + }); + + return newtreeData; }; const getSpanWithoutChildren = ( diff --git a/frontend/src/container/TraceDetail/Missingtrace.tsx b/frontend/src/container/TraceDetail/Missingtrace.tsx new file mode 100644 index 0000000000..b203f05f68 --- /dev/null +++ b/frontend/src/container/TraceDetail/Missingtrace.tsx @@ -0,0 +1,42 @@ +import { volcano } from '@ant-design/colors'; +import { InfoCircleOutlined } from '@ant-design/icons'; +import { Popover } from 'antd'; +import React from 'react'; + +function PopOverContent(): JSX.Element { + return ( +
+ More details on missing spans{' '} + + here + +
+ ); +} + +function MissingSpansMessage(): JSX.Element { + return ( + +
+ {' '} + This trace has missing spans +
+ + ); +} + +export default MissingSpansMessage; diff --git a/frontend/src/container/TraceDetail/SelectedSpanDetails/index.tsx b/frontend/src/container/TraceDetail/SelectedSpanDetails/index.tsx index 50f2aa9537..08d6c057a9 100644 --- a/frontend/src/container/TraceDetail/SelectedSpanDetails/index.tsx +++ b/frontend/src/container/TraceDetail/SelectedSpanDetails/index.tsx @@ -18,6 +18,7 @@ const { TabPane } = Tabs; function SelectedSpanDetails(props: SelectedSpanDetailsProps): JSX.Element { const { tree } = props; + const { isDarkMode } = useThemeMode(); const OverLayComponentName = useMemo(() => tree?.name, [tree?.name]); diff --git a/frontend/src/container/TraceDetail/index.tsx b/frontend/src/container/TraceDetail/index.tsx index c705123b50..816f59ec2f 100644 --- a/frontend/src/container/TraceDetail/index.tsx +++ b/frontend/src/container/TraceDetail/index.tsx @@ -17,15 +17,23 @@ import dayjs from 'dayjs'; import useUrlQuery from 'hooks/useUrlQuery'; import { spanServiceNameToColorMapping } from 'lib/getRandomColor'; import history from 'lib/history'; +import { map } from 'lodash-es'; import { SPAN_DETAILS_LEFT_COL_WIDTH } from 'pages/TraceDetail/constants'; import React, { useEffect, useMemo, useState } from 'react'; -import { ITraceTree, PayloadProps } from 'types/api/trace/getTraceItem'; +import { ITraceForest, PayloadProps } from 'types/api/trace/getTraceItem'; import { getSpanTreeMetadata } from 'utils/getSpanTreeMetadata'; import { spanToTreeUtil } from 'utils/spanToTree'; +import MissingSpansMessage from './Missingtrace'; import SelectedSpanDetails from './SelectedSpanDetails'; import * as styles from './styles'; -import { getSortedData, IIntervalUnit, INTERVAL_UNITS } from './utils'; +import { FlameGraphMissingSpansContainer, GanttChartWrapper } from './styles'; +import { + getSortedData, + getTreeLevelsCount, + IIntervalUnit, + INTERVAL_UNITS, +} from './utils'; function TraceDetail({ response }: TraceDetailProps): JSX.Element { const spanServiceColors = useMemo( @@ -43,17 +51,23 @@ function TraceDetail({ response }: TraceDetailProps): JSX.Element { const [activeHoverId, setActiveHoverId] = useState(''); const [activeSelectedId, setActiveSelectedId] = useState(spanId || ''); - const [treeData, setTreeData] = useState( + const [treesData, setTreesData] = useState( spanToTreeUtil(response[0].events), ); - const { treeData: tree, ...traceMetaData } = useMemo(() => { - const tree = getSortedData(treeData); + const { treesData: tree, ...traceMetaData } = useMemo(() => { + const sortedTreesData: ITraceForest = { + spanTree: map(treesData.spanTree, (tree) => getSortedData(tree)), + missingSpanTree: map( + treesData.missingSpanTree, + (tree) => getSortedData(tree) || [], + ), + }; // Note: Handle undefined /*eslint-disable */ - return getSpanTreeMetadata(tree as ITraceTree, spanServiceColors); + return getSpanTreeMetadata(sortedTreesData, spanServiceColors); /* eslint-enable */ - }, [treeData, spanServiceColors]); + }, [treesData, spanServiceColors]); const [globalTraceMetadata] = useState({ ...traceMetaData, @@ -69,24 +83,34 @@ function TraceDetail({ response }: TraceDetailProps): JSX.Element { }, [activeSelectedId]); const getSelectedNode = useMemo(() => { - return getNodeById(activeSelectedId, treeData); - }, [activeSelectedId, treeData]); + return getNodeById(activeSelectedId, treesData); + }, [activeSelectedId, treesData]); // const onSearchHandler = (value: string) => { // setSearchSpanString(value); // setTreeData(spanToTreeUtil(response[0].events)); // }; + const onFocusSelectedSpanHandler = (): void => { const treeNode = getNodeById(activeSelectedId, tree); + if (treeNode) { - setTreeData(treeNode); + setTreesData(treeNode); } }; const onResetHandler = (): void => { - setTreeData(spanToTreeUtil(response[0].events)); + setTreesData(spanToTreeUtil(response[0].events)); }; + const hasMissingSpans = useMemo( + (): boolean => + tree.missingSpanTree && + Array.isArray(tree.missingSpanTree) && + tree.missingSpanTree.length > 0, + [tree], + ); + return ( @@ -101,16 +125,45 @@ function TraceDetail({ response }: TraceDetailProps): JSX.Element { {traceMetaData.totalSpans} Span + {hasMissingSpans && } - + {map(tree.spanTree, (tree) => { + return ( + + ); + })} + + {hasMissingSpans && ( + + {map(tree.missingSpanTree, (tree) => { + return ( + + ); + })} + + )} @@ -122,7 +175,9 @@ function TraceDetail({ response }: TraceDetailProps): JSX.Element { justifyContent: 'center', }} > - {tree && dayjs(tree.startTime).format('hh:mm:ss a MM/DD')} + {tree && + traceMetaData.globalStart && + dayjs(traceMetaData.globalStart).format('hh:mm:ss a MM/DD')} - - {/* */} - +