From 5260b152f5f51d7372c4f399f88ea89d71329784 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 20 Dec 2022 19:54:27 +0530 Subject: [PATCH 01/36] fix: do not show result of sub queries in external calls (#1858) --- .../MetricsApplication/MetricsPageQueries/ExternalQueries.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/container/MetricsApplication/MetricsPageQueries/ExternalQueries.ts b/frontend/src/container/MetricsApplication/MetricsPageQueries/ExternalQueries.ts index a41631638f..c25953676a 100644 --- a/frontend/src/container/MetricsApplication/MetricsPageQueries/ExternalQueries.ts +++ b/frontend/src/container/MetricsApplication/MetricsPageQueries/ExternalQueries.ts @@ -31,7 +31,7 @@ export const externalCallErrorPercent = ({ const legendFormula = 'External Call Error Percentage'; const expression = 'A*100/B'; - const disabled = false; + const disabled = true; return getQueryBuilderQuerieswithAdditionalItems({ metricNameA, metricNameB, @@ -102,7 +102,7 @@ export const externalCallDurationByAddress = ({ const metricNameB = 'signoz_external_call_latency_count'; const expression = 'A/B'; const legendFormula = legend; - const disabled = false; + const disabled = true; return getQueryBuilderQuerieswithFormula({ servicename, legend, From 58ce838023320bfef5c43c63a214404eed399e6d Mon Sep 17 00:00:00 2001 From: Pranay Prateek Date: Thu, 22 Dec 2022 11:44:28 +0530 Subject: [PATCH 02/36] chore: Updating stale edition message (#1896) --- frontend/public/locales/en/translation.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/public/locales/en/translation.json b/frontend/public/locales/en/translation.json index c37480259b..a563ce92cb 100644 --- a/frontend/public/locales/en/translation.json +++ b/frontend/public/locales/en/translation.json @@ -6,7 +6,7 @@ "release_notes": "Release Notes", "read_how_to_upgrade": "Read instructions on how to upgrade", "latest_version_signoz": "You are running the latest version of SigNoz.", - "stale_version": "You are on an older version and may be losing out on the latest features we have shipped. We recommend to upgrade to the latest version", + "stale_version": "You are on an older version and may be missing out on the latest features we have shipped. We recommend to upgrade to the latest version", "oops_something_went_wrong_version": "Oops.. facing issues with fetching updated version information", "n_a": "N/A", "routes": { From dbba8b5b55c5c6d0db30bcfd03c60f7a45aa7db4 Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Thu, 22 Dec 2022 17:35:20 +0530 Subject: [PATCH 03/36] feat: event time is updated when root span is missing --- .../SelectedSpanDetails/Events/Event.tsx | 36 +++++++--------- .../Events/EventStartTime.tsx | 31 ++++++++++++++ .../Events/RelativeStartTime.tsx | 42 +++++++++++++++++++ frontend/src/container/TraceDetail/index.tsx | 2 +- 4 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 frontend/src/container/TraceDetail/SelectedSpanDetails/Events/EventStartTime.tsx create mode 100644 frontend/src/container/TraceDetail/SelectedSpanDetails/Events/RelativeStartTime.tsx diff --git a/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/Event.tsx b/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/Event.tsx index 02f08c5a0a..fb53811f45 100644 --- a/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/Event.tsx +++ b/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/Event.tsx @@ -1,6 +1,4 @@ -import { InfoCircleOutlined } from '@ant-design/icons'; -import { Collapse, Popover, Space } from 'antd'; -import { convertTimeToRelevantUnit } from 'container/TraceDetail/utils'; +import { Collapse } from 'antd'; import useThemeMode from 'hooks/useThemeMode'; import keys from 'lodash-es/keys'; import map from 'lodash-es/map'; @@ -9,6 +7,8 @@ import { ITraceTree } from 'types/api/trace/getTraceItem'; import EllipsedButton from '../EllipsedButton'; import { CustomSubText, CustomSubTitle } from '../styles'; +import EventStartTime from './EventStartTime'; +import RelativeStartTime from './RelativeStartTime'; const { Panel } = Collapse; @@ -25,10 +25,6 @@ function ErrorTag({ {map(event, ({ attributeMap, name, timeUnixNano }) => { const attributes = keys(attributeMap); - const { time, timeUnitName } = convertTimeToRelevantUnit( - timeUnixNano / 1e6 - firstSpanStartTime, - ); - return ( - - - Event Start Time - - - - - - - - {`${time.toFixed(2)} ${timeUnitName}`} - + {firstSpanStartTime ? ( + + ) : ( + + )} {map(attributes, (event) => { const value = attributeMap[event]; @@ -93,7 +85,11 @@ interface ErrorTagProps { event: ITraceTree['event']; onToggleHandler: (isOpen: boolean) => void; setText: (text: { subText: string; text: string }) => void; - firstSpanStartTime: number; + firstSpanStartTime?: number; } +ErrorTag.defaultProps = { + firstSpanStartTime: undefined, +}; + export default ErrorTag; diff --git a/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/EventStartTime.tsx b/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/EventStartTime.tsx new file mode 100644 index 0000000000..8ba3efa54b --- /dev/null +++ b/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/EventStartTime.tsx @@ -0,0 +1,31 @@ +import { Popover } from 'antd'; +import dayjs from 'dayjs'; +import useThemeMode from 'hooks/useThemeMode'; +import React from 'react'; + +import { CustomSubText, CustomSubTitle } from '../styles'; + +function EventStartTime({ timeUnixNano }: EventStartTimeProps): JSX.Element { + const { isDarkMode } = useThemeMode(); + + const humanReadableTimeInDayJs = dayjs(timeUnixNano / 1e6).format( + 'YYYY-MM-DD hh:mm:ss.SSS A', + ); + + return ( + <> + Event Time + + + {humanReadableTimeInDayJs} + + + + ); +} + +interface EventStartTimeProps { + timeUnixNano: number; +} + +export default EventStartTime; diff --git a/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/RelativeStartTime.tsx b/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/RelativeStartTime.tsx new file mode 100644 index 0000000000..709291b30d --- /dev/null +++ b/frontend/src/container/TraceDetail/SelectedSpanDetails/Events/RelativeStartTime.tsx @@ -0,0 +1,42 @@ +import { InfoCircleOutlined } from '@ant-design/icons'; +import { Popover, Space } from 'antd'; +import { convertTimeToRelevantUnit } from 'container/TraceDetail/utils'; +import useThemeMode from 'hooks/useThemeMode'; +import React from 'react'; + +import { CustomSubText, CustomSubTitle } from '../styles'; + +function StartTime({ + firstSpanStartTime, + timeUnixNano, +}: StartTimeProps): JSX.Element { + const { isDarkMode } = useThemeMode(); + + const { time, timeUnitName } = convertTimeToRelevantUnit( + timeUnixNano / 1e6 - (firstSpanStartTime || 0), + ); + + return ( + <> + + + Event Start Time + + + + + + + + {`${time.toFixed(2)} ${timeUnitName}`} + + + ); +} + +interface StartTimeProps { + timeUnixNano: number; + firstSpanStartTime: number; +} + +export default StartTime; diff --git a/frontend/src/container/TraceDetail/index.tsx b/frontend/src/container/TraceDetail/index.tsx index 0a9a5ffd1a..efcf406cd2 100644 --- a/frontend/src/container/TraceDetail/index.tsx +++ b/frontend/src/container/TraceDetail/index.tsx @@ -76,7 +76,7 @@ function TraceDetail({ response }: TraceDetailProps): JSX.Element { /* eslint-enable */ }, [treesData, spanServiceColors]); - const firstSpanStartTime = tree.spanTree[0].startTime; + const firstSpanStartTime = tree.spanTree[0]?.startTime; const [globalTraceMetadata] = useState({ ...traceMetaData, From 9c80ba6b786ec832d27cf9941ea55925581805d1 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Mon, 26 Dec 2022 15:08:43 +0530 Subject: [PATCH 04/36] fix: allow multiple spaces between a filter expression (#1897) * fix: allow multiple spaces between a filter expression * fix: regex updated to respect spaces between a search string Co-authored-by: Srikanth Chekuri --- pkg/query-service/app/logs/parser.go | 11 +++++++++-- pkg/query-service/app/logs/parser_test.go | 12 +++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 2b2ebc0015..5e4a195817 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -36,7 +36,7 @@ const ( DESC = "desc" ) -var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+ (in|nin) \([^(]+\))|([\w.]+ (gt|lt|gte|lte) (')?[\S]+(')?)|([\w.]+ (contains|ncontains)) [^\\]?'(.*?[^\\])')`) +var tokenRegex, _ = regexp.Compile(`(?i)(and( )*?|or( )*?)?(([\w.-]+( )+(in|nin)( )+\([^(]+\))|([\w.]+( )+(gt|lt|gte|lte)( )+(')?[\S]+(')?)|([\w.]+( )+(contains|ncontains))( )+[^\\]?'(.*?[^\\])')`) var operatorRegex, _ = regexp.Compile(`(?i)(?: )(in|nin|gt|lt|gte|lte|contains|ncontains)(?: )`) func ParseLogFilterParams(r *http.Request) (*model.LogsFilterParams, error) { @@ -152,6 +152,7 @@ func ParseLogAggregateParams(r *http.Request) (*model.LogsAggregateParams, error func parseLogQuery(query string) ([]string, error) { sqlQueryTokens := []string{} + filterTokens := tokenRegex.FindAllString(query, -1) if len(filterTokens) == 0 { @@ -190,7 +191,13 @@ func parseLogQuery(query string) ([]string, error) { sqlQueryTokens = append(sqlQueryTokens, f) } else { symbol := operatorMapping[strings.ToLower(op)] - sqlQueryTokens = append(sqlQueryTokens, strings.Replace(v, " "+op+" ", " "+symbol+" ", 1)+" ") + sqlExpr := strings.Replace(v, " "+op+" ", " "+symbol+" ", 1) + splittedExpr := strings.Split(sqlExpr, symbol) + if len(splittedExpr) != 2 { + return nil, fmt.Errorf("error while splitting expression: %s", sqlExpr) + } + trimmedSqlExpr := fmt.Sprintf("%s %s %s ", strings.Join(strings.Fields(splittedExpr[0]), " "), symbol, strings.TrimSpace(splittedExpr[1])) + sqlQueryTokens = append(sqlQueryTokens, trimmedSqlExpr) } } diff --git a/pkg/query-service/app/logs/parser_test.go b/pkg/query-service/app/logs/parser_test.go index a305fb6be2..d51f5b554f 100644 --- a/pkg/query-service/app/logs/parser_test.go +++ b/pkg/query-service/app/logs/parser_test.go @@ -80,7 +80,17 @@ var correctQueriesTest = []struct { { `filters with extra spaces`, `service IN ('name > 100') AND length gt 100`, - []string{`service IN ('name > 100') `, `AND length > 100 `}, + []string{`service IN ('name > 100') `, `AND length > 100 `}, + }, + { + `Extra space within a filter expression`, + `service IN ('name > 100')`, + []string{`service IN ('name > 100') `}, + }, + { + `Extra space between a query filter`, + `data contains 'hello world .'`, + []string{`data ILIKE '%hello world .%' `}, }, { `filters with special characters in key name`, From faeaeb61a0e80a458eec9255cbe8ff9a22e5a541 Mon Sep 17 00:00:00 2001 From: Amol Umbark Date: Mon, 26 Dec 2022 15:10:01 +0530 Subject: [PATCH 05/36] fix: added validations on query builder (#1906) Co-authored-by: mindhash Co-authored-by: Pranay Prateek Co-authored-by: Ankit Nayan --- .../QueryBuilder/QueryBuilder.tsx | 105 ++++++++---------- .../SearchFields/QueryBuilder/utils.ts | 14 --- .../SearchFields/Suggestions.tsx | 25 +++-- .../LogsSearchFilter/SearchFields/index.tsx | 83 +++++++++++++- .../LogsSearchFilter/SearchFields/utils.ts | 60 +++++++++- frontend/src/lib/logql/reverseParser.ts | 16 ++- frontend/src/lib/logql/tokens.ts | 58 ++++++++++ 7 files changed, 273 insertions(+), 88 deletions(-) diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/QueryBuilder.tsx b/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/QueryBuilder.tsx index 1aab2e8e75..5b06c3ca82 100644 --- a/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/QueryBuilder.tsx +++ b/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/QueryBuilder.tsx @@ -12,19 +12,15 @@ import { QueryOperatorsMultiVal, QueryOperatorsSingleVal, } from 'lib/logql/tokens'; -import { flatten } from 'lodash-es'; -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { useMemo } from 'react'; import { useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; import { ILogsReducer } from 'types/reducer/logs'; -import { v4 } from 'uuid'; -import { SearchFieldsProps } from '..'; import FieldKey from '../FieldKey'; import { QueryFieldContainer } from '../styles'; -import { createParsedQueryStructure } from '../utils'; +import { QueryFields } from '../utils'; import { Container, QueryWrapper } from './styles'; -import { hashCode, parseQuery } from './utils'; const { Option } = Select; @@ -68,7 +64,6 @@ function QueryField({ const { fields: { selected }, } = useSelector((store) => store.logs); - const getFieldType = (inputKey: string): string => { // eslint-disable-next-line no-restricted-syntax for (const selectedField of selected) { @@ -147,9 +142,12 @@ function QueryField({ /> ) : ( handleChange(2, e.target.value)} + onChange={(e): void => { + handleChange(2, e.target.value); + }} style={{ width: '100%' }} defaultValue={query[2] && query[2].value} + value={query[2] && query[2].value} /> )} @@ -165,85 +163,78 @@ function QueryField({ } interface QueryConditionFieldProps { - query: { value: string | string[]; type: string }[]; + query: QueryFields; queryIndex: number; onUpdate: (arg0: unknown, arg1: number) => void; } export type Query = { value: string | string[]; type: string }[]; +export interface QueryBuilderProps { + keyPrefix: string; + onDropDownToggleHandler: (value: boolean) => VoidFunction; + fieldsQuery: QueryFields[][]; + setFieldsQuery: (q: QueryFields[][]) => void; +} + function QueryBuilder({ - updateParsedQuery, + keyPrefix, + fieldsQuery, + setFieldsQuery, onDropDownToggleHandler, -}: SearchFieldsProps): JSX.Element { - const { - searchFilter: { parsedQuery }, - } = useSelector((store) => store.logs); - - const keyPrefixRef = useRef(hashCode(JSON.stringify(parsedQuery))); - const [keyPrefix, setKeyPrefix] = useState(keyPrefixRef.current); - const generatedQueryStructure = createParsedQueryStructure( - parsedQuery as never[], - ); - - useEffect(() => { - const incomingHashCode = hashCode(JSON.stringify(parsedQuery)); - if (incomingHashCode !== keyPrefixRef.current) { - keyPrefixRef.current = incomingHashCode; - setKeyPrefix(incomingHashCode); - } - }, [parsedQuery]); - +}: QueryBuilderProps): JSX.Element { const handleUpdate = (query: Query, queryIndex: number): void => { - const updatedParsedQuery = generatedQueryStructure; - updatedParsedQuery[queryIndex] = parseQuery(query) as never; - - const flatParsedQuery = flatten(updatedParsedQuery).filter((q) => q.value); - keyPrefixRef.current = hashCode(JSON.stringify(flatParsedQuery)); - updateParsedQuery(flatParsedQuery); + const updated = [...fieldsQuery]; + updated[queryIndex] = query as never; // parseQuery(query) as never; + setFieldsQuery(updated); }; const handleDelete = (queryIndex: number): void => { - const updatedParsedQuery = generatedQueryStructure; - updatedParsedQuery.splice(queryIndex - 1, 2); + const updated = [...fieldsQuery]; + if (queryIndex !== 0) updated.splice(queryIndex - 1, 2); + else updated.splice(queryIndex, 2); - const flatParsedQuery = flatten(updatedParsedQuery).filter((q) => q.value); - keyPrefixRef.current = v4(); - updateParsedQuery(flatParsedQuery); + setFieldsQuery(updated); }; - const QueryUI = (): JSX.Element | JSX.Element[] => - generatedQueryStructure.map((query, idx) => { - if (Array.isArray(query)) - return ( + const QueryUI = ( + fieldsQuery: QueryFields[][], + ): JSX.Element | JSX.Element[] => { + const result: JSX.Element[] = []; + fieldsQuery.forEach((query, idx) => { + if (Array.isArray(query) && query.length > 1) { + result.push( + />, ); - - return ( -
- -
- ); + } else { + result.push( +
+ +
, + ); + } }); + return result; + }; return ( <> - + LOG QUERY BUILDER - {QueryUI()} + {QueryUI(fieldsQuery)} ); } diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/utils.ts b/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/utils.ts index 2641d8af35..2025f5d8f2 100644 --- a/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/utils.ts +++ b/frontend/src/container/LogsSearchFilter/SearchFields/QueryBuilder/utils.ts @@ -21,17 +21,3 @@ export const parseQuery = (queries: Query): Query => { } return queries; }; - -export const hashCode = (s: string): string => { - if (!s) { - return '0'; - } - return `${Math.abs( - s.split('').reduce((a, b) => { - // eslint-disable-next-line no-bitwise, no-param-reassign - a = (a << 5) - a + b.charCodeAt(0); - // eslint-disable-next-line no-bitwise - return a & a; - }, 0), - )}`; -}; diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/Suggestions.tsx b/frontend/src/container/LogsSearchFilter/SearchFields/Suggestions.tsx index 838d790954..56255d1ed1 100644 --- a/frontend/src/container/LogsSearchFilter/SearchFields/Suggestions.tsx +++ b/frontend/src/container/LogsSearchFilter/SearchFields/Suggestions.tsx @@ -2,9 +2,9 @@ import { Button } from 'antd'; import CategoryHeading from 'components/Logs/CategoryHeading'; import map from 'lodash-es/map'; import React from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; -import { ADD_SEARCH_FIELD_QUERY_STRING } from 'types/actions/logs'; +// import { ADD_SEARCH_FIELD_QUERY_STRING } from 'types/actions/logs'; import { ILogsReducer } from 'types/reducer/logs'; import FieldKey from './FieldKey'; @@ -12,15 +12,15 @@ import FieldKey from './FieldKey'; interface SuggestedItemProps { name: string; type: string; + applySuggestion: (name: string) => void; } -function SuggestedItem({ name, type }: SuggestedItemProps): JSX.Element { - const dispatch = useDispatch(); - +function SuggestedItem({ + name, + type, + applySuggestion, +}: SuggestedItemProps): JSX.Element { const addSuggestedField = (): void => { - dispatch({ - type: ADD_SEARCH_FIELD_QUERY_STRING, - payload: name, - }); + applySuggestion(name); }; return ( + + ); } diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/utils.ts b/frontend/src/container/LogsSearchFilter/SearchFields/utils.ts index ae091dc061..fec1c45bee 100644 --- a/frontend/src/container/LogsSearchFilter/SearchFields/utils.ts +++ b/frontend/src/container/LogsSearchFilter/SearchFields/utils.ts @@ -2,11 +2,30 @@ // @ts-ignore // @ts-nocheck -import { QueryTypes, QueryOperatorsSingleVal } from 'lib/logql/tokens'; +import { QueryTypes, ConditionalOperators, ValidTypeSequence, ValidTypeValue } from 'lib/logql/tokens'; export interface QueryFields { type: keyof typeof QueryTypes; - value: string; + value: string | string[]; +} + + +export function fieldsQueryIsvalid(queryFields: QueryFields[]): boolean { + let lastOp: string; + let result = true; + queryFields.forEach((q, idx)=> { + + if (!q.value || q.value === null || q.value === '') result = false; + + if (Array.isArray(q.value) && q.value.length === 0 ) result = false; + + const nextOp = idx < queryFields.length ? queryFields[idx+1]: undefined; + if (!ValidTypeSequence(lastOp?.type, q?.type, nextOp?.type)) result = false + + if (!ValidTypeValue(lastOp?.value, q.value)) result = false; + lastOp = q; + }); + return result } export const queryKOVPair = (): QueryFields[] => [ @@ -23,6 +42,29 @@ export const queryKOVPair = (): QueryFields[] => [ value: null, }, ]; + +export const initQueryKOVPair = (name?: string = null, op?: string = null , value?: string | string[] = null ): QueryFields[] => [ + { + type: QueryTypes.QUERY_KEY, + value: name, + }, + { + type: QueryTypes.QUERY_OPERATOR, + value: op, + }, + { + type: QueryTypes.QUERY_VALUE, + value: value, + }, +]; + +export const prepareConditionOperator = (op?: string = ConditionalOperators.AND): QueryFields => { + return { + type: QueryTypes.CONDITIONAL_OPERATOR, + value: op, + } +} + export const createParsedQueryStructure = (parsedQuery = []) => { if (!parsedQuery.length) { return parsedQuery; @@ -64,3 +106,17 @@ export const createParsedQueryStructure = (parsedQuery = []) => { }); return structuredArray; }; + +export const hashCode = (s: string): string => { + if (!s) { + return '0'; + } + return `${Math.abs( + s.split('').reduce((a, b) => { + // eslint-disable-next-line no-bitwise, no-param-reassign + a = (a << 5) - a + b.charCodeAt(0); + // eslint-disable-next-line no-bitwise + return a & a; + }, 0), + )}`; +}; diff --git a/frontend/src/lib/logql/reverseParser.ts b/frontend/src/lib/logql/reverseParser.ts index 7a80138ad4..c83da13829 100644 --- a/frontend/src/lib/logql/reverseParser.ts +++ b/frontend/src/lib/logql/reverseParser.ts @@ -2,20 +2,34 @@ // @ts-ignore // @ts-nocheck +import { QueryTypes, StringTypeQueryOperators } from "./tokens"; + export const reverseParser = ( parserQueryArr: { type: string; value: any }[] = [], ) => { let queryString = ''; + let lastToken: { type: string; value: any }; parserQueryArr.forEach((query) => { if (queryString) { queryString += ' '; } if (Array.isArray(query.value) && query.value.length > 0) { + // if the values are array type, here we spread them in + // ('a', 'b') format queryString += `(${query.value.map((val) => `'${val}'`).join(',')})`; } else { - queryString += query.value; + if (query.type === QueryTypes.QUERY_VALUE + && lastToken.type === QueryTypes.QUERY_OPERATOR + && Object.values(StringTypeQueryOperators).includes(lastToken.value) ) { + // for operators that need string type value, here we append single + // quotes. if the content has single quote they would be removed + queryString += `'${query.value?.replace(/'/g, '')}'`; + } else { + queryString += query.value; + } } + lastToken = query; }); // console.log(queryString); diff --git a/frontend/src/lib/logql/tokens.ts b/frontend/src/lib/logql/tokens.ts index 1001f29f6a..e4eb64fe3c 100644 --- a/frontend/src/lib/logql/tokens.ts +++ b/frontend/src/lib/logql/tokens.ts @@ -7,6 +7,21 @@ export const QueryOperatorsSingleVal = { NCONTAINS: 'NCONTAINS', }; +// list of operators that support only number values +export const NumTypeQueryOperators = { + GTE: 'GTE', + GT: 'GT', + LTE: 'LTE', + LT: 'LT', +}; + +// list of operators that support only string values +export const StringTypeQueryOperators = { + CONTAINS: 'CONTAINS', + NCONTAINS: 'NCONTAINS', +}; + +// list of operators that support array values export const QueryOperatorsMultiVal = { IN: 'IN', NIN: 'NIN', @@ -23,3 +38,46 @@ export const QueryTypes = { QUERY_VALUE: 'QUERY_VALUE', CONDITIONAL_OPERATOR: 'CONDITIONAL_OPERATOR', }; + +export const ValidTypeValue = ( + op: string, + value: string | string[], +): boolean => { + if (!op) return true; + if (Object.values(NumTypeQueryOperators).includes(op)) { + if (Array.isArray(value)) return false; + return !Number.isNaN(Number(value)); + } + return true; +}; + +// ValidTypeSequence takes prior, current and next op to confirm +// the proper sequence. For example, if QUERY_VALUE needs to be +// in between QUERY_OPERATOR and (empty or CONDITIONAL_OPERATOR). +export const ValidTypeSequence = ( + prior: string | undefined, + current: string | undefined, + next: string | undefined, +): boolean => { + switch (current) { + case QueryTypes.QUERY_KEY: + // query key can have an empty prior + if (!prior) return true; + return [QueryTypes.CONDITIONAL_OPERATOR].includes(prior); + case QueryTypes.QUERY_OPERATOR: + // empty prior is not allowed + if (!prior || ![QueryTypes.QUERY_KEY].includes(prior)) return false; + if (!next || ![QueryTypes.QUERY_VALUE].includes(next)) return false; + return true; + case QueryTypes.QUERY_VALUE: + // empty prior is not allowed + if (!prior) return false; + return [QueryTypes.QUERY_OPERATOR].includes(prior); + case QueryTypes.CONDITIONAL_OPERATOR: + // empty prior is not allowed + if (!next) return false; + return [QueryTypes.QUERY_KEY].includes(next); + default: + return false; + } +}; From 09af6c262c935f5d92f5e7afdbc5866cce446844 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Mon, 26 Dec 2022 15:29:49 +0530 Subject: [PATCH 06/36] fix: proxy_read_timeout updated in nginx conf (#1885) * fix: proxy_read_timeout updated in nginx conf * fix: live tail endpoint-flush the headers first Co-authored-by: Prashant Shahi Co-authored-by: Ankit Nayan --- deploy/docker-swarm/common/nginx-config.conf | 2 ++ deploy/docker/common/nginx-config.conf | 2 ++ pkg/query-service/app/http_handler.go | 2 ++ 3 files changed, 6 insertions(+) diff --git a/deploy/docker-swarm/common/nginx-config.conf b/deploy/docker-swarm/common/nginx-config.conf index 738805f89f..a8673496a2 100644 --- a/deploy/docker-swarm/common/nginx-config.conf +++ b/deploy/docker-swarm/common/nginx-config.conf @@ -30,6 +30,8 @@ server { location /api { proxy_pass http://query-service:8080/api; + # connection will be closed if no data is read for 600s between successive read operations + proxy_read_timeout 600s; } # redirect server error pages to the static page /50x.html diff --git a/deploy/docker/common/nginx-config.conf b/deploy/docker/common/nginx-config.conf index 738805f89f..a8673496a2 100644 --- a/deploy/docker/common/nginx-config.conf +++ b/deploy/docker/common/nginx-config.conf @@ -30,6 +30,8 @@ server { location /api { proxy_pass http://query-service:8080/api; + # connection will be closed if no data is read for 600s between successive read operations + proxy_read_timeout 600s; } # redirect server error pages to the static page /50x.html diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 1dc342390e..cb1ef91dff 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -2190,6 +2190,8 @@ func (aH *APIHandler) tailLogs(w http.ResponseWriter, r *http.Request) { RespondError(w, &err, "streaming is not supported") return } + // flush the headers + flusher.Flush() for { select { From 02898d14f978ebea7099193bca6e0778b464a11d Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Mon, 26 Dec 2022 15:42:08 +0530 Subject: [PATCH 07/36] fix: removes password validations other than length (#1909) --- pkg/query-service/auth/rbac.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/pkg/query-service/auth/rbac.go b/pkg/query-service/auth/rbac.go index b07fbde5f7..d387a30779 100644 --- a/pkg/query-service/auth/rbac.go +++ b/pkg/query-service/auth/rbac.go @@ -2,9 +2,7 @@ package auth import ( "context" - "fmt" "net/http" - "regexp" "github.com/pkg/errors" "go.signoz.io/signoz/pkg/query-service/constants" @@ -74,21 +72,21 @@ func ValidatePassword(password string) error { return errors.Errorf("Password should be atleast %d characters.", minimumPasswordLength) } - num := `[0-9]{1}` - lower := `[a-z]{1}` - upper := `[A-Z]{1}` - symbol := `[!@#$&*]{1}` - if b, err := regexp.MatchString(num, password); !b || err != nil { - return fmt.Errorf("password should have atleast one number") - } - if b, err := regexp.MatchString(lower, password); !b || err != nil { - return fmt.Errorf("password should have atleast one lower case letter") - } - if b, err := regexp.MatchString(upper, password); !b || err != nil { - return fmt.Errorf("password should have atleast one upper case letter") - } - if b, err := regexp.MatchString(symbol, password); !b || err != nil { - return fmt.Errorf("password should have atleast one special character from !@#$&* ") - } + // num := `[0-9]{1}` + // lower := `[a-z]{1}` + // upper := `[A-Z]{1}` + // symbol := `[!@#$&*]{1}` + // if b, err := regexp.MatchString(num, password); !b || err != nil { + // return fmt.Errorf("password should have atleast one number") + // } + // if b, err := regexp.MatchString(lower, password); !b || err != nil { + // return fmt.Errorf("password should have atleast one lower case letter") + // } + // if b, err := regexp.MatchString(upper, password); !b || err != nil { + // return fmt.Errorf("password should have atleast one upper case letter") + // } + // if b, err := regexp.MatchString(symbol, password); !b || err != nil { + // return fmt.Errorf("password should have atleast one special character from !@#$&* ") + // } return nil } From 1cceab4d5e0633478a1ba6eafe2cb95c00fb9e85 Mon Sep 17 00:00:00 2001 From: Marius Kimmina <38843153+mariuskimmina@users.noreply.github.com> Date: Mon, 26 Dec 2022 11:32:18 +0100 Subject: [PATCH 08/36] fix(FE): remove unnecessary complexity from password check (#1904) Signed-off-by: Marius Kimmina --- frontend/src/pages/SignUp/utils.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frontend/src/pages/SignUp/utils.ts b/frontend/src/pages/SignUp/utils.ts index a71c4029cf..b6c746fcd6 100644 --- a/frontend/src/pages/SignUp/utils.ts +++ b/frontend/src/pages/SignUp/utils.ts @@ -6,10 +6,8 @@ */ export const isPasswordValid = (value: string): boolean => { // eslint-disable-next-line prefer-regex-literals - const pattern = new RegExp( - '^(?=.*?[A-Z])(?=.*?[a-z])(?=.*?[0-9])(?=.*?[#?!@$%^&*-]).{8,}$', - ); + const pattern = new RegExp('^.{8,}$'); return pattern.test(value); }; -export const isPasswordNotValidMessage = `Password must a have minimum of 8 characters with at least one lower case, one number ,one upper case and one special character`; +export const isPasswordNotValidMessage = `Password must a have minimum of 8 characters`; From ac446294e72a8695adb9051437d7dc3bc31047d0 Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Mon, 26 Dec 2022 16:20:34 +0530 Subject: [PATCH 09/36] fix: logs selection of filter is fixed (#1910) Co-authored-by: Ankit Nayan --- .../src/components/Logs/AddToQueryHOC.tsx | 114 ++---------------- 1 file changed, 12 insertions(+), 102 deletions(-) diff --git a/frontend/src/components/Logs/AddToQueryHOC.tsx b/frontend/src/components/Logs/AddToQueryHOC.tsx index 3978ca5948..ea517fa003 100644 --- a/frontend/src/components/Logs/AddToQueryHOC.tsx +++ b/frontend/src/components/Logs/AddToQueryHOC.tsx @@ -1,46 +1,21 @@ import { Button, Popover } from 'antd'; -import getStep from 'lib/getStep'; import { generateFilterQuery } from 'lib/logs/generateFilterQuery'; import React, { memo, useCallback, useMemo } from 'react'; -import { connect, useDispatch, useSelector } from 'react-redux'; -import { bindActionCreators, Dispatch } from 'redux'; -import { ThunkDispatch } from 'redux-thunk'; -import { getLogs } from 'store/actions/logs/getLogs'; -import { getLogsAggregate } from 'store/actions/logs/getLogsAggregate'; +import { useDispatch, useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; -import AppActions from 'types/actions'; -import { SET_SEARCH_QUERY_STRING, TOGGLE_LIVE_TAIL } from 'types/actions/logs'; -import { GlobalReducer } from 'types/reducer/globalTime'; +import { SET_SEARCH_QUERY_STRING } from 'types/actions/logs'; import { ILogsReducer } from 'types/reducer/logs'; -interface AddToQueryHOCProps { - fieldKey: string; - fieldValue: string; - children: React.ReactNode; - getLogs: (props: Parameters[0]) => ReturnType; - getLogsAggregate: ( - props: Parameters[0], - ) => ReturnType; -} function AddToQueryHOC({ fieldKey, fieldValue, children, - getLogs, - getLogsAggregate, }: AddToQueryHOCProps): JSX.Element { const { searchFilter: { queryString }, - logLinesPerPage, - idStart, - idEnd, - liveTail, } = useSelector((store) => store.logs); const dispatch = useDispatch(); - const { maxTime, minTime } = useSelector( - (state) => state.globalTime, - ); const generatedQuery = useMemo( () => generateFilterQuery({ fieldKey, fieldValue, type: 'IN' }), [fieldKey, fieldValue], @@ -58,69 +33,14 @@ function AddToQueryHOC({ type: SET_SEARCH_QUERY_STRING, payload: updatedQueryString, }); - if (liveTail === 'STOPPED') { - getLogs({ - q: updatedQueryString, - limit: logLinesPerPage, - orderBy: 'timestamp', - order: 'desc', - timestampStart: minTime, - timestampEnd: maxTime, - ...(idStart ? { idGt: idStart } : {}), - ...(idEnd ? { idLt: idEnd } : {}), - }); - getLogsAggregate({ - timestampStart: minTime, - timestampEnd: maxTime, - step: getStep({ - start: minTime, - end: maxTime, - inputFormat: 'ns', - }), - q: updatedQueryString, - ...(idStart ? { idGt: idStart } : {}), - ...(idEnd ? { idLt: idEnd } : {}), - }); - } else if (liveTail === 'PLAYING') { - dispatch({ - type: TOGGLE_LIVE_TAIL, - payload: 'PAUSED', - }); - setTimeout( - () => - dispatch({ - type: TOGGLE_LIVE_TAIL, - payload: liveTail, - }), - 0, - ); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [ - dispatch, - generatedQuery, - getLogs, - idEnd, - idStart, - logLinesPerPage, - maxTime, - minTime, - queryString, + }, [dispatch, generatedQuery, queryString]); + + const popOverContent = useMemo(() => Add to query: {fieldKey}, [ + fieldKey, ]); - const popOverContent = ( - Add to query: {fieldKey} - ); return ( - ); } +interface Props { + text?: string; +} + +NotFound.defaultProps = { + text: defaultText, +}; + export default NotFound; diff --git a/frontend/src/pages/TraceDetail/constants.ts b/frontend/src/pages/TraceDetail/constants.ts index 0253cbeff0..42fb237070 100644 --- a/frontend/src/pages/TraceDetail/constants.ts +++ b/frontend/src/pages/TraceDetail/constants.ts @@ -1 +1,4 @@ export const SPAN_DETAILS_LEFT_COL_WIDTH = 350; + +export const noEventMessage = + 'The requested trace id was not found. Sometimes this happens because of insertion delay in trace data. Please try again after some time'; diff --git a/frontend/src/pages/TraceDetail/index.tsx b/frontend/src/pages/TraceDetail/index.tsx index 16316aa729..97c3140408 100644 --- a/frontend/src/pages/TraceDetail/index.tsx +++ b/frontend/src/pages/TraceDetail/index.tsx @@ -1,5 +1,6 @@ import { Typography } from 'antd'; import getTraceItem from 'api/trace/getTraceItem'; +import NotFound from 'components/NotFound'; import Spinner from 'components/Spinner'; import TraceDetailContainer from 'container/TraceDetail'; import useUrlQuery from 'hooks/useUrlQuery'; @@ -8,6 +9,8 @@ import { useQuery } from 'react-query'; import { useParams } from 'react-router-dom'; import { Props as TraceDetailProps } from 'types/api/trace/getTraceItem'; +import { noEventMessage } from './constants'; + function TraceDetail(): JSX.Element { const { id } = useParams(); const urlQuery = useUrlQuery(); @@ -19,6 +22,7 @@ function TraceDetail(): JSX.Element { }), [urlQuery], ); + const { data: traceDetailResponse, error, isLoading, isError } = useQuery( `getTraceItem/${id}`, () => getTraceItem({ id, spanId, levelUp, levelDown }), @@ -39,6 +43,10 @@ function TraceDetail(): JSX.Element { return ; } + if (traceDetailResponse.payload[0].events.length === 0) { + return ; + } + return ; } From 0595cdc7af861226f106e854680e0bb86c6b5243 Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Mon, 26 Dec 2022 17:14:54 +0530 Subject: [PATCH 12/36] fix: scroll is added (#1873) Co-authored-by: Ankit Nayan --- frontend/src/container/TraceDetail/styles.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/container/TraceDetail/styles.ts b/frontend/src/container/TraceDetail/styles.ts index 5f7a8f84c2..b69b27bb26 100644 --- a/frontend/src/container/TraceDetail/styles.ts +++ b/frontend/src/container/TraceDetail/styles.ts @@ -34,9 +34,10 @@ export const traceDateAndTimelineContainer = css` export const traceDateTimeContainer = css` display: flex; - aligh-items: center; + align-items: center; justify-content: center; `; + export const timelineContainer = css` overflow: visible; margin: 0 1rem 0 0; @@ -48,7 +49,7 @@ export const ganttChartContainer = css` position: relative; flex: 1; overflow-y: auto; - overflow-x: hidden; + overflow-x: scroll; `; export const selectedSpanDetailContainer = css` From 3dcb44a75862cc4dab238fec6eb49bb202130a0a Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Mon, 26 Dec 2022 17:16:47 +0530 Subject: [PATCH 13/36] fix docker-compose for swarm and related changes for distributed clickhouse (#1863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: 🔧 fix docker-compose.yaml for swarm Signed-off-by: Prashant Shahi * chore: 🔧 add .gitkeep files for docker and swarm Signed-off-by: Prashant Shahi Signed-off-by: Prashant Shahi Co-authored-by: Ankit Nayan --- .../clickhouse-setup/data/clickhouse-2/.gitkeep | 0 .../clickhouse-setup/data/clickhouse-3/.gitkeep | 0 .../docker-swarm/clickhouse-setup/data/zookeeper-1/.gitkeep | 0 .../docker-swarm/clickhouse-setup/data/zookeeper-2/.gitkeep | 0 .../docker-swarm/clickhouse-setup/data/zookeeper-3/.gitkeep | 0 deploy/docker-swarm/clickhouse-setup/docker-compose.yaml | 6 ------ deploy/docker/clickhouse-setup/data/clickhouse-2/.gitkeep | 0 deploy/docker/clickhouse-setup/data/clickhouse-3/.gitkeep | 0 deploy/docker/clickhouse-setup/data/zookeeper-1/.gitkeep | 0 deploy/docker/clickhouse-setup/data/zookeeper-2/.gitkeep | 0 deploy/docker/clickhouse-setup/data/zookeeper-3/.gitkeep | 0 11 files changed, 6 deletions(-) create mode 100644 deploy/docker-swarm/clickhouse-setup/data/clickhouse-2/.gitkeep create mode 100644 deploy/docker-swarm/clickhouse-setup/data/clickhouse-3/.gitkeep create mode 100644 deploy/docker-swarm/clickhouse-setup/data/zookeeper-1/.gitkeep create mode 100644 deploy/docker-swarm/clickhouse-setup/data/zookeeper-2/.gitkeep create mode 100644 deploy/docker-swarm/clickhouse-setup/data/zookeeper-3/.gitkeep create mode 100644 deploy/docker/clickhouse-setup/data/clickhouse-2/.gitkeep create mode 100644 deploy/docker/clickhouse-setup/data/clickhouse-3/.gitkeep create mode 100644 deploy/docker/clickhouse-setup/data/zookeeper-1/.gitkeep create mode 100644 deploy/docker/clickhouse-setup/data/zookeeper-2/.gitkeep create mode 100644 deploy/docker/clickhouse-setup/data/zookeeper-3/.gitkeep diff --git a/deploy/docker-swarm/clickhouse-setup/data/clickhouse-2/.gitkeep b/deploy/docker-swarm/clickhouse-setup/data/clickhouse-2/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker-swarm/clickhouse-setup/data/clickhouse-3/.gitkeep b/deploy/docker-swarm/clickhouse-setup/data/clickhouse-3/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker-swarm/clickhouse-setup/data/zookeeper-1/.gitkeep b/deploy/docker-swarm/clickhouse-setup/data/zookeeper-1/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker-swarm/clickhouse-setup/data/zookeeper-2/.gitkeep b/deploy/docker-swarm/clickhouse-setup/data/zookeeper-2/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker-swarm/clickhouse-setup/data/zookeeper-3/.gitkeep b/deploy/docker-swarm/clickhouse-setup/data/zookeeper-3/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index f7506fa8fb..449f5cf15a 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -35,7 +35,6 @@ x-clickhouse-depend: &clickhouse-depend services: zookeeper-1: image: bitnami/zookeeper:3.7.0 - container_name: zookeeper-1 hostname: zookeeper-1 user: root ports: @@ -52,7 +51,6 @@ services: # zookeeper-2: # image: bitnami/zookeeper:3.7.0 - # container_name: zookeeper-2 # hostname: zookeeper-2 # user: root # ports: @@ -69,7 +67,6 @@ services: # zookeeper-3: # image: bitnami/zookeeper:3.7.0 - # container_name: zookeeper-3 # hostname: zookeeper-3 # user: root # ports: @@ -86,7 +83,6 @@ services: clickhouse: <<: *clickhouse-defaults - container_name: clickhouse hostname: clickhouse # ports: # - "9000:9000" @@ -101,7 +97,6 @@ services: # clickhouse-2: # <<: *clickhouse-defaults - # container_name: clickhouse-2 # hostname: clickhouse-2 # ports: # - "9001:9000" @@ -116,7 +111,6 @@ services: # clickhouse-3: # <<: *clickhouse-defaults - # container_name: clickhouse-3 # hostname: clickhouse-3 # ports: # - "9002:9000" diff --git a/deploy/docker/clickhouse-setup/data/clickhouse-2/.gitkeep b/deploy/docker/clickhouse-setup/data/clickhouse-2/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker/clickhouse-setup/data/clickhouse-3/.gitkeep b/deploy/docker/clickhouse-setup/data/clickhouse-3/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker/clickhouse-setup/data/zookeeper-1/.gitkeep b/deploy/docker/clickhouse-setup/data/zookeeper-1/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker/clickhouse-setup/data/zookeeper-2/.gitkeep b/deploy/docker/clickhouse-setup/data/zookeeper-2/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker/clickhouse-setup/data/zookeeper-3/.gitkeep b/deploy/docker/clickhouse-setup/data/zookeeper-3/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 From 6f6499c267fbc7c340061865b7d9de80c907cda4 Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Mon, 26 Dec 2022 17:25:55 +0530 Subject: [PATCH 14/36] fix: flush logs before starting (#1912) Co-authored-by: Ankit Nayan --- frontend/src/container/LogsSearchFilter/index.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/container/LogsSearchFilter/index.tsx b/frontend/src/container/LogsSearchFilter/index.tsx index c8f4a9021e..c7e79b14ca 100644 --- a/frontend/src/container/LogsSearchFilter/index.tsx +++ b/frontend/src/container/LogsSearchFilter/index.tsx @@ -16,7 +16,7 @@ import { getLogs } from 'store/actions/logs/getLogs'; import { getLogsAggregate } from 'store/actions/logs/getLogsAggregate'; import { AppState } from 'store/reducers'; import AppActions from 'types/actions'; -import { TOGGLE_LIVE_TAIL } from 'types/actions/logs'; +import { FLUSH_LOGS, TOGGLE_LIVE_TAIL } from 'types/actions/logs'; import { GlobalReducer } from 'types/reducer/globalTime'; import { ILogsReducer } from 'types/reducer/logs'; @@ -69,6 +69,9 @@ function SearchFilter({ type: TOGGLE_LIVE_TAIL, payload: 'PAUSED', }); + dispatch({ + type: FLUSH_LOGS, + }); setTimeout( () => dispatch({ From 4b13b0a8a4f4ab041d197d3cce16bc0f276ee407 Mon Sep 17 00:00:00 2001 From: Amol Umbark Date: Mon, 26 Dec 2022 20:31:50 +0530 Subject: [PATCH 15/36] fix: resolves issue related ops not flowing from search box to panel (#1918) --- .../container/LogsSearchFilter/SearchFields/index.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx b/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx index 5e11ec1a72..bca82bdaaf 100644 --- a/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx +++ b/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx @@ -36,7 +36,14 @@ function SearchFields({ const keyPrefixRef = useRef(hashCode(JSON.stringify(fieldsQuery))); useEffect(() => { - setFieldsQuery(createParsedQueryStructure([...parsedQuery] as never[])); + const updatedFieldsQuery = createParsedQueryStructure([ + ...parsedQuery, + ] as never[]); + setFieldsQuery(updatedFieldsQuery); + const incomingHashCode = hashCode(JSON.stringify(updatedFieldsQuery)); + if (incomingHashCode !== keyPrefixRef.current) { + keyPrefixRef.current = incomingHashCode; + } }, [parsedQuery]); const addSuggestedField = useCallback( From db704b212dc125ea00476106d838291f80cd6ec0 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Mon, 26 Dec 2022 21:52:54 +0530 Subject: [PATCH 16/36] fix: reduced rate limit to 2 of each events in 1 min --- pkg/query-service/telemetry/telemetry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 43f1652093..ef94820baa 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -44,7 +44,7 @@ const HEART_BEAT_DURATION = 6 * time.Hour // const HEART_BEAT_DURATION = 10 * time.Second const RATE_LIMIT_CHECK_DURATION = 1 * time.Minute -const RATE_LIMIT_VALUE = 10 +const RATE_LIMIT_VALUE = 2 // const RATE_LIMIT_CHECK_DURATION = 20 * time.Second // const RATE_LIMIT_VALUE = 5 From ece2988d0d3edf4647d79f156331cec85da29a08 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Mon, 26 Dec 2022 22:11:23 +0530 Subject: [PATCH 17/36] feat: added new event for length of filters in logs search page --- .../app/clickhouseReader/reader.go | 22 ++++++++++++++++--- pkg/query-service/app/logs/parser.go | 8 +++---- pkg/query-service/telemetry/telemetry.go | 1 + 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ab07dbbd23..d7da4a41e5 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -45,6 +45,7 @@ import ( am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager" "go.signoz.io/signoz/pkg/query-service/interfaces" "go.signoz.io/signoz/pkg/query-service/model" + "go.signoz.io/signoz/pkg/query-service/telemetry" "go.signoz.io/signoz/pkg/query-service/utils" "go.uber.org/zap" ) @@ -3212,11 +3213,16 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter } isPaginatePrev := logs.CheckIfPrevousPaginateAndModifyOrder(params) - filterSql, err := logs.GenerateSQLWhere(fields, params) + filterSql, lenFilters, err := logs.GenerateSQLWhere(fields, params) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} } + data := map[string]interface{}{ + "lenFilters": lenFilters, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LOGS_FILTERS, data) + query := fmt.Sprintf("%s from %s.%s", constants.LogsSQLSelect, r.logsDB, r.logsTable) if filterSql != "" { @@ -3246,10 +3252,15 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC return } - filterSql, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ + filterSql, lenFilters, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ Query: client.Filter.Query, }) + data := map[string]interface{}{ + "lenFilters": lenFilters, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LOGS_FILTERS, data) + if err != nil { client.Error <- err return @@ -3326,13 +3337,18 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs return nil, apiErr } - filterSql, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ + filterSql, lenFilters, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ Query: params.Query, }) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} } + data := map[string]interface{}{ + "lenFilters": lenFilters, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LOGS_FILTERS, data) + query := "" if params.GroupBy != "" { query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000000000), INTERVAL %d minute))*1000000000) as ts_start_interval, toString(%s) as groupBy, "+ diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 5e4a195817..5e63abdc34 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -279,20 +279,20 @@ func CheckIfPrevousPaginateAndModifyOrder(params *model.LogsFilterParams) (isPag return } -func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, error) { +func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, int, error) { var tokens []string var err error var sqlWhere string if params.Query != "" { tokens, err = parseLogQuery(params.Query) if err != nil { - return sqlWhere, err + return sqlWhere, -1, err } } tokens, err = replaceInterestingFields(allFields, tokens) if err != nil { - return sqlWhere, err + return sqlWhere, -1, err } filterTokens := []string{} @@ -342,5 +342,5 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt sqlWhere = strings.Join(tokens, "") - return sqlWhere, nil + return sqlWhere, len(tokens), nil } diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index ef94820baa..17f93f6339 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -32,6 +32,7 @@ const ( TELEMETRY_LICENSE_ACT_FAILED = "License Activation Failed" TELEMETRY_EVENT_ENVIRONMENT = "Environment" TELEMETRY_EVENT_LANGUAGE = "Language" + TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" ) const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" From a3455fb5537750b49bc273882c2366f6950cedbe Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Mon, 26 Dec 2022 23:01:54 +0530 Subject: [PATCH 18/36] feat: added distributed cluster info --- pkg/query-service/app/clickhouseReader/reader.go | 14 ++++++++++++++ pkg/query-service/interfaces/interface.go | 1 + pkg/query-service/model/response.go | 16 ++++++++++++++++ pkg/query-service/telemetry/telemetry.go | 5 +++++ 4 files changed, 36 insertions(+) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index d7da4a41e5..0711382acf 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3068,6 +3068,20 @@ func (r *ClickHouseReader) GetSamplesInfoInLastHeartBeatInterval(ctx context.Con return totalSamples, nil } + +func (r *ClickHouseReader) GetDistributedInfoInLastHeartBeatInterval(ctx context.Context) (map[string]interface{}, error) { + + clusterInfo := []model.ClusterInfo{} + + queryStr := `SELECT shard_num, shard_weight, replica_num, errors_count, slowdowns_count, estimated_recovery_time FROM system.clusters where cluster='cluster';` + r.db.Select(ctx, &clusterInfo, queryStr) + if len(clusterInfo) == 1 { + return clusterInfo[0].GetMapFromStruct(), nil + } + + return nil, nil +} + func (r *ClickHouseReader) GetLogsInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) { var totalLogLines uint64 diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 3200b10c2f..bcaf889ab6 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -63,6 +63,7 @@ type Reader interface { GetSamplesInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetLogsInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetTagsInfoInLastHeartBeatInterval(ctx context.Context) (*model.TagsInfo, error) + GetDistributedInfoInLastHeartBeatInterval(ctx context.Context) (map[string]interface{}, error) // Logs GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 7ae79be456..0f1d60d9c2 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -564,3 +564,19 @@ type TagTelemetryData struct { Env string `json:"env" ch:"env"` Language string `json:"language" ch:"language"` } + +type ClusterInfo struct { + ShardNum uint32 `json:"shard_num" ch:"shard_num"` + ShardWeight uint32 `json:"shard_weight" ch:"shard_weight"` + ReplicaNum uint32 `json:"replica_num" ch:"replica_num"` + ErrorsCount uint32 `json:"errors_count" ch:"errors_count"` + SlowdownsCount uint32 `json:"slowdowns_count" ch:"slowdowns_count"` + EstimatedRecoveryTime uint32 `json:"estimated_recovery_time" ch:"estimated_recovery_time"` +} + +func (ci *ClusterInfo) GetMapFromStruct() map[string]interface{} { + var clusterInfoMap map[string]interface{} + data, _ := json.Marshal(*ci) + json.Unmarshal(data, &clusterInfoMap) + return clusterInfoMap +} diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 17f93f6339..e90207475a 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -33,6 +33,7 @@ const ( TELEMETRY_EVENT_ENVIRONMENT = "Environment" TELEMETRY_EVENT_LANGUAGE = "Language" TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" + TELEMETRY_EVENT_DISTRIBUTED = "Distributed" ) const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" @@ -139,6 +140,10 @@ func createTelemetry() { data[key] = value } telemetry.SendEvent(TELEMETRY_EVENT_HEART_BEAT, data) + + getDistributedInfoInLastHeartBeatInterval, _ := telemetry.reader.GetDistributedInfoInLastHeartBeatInterval(context.Background()) + telemetry.SendEvent(TELEMETRY_EVENT_DISTRIBUTED, getDistributedInfoInLastHeartBeatInterval) + } } }() From 18bbb3cf36fe526b1fbe8a1558f7ab0edaa53a99 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Mon, 26 Dec 2022 23:10:55 +0530 Subject: [PATCH 19/36] fix: length of filters in logs --- pkg/query-service/app/logs/parser.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 5e63abdc34..18760ba972 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -283,11 +283,14 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt var tokens []string var err error var sqlWhere string + var lenTokens = 0 if params.Query != "" { tokens, err = parseLogQuery(params.Query) + if err != nil { return sqlWhere, -1, err } + lenTokens = len(tokens) } tokens, err = replaceInterestingFields(allFields, tokens) @@ -342,5 +345,5 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt sqlWhere = strings.Join(tokens, "") - return sqlWhere, len(tokens), nil + return sqlWhere, lenTokens, nil } From aad962d07d1bd04f3e3fb1b8c5f1eeda373f5c4f Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Tue, 27 Dec 2022 01:10:01 +0530 Subject: [PATCH 20/36] feat: dashboard metadata with no rateLimit --- pkg/query-service/app/server.go | 50 ++++++++++++++++++++++++ pkg/query-service/telemetry/telemetry.go | 18 ++++++--- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index b79b7d011f..498d8b02f3 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -1,8 +1,11 @@ package app import ( + "bytes" "context" + "encoding/json" "fmt" + "io/ioutil" "net" "net/http" _ "net/http/pprof" // http profiler @@ -235,15 +238,62 @@ func (lrw *loggingResponseWriter) Flush() { lrw.ResponseWriter.(http.Flusher).Flush() } +func extractDashboardMetaData(path string, r *http.Request) (map[string]interface{}, bool) { + pathToExtractBodyFrom := "/api/v2/metrics/query_range" + var requestBody map[string]interface{} + data := map[string]interface{}{} + + if path == pathToExtractBodyFrom && (r.Method == "POST") { + bodyBytes, _ := ioutil.ReadAll(r.Body) + r.Body.Close() // must close + r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + + json.Unmarshal(bodyBytes, &requestBody) + + } else { + return nil, false + } + + compositeMetricQuery, compositeMetricQueryExists := requestBody["compositeMetricQuery"] + compositeMetricQueryMap := compositeMetricQuery.(map[string]interface{}) + if compositeMetricQueryExists { + queryType, queryTypeExists := compositeMetricQueryMap["queryType"] + if queryTypeExists { + data["queryType"] = queryType + } + panelType, panelTypeExists := compositeMetricQueryMap["panelType"] + if panelTypeExists { + data["panelType"] = panelType + } + } + + datasource, datasourceExists := requestBody["dataSource"] + if datasourceExists { + data["datasource"] = datasource + } + + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_DASHBOARDS_METADATA, data, false) + + return data, true +} + func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route := mux.CurrentRoute(r) path, _ := route.GetPathTemplate() + dashboardMetadata, metadataExists := extractDashboardMetaData(path, r) + lrw := NewLoggingResponseWriter(w) next.ServeHTTP(lrw, r) data := map[string]interface{}{"path": path, "statusCode": lrw.statusCode} + if metadataExists { + for key, value := range dashboardMetadata { + data[key] = value + } + } + if telemetry.GetInstance().IsSampled() { if _, ok := telemetry.IgnoredPaths()[path]; !ok { telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data) diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index e90207475a..0c1d014c1f 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -34,6 +34,7 @@ const ( TELEMETRY_EVENT_LANGUAGE = "Language" TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" TELEMETRY_EVENT_DISTRIBUTED = "Distributed" + TELEMETRY_EVENT_DASHBOARDS_METADATA = "Dashboards Metadata" ) const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" @@ -213,7 +214,12 @@ func (a *Telemetry) checkEvents(event string) bool { return sendEvent } -func (a *Telemetry) SendEvent(event string, data map[string]interface{}) { +func (a *Telemetry) SendEvent(event string, data map[string]interface{}, opts ...bool) { + + rateLimitFlag := true + if len(opts) > 0 { + rateLimitFlag = opts[0] + } if !a.isTelemetryEnabled() { return @@ -224,10 +230,12 @@ func (a *Telemetry) SendEvent(event string, data map[string]interface{}) { return } - if a.rateLimits[event] < RATE_LIMIT_VALUE { - a.rateLimits[event] += 1 - } else { - return + if rateLimitFlag { + if a.rateLimits[event] < RATE_LIMIT_VALUE { + a.rateLimits[event] += 1 + } else { + return + } } // zap.S().Info(data) From eef48c54f837f0d5f0614ab0767afd973c3169d4 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 27 Dec 2022 11:28:15 +0530 Subject: [PATCH 21/36] fix(query_range): invalid memory address or nil pointer dereference (#1875) Co-authored-by: Ankit Nayan --- pkg/query-service/app/http_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index cb1ef91dff..69c0f92b8d 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1159,6 +1159,7 @@ func (aH *APIHandler) queryRangeMetrics(w http.ResponseWriter, r *http.Request) RespondError(w, &model.ApiError{model.ErrorTimeout, res.Err}, nil) } RespondError(w, &model.ApiError{model.ErrorExec, res.Err}, nil) + return } response_data := &model.QueryData{ From 13a6d7f7c6d0007466f20e802522152be58bc6ba Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Tue, 27 Dec 2022 13:36:37 +0530 Subject: [PATCH 22/36] fix: live tail time out is updated (#1899) * fix: live tail time out is updated * Update livetail.ts Co-authored-by: Pranay Prateek Co-authored-by: Ankit Nayan --- frontend/src/api/logs/livetail.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/frontend/src/api/logs/livetail.ts b/frontend/src/api/logs/livetail.ts index f1cec5472a..150f63d193 100644 --- a/frontend/src/api/logs/livetail.ts +++ b/frontend/src/api/logs/livetail.ts @@ -4,14 +4,16 @@ import { ENVIRONMENT } from 'constants/env'; import { LOCALSTORAGE } from 'constants/localStorage'; import { EventSourcePolyfill } from 'event-source-polyfill'; -export const LiveTail = (queryParams: string): EventSourcePolyfill => { - const dict = { - headers: { - Authorization: `Bearer ${getLocalStorageKey(LOCALSTORAGE.AUTH_TOKEN)}`, - }, - }; - return new EventSourcePolyfill( +// 10 min in ms +const TIMEOUT_IN_MS = 10 * 60 * 1000; + +export const LiveTail = (queryParams: string): EventSourcePolyfill => + new EventSourcePolyfill( `${ENVIRONMENT.baseURL}${apiV1}logs/tail?${queryParams}`, - dict, + { + headers: { + Authorization: `Bearer ${getLocalStorageKey(LOCALSTORAGE.AUTH_TOKEN)}`, + }, + heartbeatTimeout: TIMEOUT_IN_MS, + }, ); -}; From 48a6f536fad59d38b5f791d18d56faa342663498 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 27 Dec 2022 15:44:39 +0530 Subject: [PATCH 23/36] chore: increase dimensions_cache_size for signozspanmetrics processor (#1925) --- deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml | 2 +- deploy/docker/clickhouse-setup/otel-collector-config.yaml | 2 +- pkg/query-service/tests/test-deploy/otel-collector-config.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml index c198c2f57c..0636b518cf 100644 --- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml @@ -78,7 +78,7 @@ processors: signozspanmetrics/prometheus: metrics_exporter: prometheus latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] - dimensions_cache_size: 10000 + dimensions_cache_size: 100000 dimensions: - name: service.namespace default: default diff --git a/deploy/docker/clickhouse-setup/otel-collector-config.yaml b/deploy/docker/clickhouse-setup/otel-collector-config.yaml index 2829bf966f..c6773d187d 100644 --- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml +++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml @@ -74,7 +74,7 @@ processors: signozspanmetrics/prometheus: metrics_exporter: prometheus latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] - dimensions_cache_size: 10000 + dimensions_cache_size: 100000 dimensions: - name: service.namespace default: default diff --git a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml index 9f7ee3fd18..517c7bc643 100644 --- a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml +++ b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml @@ -74,7 +74,7 @@ processors: signozspanmetrics/prometheus: metrics_exporter: prometheus latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] - dimensions_cache_size: 10000 + dimensions_cache_size: 100000 dimensions: - name: service.namespace default: default From 40ec4517c27cde70f8a2436e55741ed01a411e22 Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Tue, 27 Dec 2022 19:01:56 +0530 Subject: [PATCH 24/36] fix: per page is added in the dependancy (#1926) --- frontend/src/container/LogsSearchFilter/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/container/LogsSearchFilter/index.tsx b/frontend/src/container/LogsSearchFilter/index.tsx index c7e79b14ca..b5cb71faa9 100644 --- a/frontend/src/container/LogsSearchFilter/index.tsx +++ b/frontend/src/container/LogsSearchFilter/index.tsx @@ -127,7 +127,7 @@ function SearchFilter({ useEffect(() => { debouncedHandleSearch(urlQueryString || ''); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [urlQueryString, maxTime, minTime, idEnd, idStart]); + }, [urlQueryString, maxTime, minTime, idEnd, idStart, logLinesPerPage]); return ( From 35f5fb69578dc70c896bd9fd94da902837cf5923 Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Tue, 27 Dec 2022 21:09:36 +0530 Subject: [PATCH 25/36] fix: respect durationSort feature flag on getSpanFilters API (#1900) * fix: respect durationSort feature flag on getSpanFilters API * chore: update DB query --- .../app/clickhouseReader/reader.go | 71 ++++++++++++------- pkg/query-service/model/response.go | 5 ++ 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ab07dbbd23..c7f1e45816 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1177,33 +1177,54 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode traceFilterReponse.Status = map[string]uint64{"ok": 0, "error": 0} } case constants.Duration: - finalQuery := fmt.Sprintf("SELECT durationNano as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.durationTable) - finalQuery += query - finalQuery += " ORDER BY durationNano LIMIT 1" - var dBResponse []model.DBResponseTotal - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.S().Info(finalQuery) + err := r.featureFlags.CheckFeature(constants.DurationSort) + durationSortEnabled := err == nil + finalQuery := "" + if !durationSortEnabled { + // if duration sort is not enabled, we need to get the min and max duration from the index table + finalQuery = fmt.Sprintf("SELECT min(durationNano) as min, max(durationNano) as max FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery += query + var dBResponse []model.DBResponseMinMax + err = r.db.Select(ctx, &dBResponse, finalQuery, args...) + zap.S().Info(finalQuery) + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("Error in processing sql query: %s", err)} + } + if len(dBResponse) > 0 { + traceFilterReponse.Duration = map[string]uint64{"minDuration": dBResponse[0].Min, "maxDuration": dBResponse[0].Max} + } + } else { + // when duration sort is enabled, we need to get the min and max duration from the duration table + finalQuery = fmt.Sprintf("SELECT durationNano as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.durationTable) + finalQuery += query + finalQuery += " ORDER BY durationNano LIMIT 1" + var dBResponse []model.DBResponseTotal + err = r.db.Select(ctx, &dBResponse, finalQuery, args...) + zap.S().Info(finalQuery) - if err != nil { - zap.S().Debug("Error in processing sql query: ", err) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("Error in processing sql query: %s", err)} - } - finalQuery = fmt.Sprintf("SELECT durationNano as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.durationTable) - finalQuery += query - finalQuery += " ORDER BY durationNano DESC LIMIT 1" - var dBResponse2 []model.DBResponseTotal - err = r.db.Select(ctx, &dBResponse2, finalQuery, args...) - zap.S().Info(finalQuery) + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("Error in processing sql query: %s", err)} + } - if err != nil { - zap.S().Debug("Error in processing sql query: ", err) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("Error in processing sql query: %s", err)} - } - if len(dBResponse) > 0 { - traceFilterReponse.Duration["minDuration"] = dBResponse[0].NumTotal - } - if len(dBResponse2) > 0 { - traceFilterReponse.Duration["maxDuration"] = dBResponse2[0].NumTotal + finalQuery = fmt.Sprintf("SELECT durationNano as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.durationTable) + finalQuery += query + finalQuery += " ORDER BY durationNano DESC LIMIT 1" + var dBResponse2 []model.DBResponseTotal + err = r.db.Select(ctx, &dBResponse2, finalQuery, args...) + zap.S().Info(finalQuery) + + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("Error in processing sql query: %s", err)} + } + if len(dBResponse) > 0 { + traceFilterReponse.Duration["minDuration"] = dBResponse[0].NumTotal + } + if len(dBResponse2) > 0 { + traceFilterReponse.Duration["maxDuration"] = dBResponse2[0].NumTotal + } } case constants.RPCMethod: finalQuery := fmt.Sprintf("SELECT rpcMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 7ae79be456..d80a56c642 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -399,6 +399,11 @@ type DBResponseTotal struct { NumTotal uint64 `ch:"numTotal"` } +type DBResponseMinMax struct { + Min uint64 `ch:"min"` + Max uint64 `ch:"max"` +} + type SpanFiltersResponse struct { ServiceName map[string]uint64 `json:"serviceName"` Status map[string]uint64 `json:"status"` From 895c721b37f09371bb3f3d828a23c7713146295d Mon Sep 17 00:00:00 2001 From: Yash Joshi Date: Tue, 27 Dec 2022 23:13:13 +0530 Subject: [PATCH 26/36] fix(version): use link instead of click handler (#1931) Co-authored-by: Palash Gupta --- frontend/src/container/Version/index.tsx | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/frontend/src/container/Version/index.tsx b/frontend/src/container/Version/index.tsx index e36a0c6d18..cad08351e3 100644 --- a/frontend/src/container/Version/index.tsx +++ b/frontend/src/container/Version/index.tsx @@ -1,6 +1,6 @@ import { WarningFilled } from '@ant-design/icons'; import { Button, Card, Form, Space, Typography } from 'antd'; -import React, { useCallback } from 'react'; +import React from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; @@ -14,10 +14,6 @@ function Version(): JSX.Element { const [form] = Form.useForm(); const { t } = useTranslation(); - const onClickUpgradeHandler = useCallback((link: string) => { - window.open(link, '_blank'); - }, []); - const { currentVersion, latestVersion, @@ -60,9 +56,8 @@ function Version(): JSX.Element { placeholder={t('latest_version')} /> From c3253687d0a0f7607c4373da17ecb02630bb1ea1 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Wed, 28 Dec 2022 02:09:44 +0530 Subject: [PATCH 27/36] feat: active user --- pkg/query-service/app/http_handler.go | 3 +++ pkg/query-service/app/server.go | 15 +++++++++++++++ pkg/query-service/telemetry/telemetry.go | 20 ++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index cb1ef91dff..1ade0592e9 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1332,6 +1332,9 @@ func (aH *APIHandler) getServices(w http.ResponseWriter, r *http.Request) { } telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_NUMBER_OF_SERVICES, data) + if (data["number"] != 0) || (data["number"] != telemetry.DEFAULT_NUMBER_OF_SERVICES) { + telemetry.GetInstance().AddActiveTracesUser() + } aH.WriteJSON(w, r, result) } diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 498d8b02f3..3f00483d10 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -277,12 +277,27 @@ func extractDashboardMetaData(path string, r *http.Request) (map[string]interfac return data, true } +func getActiveMetricsOrLogs(path string, r *http.Request) { + if path == "/api/v1/dashboards/{uuid}" { + telemetry.GetInstance().AddActiveMetricsUser() + } + if path == "/api/v1/logs" { + hasFilters := len(r.URL.Query().Get("q")) + if hasFilters > 0 { + telemetry.GetInstance().AddActiveLogsUser() + } + + } + +} + func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route := mux.CurrentRoute(r) path, _ := route.GetPathTemplate() dashboardMetadata, metadataExists := extractDashboardMetaData(path, r) + getActiveMetricsOrLogs(path, r) lrw := NewLoggingResponseWriter(w) next.ServeHTTP(lrw, r) diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 0c1d014c1f..4e00933151 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -35,12 +35,14 @@ const ( TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" TELEMETRY_EVENT_DISTRIBUTED = "Distributed" TELEMETRY_EVENT_DASHBOARDS_METADATA = "Dashboards Metadata" + TELEMETRY_EVENT_ACTIVE_USER = "Active User" ) const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" const ph_api_key = "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w" const IP_NOT_FOUND_PLACEHOLDER = "NA" +const DEFAULT_NUMBER_OF_SERVICES = 6 const HEART_BEAT_DURATION = 6 * time.Hour @@ -67,6 +69,16 @@ func (a *Telemetry) IsSampled() bool { } +func (telemetry *Telemetry) AddActiveTracesUser() { + telemetry.activeUser["traces"] = 1 +} +func (telemetry *Telemetry) AddActiveMetricsUser() { + telemetry.activeUser["metrics"] = 1 +} +func (telemetry *Telemetry) AddActiveLogsUser() { + telemetry.activeUser["logs"] = 1 +} + type Telemetry struct { operator analytics.Client phOperator ph.Client @@ -79,6 +91,7 @@ type Telemetry struct { minRandInt int maxRandInt int rateLimits map[string]int8 + activeUser map[string]int8 } func createTelemetry() { @@ -114,6 +127,13 @@ func createTelemetry() { for { select { case <-ticker.C: + + if (telemetry.activeUser["traces"] != 0) || (telemetry.activeUser["metrics"] != 0) || (telemetry.activeUser["logs"] != 0) { + telemetry.activeUser["any"] = 1 + } + telemetry.SendEvent(TELEMETRY_EVENT_ACTIVE_USER, map[string]interface{}{"traces": telemetry.activeUser["traces"], "metrics": telemetry.activeUser["metrics"], "logs": telemetry.activeUser["logs"], "any": telemetry.activeUser["any"]}) + telemetry.activeUser = map[string]int8{"traces": 0, "metrics": 0, "logs": 0, "any": 0} + tagsInfo, _ := telemetry.reader.GetTagsInfoInLastHeartBeatInterval(context.Background()) if len(tagsInfo.Env) != 0 { From b11f79b4c717c2b4a1fc0bfa422c533ff32db829 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Wed, 28 Dec 2022 02:16:46 +0530 Subject: [PATCH 28/36] Chore/analytics (#1922) * fix: reduced rate limit to 2 of each events in 1 min * feat: added new event for length of filters in logs search page * feat: added distributed cluster info * fix: length of filters in logs * feat: dashboard metadata with no rateLimit * feat: active user Co-authored-by: Srikanth Chekuri --- .../app/clickhouseReader/reader.go | 36 +++++++++- pkg/query-service/app/http_handler.go | 3 + pkg/query-service/app/logs/parser.go | 11 ++-- pkg/query-service/app/server.go | 65 +++++++++++++++++++ pkg/query-service/interfaces/interface.go | 1 + pkg/query-service/model/response.go | 16 +++++ pkg/query-service/telemetry/telemetry.go | 46 +++++++++++-- 7 files changed, 165 insertions(+), 13 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index c7f1e45816..a9626d887b 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -45,6 +45,7 @@ import ( am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager" "go.signoz.io/signoz/pkg/query-service/interfaces" "go.signoz.io/signoz/pkg/query-service/model" + "go.signoz.io/signoz/pkg/query-service/telemetry" "go.signoz.io/signoz/pkg/query-service/utils" "go.uber.org/zap" ) @@ -3088,6 +3089,20 @@ func (r *ClickHouseReader) GetSamplesInfoInLastHeartBeatInterval(ctx context.Con return totalSamples, nil } + +func (r *ClickHouseReader) GetDistributedInfoInLastHeartBeatInterval(ctx context.Context) (map[string]interface{}, error) { + + clusterInfo := []model.ClusterInfo{} + + queryStr := `SELECT shard_num, shard_weight, replica_num, errors_count, slowdowns_count, estimated_recovery_time FROM system.clusters where cluster='cluster';` + r.db.Select(ctx, &clusterInfo, queryStr) + if len(clusterInfo) == 1 { + return clusterInfo[0].GetMapFromStruct(), nil + } + + return nil, nil +} + func (r *ClickHouseReader) GetLogsInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) { var totalLogLines uint64 @@ -3233,11 +3248,16 @@ func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilter } isPaginatePrev := logs.CheckIfPrevousPaginateAndModifyOrder(params) - filterSql, err := logs.GenerateSQLWhere(fields, params) + filterSql, lenFilters, err := logs.GenerateSQLWhere(fields, params) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} } + data := map[string]interface{}{ + "lenFilters": lenFilters, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LOGS_FILTERS, data) + query := fmt.Sprintf("%s from %s.%s", constants.LogsSQLSelect, r.logsDB, r.logsTable) if filterSql != "" { @@ -3267,10 +3287,15 @@ func (r *ClickHouseReader) TailLogs(ctx context.Context, client *model.LogsTailC return } - filterSql, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ + filterSql, lenFilters, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ Query: client.Filter.Query, }) + data := map[string]interface{}{ + "lenFilters": lenFilters, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LOGS_FILTERS, data) + if err != nil { client.Error <- err return @@ -3347,13 +3372,18 @@ func (r *ClickHouseReader) AggregateLogs(ctx context.Context, params *model.Logs return nil, apiErr } - filterSql, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ + filterSql, lenFilters, err := logs.GenerateSQLWhere(fields, &model.LogsFilterParams{ Query: params.Query, }) if err != nil { return nil, &model.ApiError{Err: err, Typ: model.ErrorBadData} } + data := map[string]interface{}{ + "lenFilters": lenFilters, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_LOGS_FILTERS, data) + query := "" if params.GroupBy != "" { query = fmt.Sprintf("SELECT toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(timestamp/1000000000), INTERVAL %d minute))*1000000000) as ts_start_interval, toString(%s) as groupBy, "+ diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 69c0f92b8d..927b825792 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1333,6 +1333,9 @@ func (aH *APIHandler) getServices(w http.ResponseWriter, r *http.Request) { } telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_NUMBER_OF_SERVICES, data) + if (data["number"] != 0) || (data["number"] != telemetry.DEFAULT_NUMBER_OF_SERVICES) { + telemetry.GetInstance().AddActiveTracesUser() + } aH.WriteJSON(w, r, result) } diff --git a/pkg/query-service/app/logs/parser.go b/pkg/query-service/app/logs/parser.go index 5e4a195817..18760ba972 100644 --- a/pkg/query-service/app/logs/parser.go +++ b/pkg/query-service/app/logs/parser.go @@ -279,20 +279,23 @@ func CheckIfPrevousPaginateAndModifyOrder(params *model.LogsFilterParams) (isPag return } -func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, error) { +func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilterParams) (string, int, error) { var tokens []string var err error var sqlWhere string + var lenTokens = 0 if params.Query != "" { tokens, err = parseLogQuery(params.Query) + if err != nil { - return sqlWhere, err + return sqlWhere, -1, err } + lenTokens = len(tokens) } tokens, err = replaceInterestingFields(allFields, tokens) if err != nil { - return sqlWhere, err + return sqlWhere, -1, err } filterTokens := []string{} @@ -342,5 +345,5 @@ func GenerateSQLWhere(allFields *model.GetFieldsResponse, params *model.LogsFilt sqlWhere = strings.Join(tokens, "") - return sqlWhere, nil + return sqlWhere, lenTokens, nil } diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index b79b7d011f..3f00483d10 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -1,8 +1,11 @@ package app import ( + "bytes" "context" + "encoding/json" "fmt" + "io/ioutil" "net" "net/http" _ "net/http/pprof" // http profiler @@ -235,15 +238,77 @@ func (lrw *loggingResponseWriter) Flush() { lrw.ResponseWriter.(http.Flusher).Flush() } +func extractDashboardMetaData(path string, r *http.Request) (map[string]interface{}, bool) { + pathToExtractBodyFrom := "/api/v2/metrics/query_range" + var requestBody map[string]interface{} + data := map[string]interface{}{} + + if path == pathToExtractBodyFrom && (r.Method == "POST") { + bodyBytes, _ := ioutil.ReadAll(r.Body) + r.Body.Close() // must close + r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + + json.Unmarshal(bodyBytes, &requestBody) + + } else { + return nil, false + } + + compositeMetricQuery, compositeMetricQueryExists := requestBody["compositeMetricQuery"] + compositeMetricQueryMap := compositeMetricQuery.(map[string]interface{}) + if compositeMetricQueryExists { + queryType, queryTypeExists := compositeMetricQueryMap["queryType"] + if queryTypeExists { + data["queryType"] = queryType + } + panelType, panelTypeExists := compositeMetricQueryMap["panelType"] + if panelTypeExists { + data["panelType"] = panelType + } + } + + datasource, datasourceExists := requestBody["dataSource"] + if datasourceExists { + data["datasource"] = datasource + } + + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_DASHBOARDS_METADATA, data, false) + + return data, true +} + +func getActiveMetricsOrLogs(path string, r *http.Request) { + if path == "/api/v1/dashboards/{uuid}" { + telemetry.GetInstance().AddActiveMetricsUser() + } + if path == "/api/v1/logs" { + hasFilters := len(r.URL.Query().Get("q")) + if hasFilters > 0 { + telemetry.GetInstance().AddActiveLogsUser() + } + + } + +} + func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { route := mux.CurrentRoute(r) path, _ := route.GetPathTemplate() + dashboardMetadata, metadataExists := extractDashboardMetaData(path, r) + getActiveMetricsOrLogs(path, r) + lrw := NewLoggingResponseWriter(w) next.ServeHTTP(lrw, r) data := map[string]interface{}{"path": path, "statusCode": lrw.statusCode} + if metadataExists { + for key, value := range dashboardMetadata { + data[key] = value + } + } + if telemetry.GetInstance().IsSampled() { if _, ok := telemetry.IgnoredPaths()[path]; !ok { telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index 3200b10c2f..bcaf889ab6 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -63,6 +63,7 @@ type Reader interface { GetSamplesInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetLogsInfoInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetTagsInfoInLastHeartBeatInterval(ctx context.Context) (*model.TagsInfo, error) + GetDistributedInfoInLastHeartBeatInterval(ctx context.Context) (map[string]interface{}, error) // Logs GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index d80a56c642..5441f894b2 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -569,3 +569,19 @@ type TagTelemetryData struct { Env string `json:"env" ch:"env"` Language string `json:"language" ch:"language"` } + +type ClusterInfo struct { + ShardNum uint32 `json:"shard_num" ch:"shard_num"` + ShardWeight uint32 `json:"shard_weight" ch:"shard_weight"` + ReplicaNum uint32 `json:"replica_num" ch:"replica_num"` + ErrorsCount uint32 `json:"errors_count" ch:"errors_count"` + SlowdownsCount uint32 `json:"slowdowns_count" ch:"slowdowns_count"` + EstimatedRecoveryTime uint32 `json:"estimated_recovery_time" ch:"estimated_recovery_time"` +} + +func (ci *ClusterInfo) GetMapFromStruct() map[string]interface{} { + var clusterInfoMap map[string]interface{} + data, _ := json.Marshal(*ci) + json.Unmarshal(data, &clusterInfoMap) + return clusterInfoMap +} diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 43f1652093..4e00933151 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -32,19 +32,24 @@ const ( TELEMETRY_LICENSE_ACT_FAILED = "License Activation Failed" TELEMETRY_EVENT_ENVIRONMENT = "Environment" TELEMETRY_EVENT_LANGUAGE = "Language" + TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" + TELEMETRY_EVENT_DISTRIBUTED = "Distributed" + TELEMETRY_EVENT_DASHBOARDS_METADATA = "Dashboards Metadata" + TELEMETRY_EVENT_ACTIVE_USER = "Active User" ) const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" const ph_api_key = "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w" const IP_NOT_FOUND_PLACEHOLDER = "NA" +const DEFAULT_NUMBER_OF_SERVICES = 6 const HEART_BEAT_DURATION = 6 * time.Hour // const HEART_BEAT_DURATION = 10 * time.Second const RATE_LIMIT_CHECK_DURATION = 1 * time.Minute -const RATE_LIMIT_VALUE = 10 +const RATE_LIMIT_VALUE = 2 // const RATE_LIMIT_CHECK_DURATION = 20 * time.Second // const RATE_LIMIT_VALUE = 5 @@ -64,6 +69,16 @@ func (a *Telemetry) IsSampled() bool { } +func (telemetry *Telemetry) AddActiveTracesUser() { + telemetry.activeUser["traces"] = 1 +} +func (telemetry *Telemetry) AddActiveMetricsUser() { + telemetry.activeUser["metrics"] = 1 +} +func (telemetry *Telemetry) AddActiveLogsUser() { + telemetry.activeUser["logs"] = 1 +} + type Telemetry struct { operator analytics.Client phOperator ph.Client @@ -76,6 +91,7 @@ type Telemetry struct { minRandInt int maxRandInt int rateLimits map[string]int8 + activeUser map[string]int8 } func createTelemetry() { @@ -111,6 +127,13 @@ func createTelemetry() { for { select { case <-ticker.C: + + if (telemetry.activeUser["traces"] != 0) || (telemetry.activeUser["metrics"] != 0) || (telemetry.activeUser["logs"] != 0) { + telemetry.activeUser["any"] = 1 + } + telemetry.SendEvent(TELEMETRY_EVENT_ACTIVE_USER, map[string]interface{}{"traces": telemetry.activeUser["traces"], "metrics": telemetry.activeUser["metrics"], "logs": telemetry.activeUser["logs"], "any": telemetry.activeUser["any"]}) + telemetry.activeUser = map[string]int8{"traces": 0, "metrics": 0, "logs": 0, "any": 0} + tagsInfo, _ := telemetry.reader.GetTagsInfoInLastHeartBeatInterval(context.Background()) if len(tagsInfo.Env) != 0 { @@ -138,6 +161,10 @@ func createTelemetry() { data[key] = value } telemetry.SendEvent(TELEMETRY_EVENT_HEART_BEAT, data) + + getDistributedInfoInLastHeartBeatInterval, _ := telemetry.reader.GetDistributedInfoInLastHeartBeatInterval(context.Background()) + telemetry.SendEvent(TELEMETRY_EVENT_DISTRIBUTED, getDistributedInfoInLastHeartBeatInterval) + } } }() @@ -207,7 +234,12 @@ func (a *Telemetry) checkEvents(event string) bool { return sendEvent } -func (a *Telemetry) SendEvent(event string, data map[string]interface{}) { +func (a *Telemetry) SendEvent(event string, data map[string]interface{}, opts ...bool) { + + rateLimitFlag := true + if len(opts) > 0 { + rateLimitFlag = opts[0] + } if !a.isTelemetryEnabled() { return @@ -218,10 +250,12 @@ func (a *Telemetry) SendEvent(event string, data map[string]interface{}) { return } - if a.rateLimits[event] < RATE_LIMIT_VALUE { - a.rateLimits[event] += 1 - } else { - return + if rateLimitFlag { + if a.rateLimits[event] < RATE_LIMIT_VALUE { + a.rateLimits[event] += 1 + } else { + return + } } // zap.S().Info(data) From 7f42b39684ee6124f588a2a64663f2810dd75f8f Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Wed, 28 Dec 2022 02:33:21 +0530 Subject: [PATCH 29/36] fix: changed or to and --- pkg/query-service/app/http_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 927b825792..6bb3a21999 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -1333,7 +1333,7 @@ func (aH *APIHandler) getServices(w http.ResponseWriter, r *http.Request) { } telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_NUMBER_OF_SERVICES, data) - if (data["number"] != 0) || (data["number"] != telemetry.DEFAULT_NUMBER_OF_SERVICES) { + if (data["number"] != 0) && (data["number"] != telemetry.DEFAULT_NUMBER_OF_SERVICES) { telemetry.GetInstance().AddActiveTracesUser() } From 1916fc87b093e462262b196a222a1fed9bd654f5 Mon Sep 17 00:00:00 2001 From: Amol Umbark Date: Wed, 28 Dec 2022 14:30:37 +0530 Subject: [PATCH 30/36] fix: added clear filters button (#1920) * fix: added clear filters button * fix: removed console log Co-authored-by: mindhash Co-authored-by: Pranay Prateek Co-authored-by: Ankit Nayan --- .../SearchFields/ActionBar.tsx | 36 ++++++++++++++ .../LogsSearchFilter/SearchFields/index.tsx | 47 ++++++++++--------- 2 files changed, 61 insertions(+), 22 deletions(-) create mode 100644 frontend/src/container/LogsSearchFilter/SearchFields/ActionBar.tsx diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/ActionBar.tsx b/frontend/src/container/LogsSearchFilter/SearchFields/ActionBar.tsx new file mode 100644 index 0000000000..a8d5c777c9 --- /dev/null +++ b/frontend/src/container/LogsSearchFilter/SearchFields/ActionBar.tsx @@ -0,0 +1,36 @@ +import { Button, Row } from 'antd'; +import React from 'react'; + +import { QueryFields } from './utils'; + +interface SearchFieldsActionBarProps { + fieldsQuery: QueryFields[][]; + applyUpdate: () => void; + clearFilters: () => void; +} + +export function SearchFieldsActionBar({ + fieldsQuery, + applyUpdate, + clearFilters, +}: SearchFieldsActionBarProps): JSX.Element | null { + if (fieldsQuery.length === 0) { + return null; + } + + return ( + + + + + ); +} +export default SearchFieldsActionBar; diff --git a/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx b/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx index bca82bdaaf..a7228c6c45 100644 --- a/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx +++ b/frontend/src/container/LogsSearchFilter/SearchFields/index.tsx @@ -1,10 +1,11 @@ -import { Button, notification, Row } from 'antd'; +import { notification } from 'antd'; import { flatten } from 'lodash-es'; import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; import { ILogsReducer } from 'types/reducer/logs'; +import { SearchFieldsActionBar } from './ActionBar'; import QueryBuilder from './QueryBuilder/QueryBuilder'; import Suggestions from './Suggestions'; import { @@ -68,24 +69,26 @@ function SearchFields({ [fieldsQuery, setFieldsQuery], ); - const applyUpdate = useCallback( - (e): void => { - e.preventDefault(); - const flatParsedQuery = flatten(fieldsQuery); + const applyUpdate = useCallback((): void => { + const flatParsedQuery = flatten(fieldsQuery); - if (!fieldsQueryIsvalid(flatParsedQuery)) { - notification.error({ - message: 'Please enter a valid criteria for each of the selected fields', - }); - return; - } + if (!fieldsQueryIsvalid(flatParsedQuery)) { + notification.error({ + message: 'Please enter a valid criteria for each of the selected fields', + }); + return; + } - keyPrefixRef.current = hashCode(JSON.stringify(flatParsedQuery)); - updateParsedQuery(flatParsedQuery); - onDropDownToggleHandler(false)(); - }, - [onDropDownToggleHandler, fieldsQuery, updateParsedQuery], - ); + keyPrefixRef.current = hashCode(JSON.stringify(flatParsedQuery)); + updateParsedQuery(flatParsedQuery); + onDropDownToggleHandler(false)(); + }, [onDropDownToggleHandler, fieldsQuery, updateParsedQuery]); + + const clearFilters = useCallback((): void => { + keyPrefixRef.current = hashCode(JSON.stringify([])); + updateParsedQuery([]); + onDropDownToggleHandler(false)(); + }, [onDropDownToggleHandler, updateParsedQuery]); return ( <> @@ -96,11 +99,11 @@ function SearchFields({ fieldsQuery={fieldsQuery} setFieldsQuery={setFieldsQuery} /> - - - + ); From 2e58f6db7aa6283ad9652994cfa65aa73989f1ec Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Wed, 28 Dec 2022 14:31:57 +0530 Subject: [PATCH 31/36] fix: error handling for index removal from selected field (#1935) Co-authored-by: Ankit Nayan --- pkg/query-service/app/clickhouseReader/reader.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index a9626d887b..9eb90ac6df 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3233,7 +3233,8 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda // remove index query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s DROP INDEX IF EXISTS %s_idx", r.logsDB, r.logsLocalTable, cluster, field.Name) err := r.db.Exec(ctx, query) - if err != nil { + // we are ignoring errors with code 341 as it is an error with updating old part https://github.com/SigNoz/engineering-pod/issues/919#issuecomment-1366344346 + if err != nil && !strings.HasPrefix(err.Error(), "code: 341") { return &model.ApiError{Err: err, Typ: model.ErrorInternal} } } From 88a97fc4b823b6978f1707b07c0da5571cf12a6a Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Wed, 28 Dec 2022 14:54:15 +0530 Subject: [PATCH 32/36] add exception page filters support (#1919) * feat: backend changes for supporting exception filters * feat: frontend changes for exception page filter support * chore: extractSingleFilterValue is updated * fix: handle frontend edge case Co-authored-by: Ankit Nayan Co-authored-by: Palash Gupta --- frontend/src/container/AllError/constant.ts | 9 ++ frontend/src/container/AllError/index.tsx | 110 ++++++++++++++---- frontend/src/container/AllError/utils.ts | 101 +++++++++++++++- frontend/src/types/api/errors/getAll.ts | 2 + .../app/clickhouseReader/reader.go | 38 +++++- pkg/query-service/app/parser.go | 16 ++- pkg/query-service/model/queryParams.go | 20 ++-- 7 files changed, 258 insertions(+), 38 deletions(-) create mode 100644 frontend/src/container/AllError/constant.ts diff --git a/frontend/src/container/AllError/constant.ts b/frontend/src/container/AllError/constant.ts new file mode 100644 index 0000000000..268f3ff89c --- /dev/null +++ b/frontend/src/container/AllError/constant.ts @@ -0,0 +1,9 @@ +const DEFAULT_FILTER_VALUE = ''; +const EXCEPTION_TYPE_FILTER_NAME = 'exceptionType'; +const SERVICE_NAME_FILTER_NAME = 'serviceName'; + +export { + DEFAULT_FILTER_VALUE, + EXCEPTION_TYPE_FILTER_NAME, + SERVICE_NAME_FILTER_NAME, +}; diff --git a/frontend/src/container/AllError/index.tsx b/frontend/src/container/AllError/index.tsx index 6db945b9bc..3b3939a4a7 100644 --- a/frontend/src/container/AllError/index.tsx +++ b/frontend/src/container/AllError/index.tsx @@ -17,6 +17,7 @@ import getAll from 'api/errors/getAll'; import getErrorCounts from 'api/errors/getErrorCounts'; import ROUTES from 'constants/routes'; import dayjs from 'dayjs'; +import useUrlQuery from 'hooks/useUrlQuery'; import createQueryParams from 'lib/createQueryParams'; import history from 'lib/history'; import React, { useCallback, useEffect, useMemo } from 'react'; @@ -30,7 +31,11 @@ import { Exception, PayloadProps } from 'types/api/errors/getAll'; import { GlobalReducer } from 'types/reducer/globalTime'; import { + extractFilterValues, + getDefaultFilterValue, getDefaultOrder, + getFilterString, + getFilterValues, getNanoSeconds, getOffSet, getOrder, @@ -43,15 +48,27 @@ function AllErrors(): JSX.Element { const { maxTime, minTime, loading } = useSelector( (state) => state.globalTime, ); - const { search, pathname } = useLocation(); - const params = useMemo(() => new URLSearchParams(search), [search]); - + const { pathname } = useLocation(); + const params = useUrlQuery(); const { t } = useTranslation(['common']); - - const updatedOrder = getOrder(params.get(urlKey.order)); - const getUpdatedOffset = getOffSet(params.get(urlKey.offset)); - const getUpdatedParams = getOrderParams(params.get(urlKey.orderParam)); - const getUpdatedPageSize = getUpdatePageSize(params.get(urlKey.pageSize)); + const { + updatedOrder, + getUpdatedOffset, + getUpdatedParams, + getUpdatedPageSize, + getUpdatedExceptionType, + getUpdatedServiceName, + } = useMemo( + () => ({ + updatedOrder: getOrder(params.get(urlKey.order)), + getUpdatedOffset: getOffSet(params.get(urlKey.offset)), + getUpdatedParams: getOrderParams(params.get(urlKey.orderParam)), + getUpdatedPageSize: getUpdatePageSize(params.get(urlKey.pageSize)), + getUpdatedExceptionType: getFilterString(params.get(urlKey.exceptionType)), + getUpdatedServiceName: getFilterString(params.get(urlKey.serviceName)), + }), + [params], + ); const updatedPath = useMemo( () => @@ -60,6 +77,8 @@ function AllErrors(): JSX.Element { offset: getUpdatedOffset, orderParam: getUpdatedParams, pageSize: getUpdatedPageSize, + exceptionType: getUpdatedExceptionType, + serviceName: getUpdatedServiceName, })}`, [ pathname, @@ -67,6 +86,8 @@ function AllErrors(): JSX.Element { getUpdatedOffset, getUpdatedParams, getUpdatedPageSize, + getUpdatedExceptionType, + getUpdatedServiceName, ], ); @@ -81,6 +102,8 @@ function AllErrors(): JSX.Element { limit: getUpdatedPageSize, offset: getUpdatedOffset, orderParam: getUpdatedParams, + exceptionType: getUpdatedExceptionType, + serviceName: getUpdatedServiceName, }), enabled: !loading, }, @@ -108,14 +131,43 @@ function AllErrors(): JSX.Element { const filterIcon = useCallback(() => , []); - const handleSearch = ( - confirm: (param?: FilterConfirmProps) => void, - ): VoidFunction => (): void => { - confirm(); - }; + const handleSearch = useCallback( + ( + confirm: (param?: FilterConfirmProps) => void, + filterValue: string, + filterKey: string, + ): VoidFunction => (): void => { + const { exceptionFilterValue, serviceFilterValue } = getFilterValues( + getUpdatedServiceName, + getUpdatedExceptionType, + filterKey, + filterValue, + ); + history.replace( + `${pathname}?${createQueryParams({ + order: updatedOrder, + offset: getUpdatedOffset, + orderParam: getUpdatedParams, + pageSize: getUpdatedPageSize, + exceptionType: exceptionFilterValue, + serviceName: serviceFilterValue, + })}`, + ); + confirm(); + }, + [ + getUpdatedExceptionType, + getUpdatedOffset, + getUpdatedPageSize, + getUpdatedParams, + getUpdatedServiceName, + pathname, + updatedOrder, + ], + ); const filterDropdownWrapper = useCallback( - ({ setSelectedKeys, selectedKeys, confirm, placeholder }) => { + ({ setSelectedKeys, selectedKeys, confirm, placeholder, filterKey }) => { return ( @@ -126,11 +178,16 @@ function AllErrors(): JSX.Element { setSelectedKeys(e.target.value ? [e.target.value] : []) } allowClear - onPressEnter={handleSearch(confirm)} + defaultValue={getDefaultFilterValue( + filterKey, + getUpdatedServiceName, + getUpdatedExceptionType, + )} + onPressEnter={handleSearch(confirm, selectedKeys[0], filterKey)} />