diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index db5cd61a9b..b6d934f88d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,8 +8,4 @@ /frontend/src/container/NewWidget/RightContainer/types.ts @srikanthccv /deploy/ @prashant-shahi /sample-apps/ @prashant-shahi -**/query-service/ @srikanthccv -Makefile @srikanthccv -go.* @srikanthccv -.git* @srikanthccv .github @prashant-shahi diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 763e2ce4bf..30e5deecc1 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -146,7 +146,7 @@ services: condition: on-failure query-service: - image: signoz/query-service:0.35.0 + image: signoz/query-service:0.35.1 command: [ "-config=/root/config/prometheus.yml", @@ -186,7 +186,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:0.35.0 + image: signoz/frontend:0.35.1 deploy: restart_policy: condition: on-failure @@ -199,7 +199,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-config.yaml", @@ -237,7 +237,7 @@ services: - query-service otel-collector-migrator: - image: signoz/signoz-schema-migrator:0.88.1 + image: signoz/signoz-schema-migrator:0.88.3 deploy: restart_policy: condition: on-failure @@ -250,7 +250,7 @@ services: # - clickhouse-3 otel-collector-metrics: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-metrics-config.yaml", diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml index d4f13e4d26..4e3f5d8bbe 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml @@ -66,7 +66,7 @@ services: - --storage.path=/data otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.3} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -81,7 +81,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` otel-collector: container_name: signoz-otel-collector - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-config.yaml", @@ -118,7 +118,7 @@ services: otel-collector-metrics: container_name: signoz-otel-collector-metrics - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-metrics-config.yaml", diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index ddf80d6e14..2e54477bcf 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -164,7 +164,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: - image: signoz/query-service:${DOCKER_TAG:-0.35.0} + image: signoz/query-service:${DOCKER_TAG:-0.35.1} container_name: signoz-query-service command: [ @@ -203,7 +203,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:${DOCKER_TAG:-0.35.0} + image: signoz/frontend:${DOCKER_TAG:-0.35.1} container_name: signoz-frontend restart: on-failure depends_on: @@ -215,7 +215,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.3} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -229,7 +229,7 @@ services: otel-collector: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.3} container_name: signoz-otel-collector command: [ @@ -269,7 +269,7 @@ services: condition: service_healthy otel-collector-metrics: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.3} container_name: signoz-otel-collector-metrics command: [ diff --git a/ee/query-service/app/api/pat.go b/ee/query-service/app/api/pat.go index 619c875c8f..b0fcf073a4 100644 --- a/ee/query-service/app/api/pat.go +++ b/ee/query-service/app/api/pat.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/mux" "go.signoz.io/signoz/ee/query-service/model" "go.signoz.io/signoz/pkg/query-service/auth" + basemodel "go.signoz.io/signoz/pkg/query-service/model" "go.uber.org/zap" ) @@ -47,8 +48,18 @@ func (ah *APIHandler) createPAT(w http.ResponseWriter, r *http.Request) { req.CreatedAt = time.Now().Unix() req.Token = generatePATToken() + // default expiry is 30 days + if req.ExpiresAt == 0 { + req.ExpiresAt = time.Now().AddDate(0, 0, 30).Unix() + } + // max expiry is 1 year + if req.ExpiresAt > time.Now().AddDate(1, 0, 0).Unix() { + req.ExpiresAt = time.Now().AddDate(1, 0, 0).Unix() + } + zap.S().Debugf("Got PAT request: %+v", req) - if apierr := ah.AppDao().CreatePAT(ctx, &req); apierr != nil { + var apierr basemodel.BaseApiError + if req, apierr = ah.AppDao().CreatePAT(ctx, req); apierr != nil { RespondError(w, apierr, nil) return } diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index bed3855f17..699894e691 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -480,7 +480,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { } } - if _, ok := telemetry.IgnoredPaths()[path]; !ok { + if _, ok := telemetry.EnabledPaths()[path]; ok { userEmail, err := auth.GetEmailFromJwt(r.Context()) if err == nil { telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data, userEmail) diff --git a/ee/query-service/dao/interface.go b/ee/query-service/dao/interface.go index 1a8f3b2460..479ca56edc 100644 --- a/ee/query-service/dao/interface.go +++ b/ee/query-service/dao/interface.go @@ -33,7 +33,7 @@ type ModelDao interface { DeleteDomain(ctx context.Context, id uuid.UUID) basemodel.BaseApiError GetDomainByEmail(ctx context.Context, email string) (*model.OrgDomain, basemodel.BaseApiError) - CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseApiError + CreatePAT(ctx context.Context, p model.PAT) (model.PAT, basemodel.BaseApiError) GetPAT(ctx context.Context, pat string) (*model.PAT, basemodel.BaseApiError) GetPATByID(ctx context.Context, id string) (*model.PAT, basemodel.BaseApiError) GetUserByPAT(ctx context.Context, token string) (*basemodel.UserPayload, basemodel.BaseApiError) diff --git a/ee/query-service/dao/sqlite/pat.go b/ee/query-service/dao/sqlite/pat.go index cc4de546c5..5bd1b78a62 100644 --- a/ee/query-service/dao/sqlite/pat.go +++ b/ee/query-service/dao/sqlite/pat.go @@ -3,14 +3,15 @@ package sqlite import ( "context" "fmt" + "strconv" "go.signoz.io/signoz/ee/query-service/model" basemodel "go.signoz.io/signoz/pkg/query-service/model" "go.uber.org/zap" ) -func (m *modelDao) CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseApiError { - _, err := m.DB().ExecContext(ctx, +func (m *modelDao) CreatePAT(ctx context.Context, p model.PAT) (model.PAT, basemodel.BaseApiError) { + result, err := m.DB().ExecContext(ctx, "INSERT INTO personal_access_tokens (user_id, token, name, created_at, expires_at) VALUES ($1, $2, $3, $4, $5)", p.UserID, p.Token, @@ -19,9 +20,15 @@ func (m *modelDao) CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseAp p.ExpiresAt) if err != nil { zap.S().Errorf("Failed to insert PAT in db, err: %v", zap.Error(err)) - return model.InternalError(fmt.Errorf("PAT insertion failed")) + return model.PAT{}, model.InternalError(fmt.Errorf("PAT insertion failed")) } - return nil + id, err := result.LastInsertId() + if err != nil { + zap.S().Errorf("Failed to get last inserted id, err: %v", zap.Error(err)) + return model.PAT{}, model.InternalError(fmt.Errorf("PAT insertion failed")) + } + p.Id = strconv.Itoa(int(id)) + return p, nil } func (m *modelDao) ListPATs(ctx context.Context, userID string) ([]model.PAT, basemodel.BaseApiError) { @@ -90,7 +97,7 @@ func (m *modelDao) GetUserByPAT(ctx context.Context, token string) (*basemodel.U u.org_id, u.group_id FROM users u, personal_access_tokens p - WHERE u.id = p.user_id and p.token=?;` + WHERE u.id = p.user_id and p.token=? and p.expires_at >= strftime('%s', 'now');` if err := m.DB().Select(&users, query, token); err != nil { return nil, model.InternalError(fmt.Errorf("failed to fetch user from PAT, err: %v", err)) diff --git a/ee/query-service/model/pat.go b/ee/query-service/model/pat.go index c22282060b..f320d0be7c 100644 --- a/ee/query-service/model/pat.go +++ b/ee/query-service/model/pat.go @@ -6,5 +6,5 @@ type PAT struct { Token string `json:"token" db:"token"` Name string `json:"name" db:"name"` CreatedAt int64 `json:"createdAt" db:"created_at"` - ExpiresAt int64 `json:"expiresAt" db:"expires_at"` // unused as of now + ExpiresAt int64 `json:"expiresAt" db:"expires_at"` } diff --git a/ee/query-service/model/plans.go b/ee/query-service/model/plans.go index caa2a4df68..5b6f230550 100644 --- a/ee/query-service/model/plans.go +++ b/ee/query-service/model/plans.go @@ -52,14 +52,14 @@ var BasicPlan = basemodel.FeatureSet{ Name: basemodel.QueryBuilderPanels, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 20, Route: "", }, basemodel.Feature{ Name: basemodel.QueryBuilderAlerts, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 10, Route: "", }, basemodel.Feature{ diff --git a/frontend/.husky/commit-msg b/frontend/.husky/commit-msg index be422b654c..cb50d87a2d 100755 --- a/frontend/.husky/commit-msg +++ b/frontend/.husky/commit-msg @@ -2,3 +2,19 @@ . "$(dirname "$0")/_/husky.sh" cd frontend && yarn run commitlint --edit $1 + +branch="$(git rev-parse --abbrev-ref HEAD)" + +color_red="$(tput setaf 1)" +bold="$(tput bold)" +reset="$(tput sgr0)" + +if [ "$branch" = "main" ]; then + echo "${color_red}${bold}You can't commit directly to the main branch${reset}" + exit 1 +fi + +if [ "$branch" = "develop" ]; then + echo "${color_red}${bold}You can't commit directly to the develop branch${reset}" + exit 1 +fi \ No newline at end of file diff --git a/frontend/jest.config.ts b/frontend/jest.config.ts index b7cac3cc02..c9a67b3d26 100644 --- a/frontend/jest.config.ts +++ b/frontend/jest.config.ts @@ -22,7 +22,7 @@ const config: Config.InitialOptions = { '^.+\\.(js|jsx)$': 'babel-jest', }, transformIgnorePatterns: [ - 'node_modules/(?!(lodash-es|react-dnd|core-dnd|@react-dnd|dnd-core|react-dnd-html5-backend)/)', + 'node_modules/(?!(lodash-es|react-dnd|core-dnd|@react-dnd|dnd-core|react-dnd-html5-backend|axios)/)', ], setupFilesAfterEnv: ['jest.setup.ts'], testPathIgnorePatterns: ['/node_modules/', '/public/'], diff --git a/frontend/package.json b/frontend/package.json index 25036d24fe..64ac911fc4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -38,7 +38,7 @@ "ansi-to-html": "0.7.2", "antd": "5.11.0", "antd-table-saveas-excel": "2.2.1", - "axios": "^0.21.0", + "axios": "1.6.2", "babel-eslint": "^10.1.0", "babel-jest": "^29.6.4", "babel-loader": "9.1.3", @@ -87,7 +87,7 @@ "react-helmet-async": "1.3.0", "react-i18next": "^11.16.1", "react-markdown": "8.0.7", - "react-query": "^3.34.19", + "react-query": "3.39.3", "react-redux": "^7.2.2", "react-router-dom": "^5.2.0", "react-syntax-highlighter": "15.5.0", diff --git a/frontend/src/api/ErrorResponseHandler.ts b/frontend/src/api/ErrorResponseHandler.ts index 2c42f0951b..3f28ff418d 100644 --- a/frontend/src/api/ErrorResponseHandler.ts +++ b/frontend/src/api/ErrorResponseHandler.ts @@ -1,4 +1,4 @@ -import { AxiosError } from 'axios'; +import { AxiosError, AxiosResponse } from 'axios'; import { ErrorResponse } from 'types/api'; import { ErrorStatusCode } from 'types/common'; @@ -10,7 +10,7 @@ export function ErrorResponseHandler(error: AxiosError): ErrorResponse { const statusCode = response.status as ErrorStatusCode; if (statusCode >= 400 && statusCode < 500) { - const { data } = response; + const { data } = response as AxiosResponse; if (statusCode === 404) { return { diff --git a/frontend/src/api/dashboard/get.ts b/frontend/src/api/dashboard/get.ts index 9b10f6467d..01e04c6c0f 100644 --- a/frontend/src/api/dashboard/get.ts +++ b/frontend/src/api/dashboard/get.ts @@ -3,9 +3,9 @@ import { ApiResponse } from 'types/api'; import { Props } from 'types/api/dashboard/get'; import { Dashboard } from 'types/api/dashboard/getAll'; -const get = (props: Props): Promise => +const getDashboard = (props: Props): Promise => axios .get>(`/dashboards/${props.uuid}`) .then((res) => res.data.data); -export default get; +export default getDashboard; diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index 9739f24e49..bde915f201 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -4,7 +4,7 @@ import getLocalStorageApi from 'api/browser/localstorage/get'; import loginApi from 'api/user/login'; import afterLogin from 'AppRoutes/utils'; -import axios, { AxiosRequestConfig, AxiosResponse } from 'axios'; +import axios, { AxiosResponse, InternalAxiosRequestConfig } from 'axios'; import { ENVIRONMENT } from 'constants/env'; import { LOCALSTORAGE } from 'constants/localStorage'; import store from 'store'; @@ -17,14 +17,16 @@ const interceptorsResponse = ( ): Promise> => Promise.resolve(value); const interceptorsRequestResponse = ( - value: AxiosRequestConfig, -): AxiosRequestConfig => { + value: InternalAxiosRequestConfig, +): InternalAxiosRequestConfig => { const token = store.getState().app.user?.accessJwt || getLocalStorageApi(LOCALSTORAGE.AUTH_TOKEN) || ''; - value.headers.Authorization = token ? `Bearer ${token}` : ''; + if (value && value.headers) { + value.headers.Authorization = token ? `Bearer ${token}` : ''; + } return value; }; @@ -92,8 +94,8 @@ const instance = axios.create({ baseURL: `${ENVIRONMENT.baseURL}${apiV1}`, }); -instance.interceptors.response.use(interceptorsResponse, interceptorRejected); instance.interceptors.request.use(interceptorsRequestResponse); +instance.interceptors.response.use(interceptorsResponse, interceptorRejected); export const AxiosAlertManagerInstance = axios.create({ baseURL: `${ENVIRONMENT.baseURL}${apiAlertManager}`, diff --git a/frontend/src/api/metrics/getQueryRange.ts b/frontend/src/api/metrics/getQueryRange.ts index bee657d904..984d381e10 100644 --- a/frontend/src/api/metrics/getQueryRange.ts +++ b/frontend/src/api/metrics/getQueryRange.ts @@ -9,9 +9,10 @@ import { export const getMetricsQueryRange = async ( props: QueryRangePayload, + signal: AbortSignal, ): Promise | ErrorResponse> => { try { - const response = await axios.post('/query_range', props); + const response = await axios.post('/query_range', props, { signal }); return { statusCode: 200, diff --git a/frontend/src/container/FormAlertRules/ChartPreview/index.tsx b/frontend/src/container/FormAlertRules/ChartPreview/index.tsx index 400a6f85be..3551c3a87a 100644 --- a/frontend/src/container/FormAlertRules/ChartPreview/index.tsx +++ b/frontend/src/container/FormAlertRules/ChartPreview/index.tsx @@ -32,6 +32,7 @@ export interface ChartPreviewProps { alertDef?: AlertDef; userQueryKey?: string; allowSelectedIntervalForStepGen?: boolean; + yAxisUnit: string; } function ChartPreview({ @@ -44,6 +45,7 @@ function ChartPreview({ userQueryKey, allowSelectedIntervalForStepGen = false, alertDef, + yAxisUnit, }: ChartPreviewProps): JSX.Element | null { const { t } = useTranslation('alerts'); const threshold = alertDef?.condition.target || 0; @@ -112,7 +114,7 @@ function ChartPreview({ () => getUPlotChartOptions({ id: 'alert_legend_widget', - yAxisUnit: query?.unit, + yAxisUnit, apiResponse: queryResponse?.data?.payload, dimensions: containerDimensions, isDarkMode, @@ -129,14 +131,14 @@ function ChartPreview({ optionName, threshold, alertDef?.condition.targetUnit, - query?.unit, + yAxisUnit, )})`, thresholdUnit: alertDef?.condition.targetUnit, }, ], }), [ - query?.unit, + yAxisUnit, queryResponse?.data?.payload, containerDimensions, isDarkMode, @@ -168,7 +170,7 @@ function ChartPreview({ name={name || 'Chart Preview'} panelData={queryResponse.data?.payload.data.newResult.data.result || []} query={query || initialQueriesMap.metrics} - yAxisUnit={query?.unit} + yAxisUnit={yAxisUnit} /> )} diff --git a/frontend/src/container/FormAlertRules/ChartPreview/utils.ts b/frontend/src/container/FormAlertRules/ChartPreview/utils.ts index f17a6e3865..1fc2a2e247 100644 --- a/frontend/src/container/FormAlertRules/ChartPreview/utils.ts +++ b/frontend/src/container/FormAlertRules/ChartPreview/utils.ts @@ -61,8 +61,20 @@ export const getThresholdLabel = ( unit === MiscellaneousFormats.PercentUnit || yAxisUnit === MiscellaneousFormats.PercentUnit ) { + if (unit === MiscellaneousFormats.Percent) { + return `${value}%`; + } return `${value * 100}%`; } + if ( + unit === MiscellaneousFormats.Percent || + yAxisUnit === MiscellaneousFormats.Percent + ) { + if (unit === MiscellaneousFormats.PercentUnit) { + return `${value * 100}%`; + } + return `${value}%`; + } return `${value} ${optionName}`; }; diff --git a/frontend/src/container/FormAlertRules/index.tsx b/frontend/src/container/FormAlertRules/index.tsx index 0076449faf..eafbcd363a 100644 --- a/frontend/src/container/FormAlertRules/index.tsx +++ b/frontend/src/container/FormAlertRules/index.tsx @@ -82,6 +82,7 @@ function FormAlertRules({ // alertDef holds the form values to be posted const [alertDef, setAlertDef] = useState(initialValue); + const [yAxisUnit, setYAxisUnit] = useState(currentQuery.unit || ''); // initQuery contains initial query when component was mounted const initQuery = useMemo(() => initialValue.condition.compositeQuery, [ @@ -400,6 +401,7 @@ function FormAlertRules({ query={stagedQuery} selectedInterval={globalSelectedInterval} alertDef={alertDef} + yAxisUnit={yAxisUnit || ''} /> ); @@ -415,6 +417,7 @@ function FormAlertRules({ query={stagedQuery} alertDef={alertDef} selectedInterval={globalSelectedInterval} + yAxisUnit={yAxisUnit || ''} /> ); @@ -427,7 +430,8 @@ function FormAlertRules({ currentQuery.queryType === EQueryType.QUERY_BUILDER && alertType !== AlertTypes.METRICS_BASED_ALERT; - const onUnitChangeHandler = (): void => { + const onUnitChangeHandler = (value: string): void => { + setYAxisUnit(value); // reset target unit setAlertDef((def) => ({ ...def, @@ -457,7 +461,10 @@ function FormAlertRules({ renderPromAndChQueryChartPreview()} - + {layouts.map((layout) => { const { i: id } = layout; diff --git a/frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts b/frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts new file mode 100644 index 0000000000..27833f5415 --- /dev/null +++ b/frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts @@ -0,0 +1,19 @@ +import { TopOperationList } from '../TopOperationsTable'; + +interface TopOperation { + numCalls: number; + errorCount: number; +} + +export const getTopOperationList = ({ + errorCount, + numCalls, +}: TopOperation): TopOperationList => + ({ + p50: 0, + errorCount, + name: 'test', + numCalls, + p95: 0, + p99: 0, + } as TopOperationList); diff --git a/frontend/src/container/MetricsApplication/utils.test.ts b/frontend/src/container/MetricsApplication/utils.test.ts new file mode 100644 index 0000000000..7ad338d850 --- /dev/null +++ b/frontend/src/container/MetricsApplication/utils.test.ts @@ -0,0 +1,70 @@ +import { getTopOperationList } from './__mocks__/getTopOperation'; +import { TopOperationList } from './TopOperationsTable'; +import { + convertedTracesToDownloadData, + getErrorRate, + getNearestHighestBucketValue, +} from './utils'; + +describe('Error Rate', () => { + test('should return correct error rate', () => { + const list: TopOperationList = getTopOperationList({ + errorCount: 10, + numCalls: 100, + }); + + expect(getErrorRate(list)).toBe(10); + }); + + test('should handle no errors gracefully', () => { + const list = getTopOperationList({ errorCount: 0, numCalls: 100 }); + expect(getErrorRate(list)).toBe(0); + }); + + test('should handle zero calls', () => { + const list = getTopOperationList({ errorCount: 0, numCalls: 0 }); + expect(getErrorRate(list)).toBe(0); + }); +}); + +describe('getNearestHighestBucketValue', () => { + test('should return nearest higher bucket value', () => { + expect(getNearestHighestBucketValue(50, [10, 20, 30, 40, 60, 70])).toBe('60'); + }); + + test('should return +Inf for value higher than any bucket', () => { + expect(getNearestHighestBucketValue(80, [10, 20, 30, 40, 60, 70])).toBe( + '+Inf', + ); + }); + + test('should return the first bucket for value lower than all buckets', () => { + expect(getNearestHighestBucketValue(5, [10, 20, 30, 40, 60, 70])).toBe('10'); + }); +}); + +describe('convertedTracesToDownloadData', () => { + test('should convert trace data correctly', () => { + const data = [ + { + name: 'op1', + p50: 50000000, + p95: 95000000, + p99: 99000000, + numCalls: 100, + errorCount: 10, + }, + ]; + + expect(convertedTracesToDownloadData(data)).toEqual([ + { + Name: 'op1', + 'P50 (in ms)': '50.00', + 'P95 (in ms)': '95.00', + 'P99 (in ms)': '99.00', + 'Number of calls': '100', + 'Error Rate (%)': '10.00', + }, + ]); + }); +}); diff --git a/frontend/src/container/MetricsApplication/utils.ts b/frontend/src/container/MetricsApplication/utils.ts index a6aec561d7..0fdf307ace 100644 --- a/frontend/src/container/MetricsApplication/utils.ts +++ b/frontend/src/container/MetricsApplication/utils.ts @@ -5,8 +5,12 @@ import history from 'lib/history'; import { TopOperationList } from './TopOperationsTable'; import { NavigateToTraceProps } from './types'; -export const getErrorRate = (list: TopOperationList): number => - (list.errorCount / list.numCalls) * 100; +export const getErrorRate = (list: TopOperationList): number => { + if (list.errorCount === 0 && list.numCalls === 0) { + return 0; + } + return (list.errorCount / list.numCalls) * 100; +}; export const navigateToTrace = ({ servicename, diff --git a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx index 91faea4fcc..76f0464bd2 100644 --- a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx +++ b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx @@ -140,11 +140,12 @@ function VariableItem({ enabled: false, queryFn: () => dashboardVariablesQuery({ - query: variableData.queryValue || '', + query: variableQueryValue || '', variables: variablePropsToPayloadVariables(existingVariables), }), refetchOnWindowFocus: false, onSuccess: (response) => { + setErrorPreview(null); handleQueryResult(response); }, onError: (error: { diff --git a/frontend/src/container/NewWidget/RightContainer/alertFomatCategories.ts b/frontend/src/container/NewWidget/RightContainer/alertFomatCategories.ts index 09acc57d0a..aa29303926 100644 --- a/frontend/src/container/NewWidget/RightContainer/alertFomatCategories.ts +++ b/frontend/src/container/NewWidget/RightContainer/alertFomatCategories.ts @@ -78,6 +78,7 @@ export const alertsCategory = [ name: CategoryNames.Miscellaneous, formats: [ { name: 'Percent (0.0-1.0)', id: MiscellaneousFormats.PercentUnit }, + { name: 'Percent (0 - 100)', id: MiscellaneousFormats.Percent }, ], }, { diff --git a/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts b/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts index 7e739fdf52..cd8748788d 100644 --- a/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts +++ b/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts @@ -53,7 +53,7 @@ const usePipelinePreview = ({ isLoading: isFetching, outputLogs, isError, - errorMsg: error?.response?.data?.error || '', + errorMsg: error?.message || '', }; }; diff --git a/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/BuilderUnits.tsx b/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/BuilderUnits.tsx index 47aa20aaf1..c128abc484 100644 --- a/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/BuilderUnits.tsx +++ b/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/BuilderUnits.tsx @@ -10,10 +10,11 @@ import { filterOption } from './utils'; function BuilderUnitsFilter({ onChange, + yAxisUnit, }: IBuilderUnitsFilterProps): JSX.Element { const { currentQuery, handleOnUnitsChange } = useQueryBuilder(); - const selectedValue = currentQuery?.unit; + const selectedValue = yAxisUnit || currentQuery?.unit; const allOptions = categoryToSupport.map((category) => ({ label: category, diff --git a/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts b/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts index 693dab7be6..4474c7d20f 100644 --- a/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts +++ b/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts @@ -1,3 +1,4 @@ export interface IBuilderUnitsFilterProps { onChange?: (value: string) => void; + yAxisUnit?: string; } diff --git a/frontend/src/container/TriggeredAlerts/Filter.tsx b/frontend/src/container/TriggeredAlerts/Filter.tsx index 4a54916685..e7ebdb0d76 100644 --- a/frontend/src/container/TriggeredAlerts/Filter.tsx +++ b/frontend/src/container/TriggeredAlerts/Filter.tsx @@ -1,11 +1,35 @@ /* eslint-disable react/no-unstable-nested-components */ import type { SelectProps } from 'antd'; -import { Tag } from 'antd'; -import { Dispatch, SetStateAction, useCallback, useMemo } from 'react'; +import { Tag, Tooltip } from 'antd'; +import { BaseOptionType } from 'antd/es/select'; +import { Dispatch, SetStateAction, useCallback, useMemo, useRef } from 'react'; import { Alerts } from 'types/api/alerts/getTriggered'; import { Container, Select } from './styles'; +function TextOverflowTooltip({ + option, +}: { + option: BaseOptionType; +}): JSX.Element { + const contentRef = useRef(null); + const isOverflow = contentRef.current + ? contentRef.current?.offsetWidth < contentRef.current?.scrollWidth + : false; + return ( + +
+ {option.value} +
+ + ); +} + function Filter({ setSelectedFilter, setSelectedGroup, @@ -51,6 +75,7 @@ function Filter({ const options = uniqueLabels.map((e) => ({ value: e, + title: '', })); const getTags: SelectProps['tagRender'] = (props): JSX.Element => { @@ -88,6 +113,9 @@ function Filter({ placeholder="Group by any tag" tagRender={(props): JSX.Element => getTags(props)} options={options} + optionRender={(option): JSX.Element => ( + + )} /> ); diff --git a/frontend/src/hooks/queryBuilder/useGetQueryRange.ts b/frontend/src/hooks/queryBuilder/useGetQueryRange.ts index 9aa76405c2..c54a07461d 100644 --- a/frontend/src/hooks/queryBuilder/useGetQueryRange.ts +++ b/frontend/src/hooks/queryBuilder/useGetQueryRange.ts @@ -15,14 +15,19 @@ type UseGetQueryRange = ( export const useGetQueryRange: UseGetQueryRange = (requestData, options) => { const queryKey = useMemo(() => { - if (options?.queryKey) { + if (options?.queryKey && Array.isArray(options.queryKey)) { return [...options.queryKey]; } + + if (options?.queryKey && typeof options.queryKey === 'string') { + return options.queryKey; + } + return [REACT_QUERY_KEY.GET_QUERY_RANGE, requestData]; }, [options?.queryKey, requestData]); return useQuery, Error>({ - queryFn: async () => GetMetricQueryRange(requestData), + queryFn: async ({ signal }) => GetMetricQueryRange(requestData, signal), ...options, queryKey, }); diff --git a/frontend/src/lib/dashboard/getQueryResults.ts b/frontend/src/lib/dashboard/getQueryResults.ts index 47f21bdd25..89ba08f891 100644 --- a/frontend/src/lib/dashboard/getQueryResults.ts +++ b/frontend/src/lib/dashboard/getQueryResults.ts @@ -17,10 +17,11 @@ import { prepareQueryRangePayload } from './prepareQueryRangePayload'; export async function GetMetricQueryRange( props: GetQueryResultsProps, + signal?: AbortSignal, ): Promise> { const { legendMap, queryPayload } = prepareQueryRangePayload(props); - const response = await getMetricsQueryRange(queryPayload); + const response = await getMetricsQueryRange(queryPayload, signal); if (response.statusCode >= 400) { throw new Error( diff --git a/frontend/src/lib/getConvertedValue.ts b/frontend/src/lib/getConvertedValue.ts index 229b2d2677..0e8c267236 100644 --- a/frontend/src/lib/getConvertedValue.ts +++ b/frontend/src/lib/getConvertedValue.ts @@ -232,6 +232,11 @@ const unitsMapping = [ { label: 'Percent (0.0-1.0)', value: 'percentunit', + factor: 100, + }, + { + label: 'Percent (0 - 100)', + value: 'percent', factor: 1, }, ], diff --git a/frontend/src/lib/uPlotLib/getUplotChartOptions.ts b/frontend/src/lib/uPlotLib/getUplotChartOptions.ts index 3a5ec4b2c6..354deb0b5a 100644 --- a/frontend/src/lib/uPlotLib/getUplotChartOptions.ts +++ b/frontend/src/lib/uPlotLib/getUplotChartOptions.ts @@ -55,6 +55,7 @@ export const getUPlotChartOptions = ({ legend: { show: true, live: false, + isolate: true, }, focus: { alpha: 0.3, @@ -158,16 +159,24 @@ export const getUPlotChartOptions = ({ (self): void => { const legend = self.root.querySelector('.u-legend'); if (legend) { - const seriesEls = legend.querySelectorAll('.u-label'); + const seriesEls = legend.querySelectorAll('.u-series'); const seriesArray = Array.from(seriesEls); seriesArray.forEach((seriesEl, index) => { seriesEl.addEventListener('click', () => { if (graphsVisibilityStates) { setGraphsVisibilityStates?.((prev) => { const newGraphVisibilityStates = [...prev]; - newGraphVisibilityStates[index + 1] = !newGraphVisibilityStates[ - index + 1 - ]; + if ( + newGraphVisibilityStates[index + 1] && + newGraphVisibilityStates.every((value, i) => + i === index + 1 ? value : !value, + ) + ) { + newGraphVisibilityStates.fill(true); + } else { + newGraphVisibilityStates.fill(false); + newGraphVisibilityStates[index + 1] = true; + } return newGraphVisibilityStates; }); } diff --git a/frontend/src/pages/NewDashboard/DashboardPage.tsx b/frontend/src/pages/NewDashboard/DashboardPage.tsx index d880413628..7da194aad0 100644 --- a/frontend/src/pages/NewDashboard/DashboardPage.tsx +++ b/frontend/src/pages/NewDashboard/DashboardPage.tsx @@ -12,7 +12,9 @@ function DashboardPage(): JSX.Element { const { isFetching, isError, isLoading } = dashboardResponse; const errorMessage = isError - ? (dashboardResponse?.error as AxiosError)?.response?.data.errorType + ? // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + (dashboardResponse?.error as AxiosError)?.response?.data?.errorType : 'Something went wrong'; if (isError && !isFetching && errorMessage === ErrorType.NotFound) { diff --git a/frontend/src/pages/Support/Support.tsx b/frontend/src/pages/Support/Support.tsx index 61d6c11c38..96276507f9 100644 --- a/frontend/src/pages/Support/Support.tsx +++ b/frontend/src/pages/Support/Support.tsx @@ -1,6 +1,7 @@ import './Support.styles.scss'; import { Button, Card, Typography } from 'antd'; +import useAnalytics from 'hooks/analytics/useAnalytics'; import { Book, Cable, @@ -82,6 +83,8 @@ const supportChannels = [ ]; export default function Support(): JSX.Element { + const { trackEvent } = useAnalytics(); + const handleChannelWithRedirects = (url: string): void => { window.open(url, '_blank'); }; @@ -111,6 +114,8 @@ export default function Support(): JSX.Element { }; const handleChannelClick = (channel: Channel): void => { + trackEvent(`Support : ${channel.name}`); + switch (channel.key) { case channelsMap.documentation: case channelsMap.github: diff --git a/frontend/src/providers/Dashboard/Dashboard.tsx b/frontend/src/providers/Dashboard/Dashboard.tsx index 4365b6c35b..3df954e1b7 100644 --- a/frontend/src/providers/Dashboard/Dashboard.tsx +++ b/frontend/src/providers/Dashboard/Dashboard.tsx @@ -1,5 +1,5 @@ import Modal from 'antd/es/modal'; -import get from 'api/dashboard/get'; +import getDashboard from 'api/dashboard/get'; import lockDashboardApi from 'api/dashboard/lockDashboard'; import unlockDashboardApi from 'api/dashboard/unlockDashboard'; import { REACT_QUERY_KEY } from 'constants/reactQueryKeys'; @@ -107,7 +107,7 @@ export function DashboardProvider({ { enabled: (!!isDashboardPage || !!isDashboardWidgetPage) && isLoggedIn, queryFn: () => - get({ + getDashboard({ uuid: dashboardId, }), refetchOnWindowFocus: false, diff --git a/frontend/yarn.lock b/frontend/yarn.lock index bc6da58703..404b83ba28 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -4639,7 +4639,16 @@ axe-core@^4.6.2: resolved "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz" integrity sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ== -axios@^0.21.0, axios@^0.21.1: +axios@1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.2.tgz#de67d42c755b571d3e698df1b6504cde9b0ee9f2" + integrity sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: version "0.21.4" resolved "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz" integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== @@ -7710,6 +7719,11 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0: resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz" integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== +follow-redirects@^1.15.0: + version "1.15.3" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a" + integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + fontfaceobserver@2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/fontfaceobserver/-/fontfaceobserver-2.3.0.tgz" @@ -7759,6 +7773,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + format@^0.2.0: version "0.2.2" resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" @@ -12294,6 +12317,11 @@ proxy-addr@~2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz" @@ -12962,9 +12990,9 @@ react-markdown@8.0.7, react-markdown@~8.0.0: unist-util-visit "^4.0.0" vfile "^5.0.0" -react-query@^3.34.19: +react-query@3.39.3: version "3.39.3" - resolved "https://registry.npmjs.org/react-query/-/react-query-3.39.3.tgz" + resolved "https://registry.yarnpkg.com/react-query/-/react-query-3.39.3.tgz#4cea7127c6c26bdea2de5fb63e51044330b03f35" integrity sha512-nLfLz7GiohKTJDuT4us4X3h/8unOh+00MLb2yJoGTPjxKs2bc1iDhkNx2bd5MKklXnOD3NrVZ+J2UXujA5In4g== dependencies: "@babel/runtime" "^7.5.5" diff --git a/go.mod b/go.mod index 9009f9da9f..a10d4f904d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/ClickHouse/clickhouse-go/v2 v2.15.0 github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb - github.com/SigNoz/signoz-otel-collector v0.88.1 + github.com/SigNoz/signoz-otel-collector v0.88.3 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974 github.com/antonmedv/expr v1.15.3 diff --git a/go.sum b/go.sum index 9b7192550d..02d869f765 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb h1:bneLSKPf9YUSFm github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb/go.mod h1:JznGDNg9x1cujDKa22RaQOimOvvEfy3nxzDGd8XDgmA= github.com/SigNoz/prometheus v1.9.78 h1:bB3yuDrRzi/Mv00kWayR9DZbyjTuGfendSqISyDcXiY= github.com/SigNoz/prometheus v1.9.78/go.mod h1:MffmFu2qFILQrOHehx3D0XjYtaZMVfI+Ppeiv98x4Ww= -github.com/SigNoz/signoz-otel-collector v0.88.1 h1:Xeu6Kn8VA0g6it60PMIAclayYSIogBq0rnkodlpxllI= -github.com/SigNoz/signoz-otel-collector v0.88.1/go.mod h1:KyEc6JSFS6f8Nw3UdSm4aGDGucEpQYZUdYwjvY8uMVc= +github.com/SigNoz/signoz-otel-collector v0.88.3 h1:30sEJZmCQjfjo8CZGxqXKZkWE7Zij9TeS1uUqNFEZRU= +github.com/SigNoz/signoz-otel-collector v0.88.3/go.mod h1:KyEc6JSFS6f8Nw3UdSm4aGDGucEpQYZUdYwjvY8uMVc= github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc= github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo= github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY= diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 3369dbe23f..ffbc2f53a8 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -50,8 +50,8 @@ func (r *Repo) GetConfigHistory( disabled, deploy_status, deploy_result, - last_hash, - last_config + coalesce(last_hash, '') as last_hash, + coalesce(last_config, '{}') as last_config FROM agent_config_versions AS v WHERE element_type = $1 ORDER BY created_at desc, version desc @@ -89,8 +89,8 @@ func (r *Repo) GetConfigVersion( disabled, deploy_status, deploy_result, - last_hash, - last_config + coalesce(last_hash, '') as last_hash, + coalesce(last_config, '{}') as last_config FROM agent_config_versions v WHERE element_type = $1 AND version = $2`, typ, v) diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index a919185d0d..0e77383f7e 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -172,21 +172,6 @@ func (m *Manager) ReportConfigDeploymentStatus( } } -// Ready indicates if Manager can accept new config update requests -func (mgr *Manager) Ready() bool { - if atomic.LoadUint32(&mgr.lock) != 0 { - return false - } - return opamp.Ready() -} - -// Static methods for working with default manager instance in this module. - -// Ready indicates if Manager can accept new config update requests -func Ready() bool { - return m.Ready() -} - func GetLatestVersion( ctx context.Context, elementType ElementTypeDef, ) (*ConfigVersion, *model.ApiError) { @@ -210,11 +195,6 @@ func StartNewVersion( ctx context.Context, userId string, eleType ElementTypeDef, elementIds []string, ) (*ConfigVersion, *model.ApiError) { - if !m.Ready() { - // agent is already being updated, ask caller to wait and re-try after sometime - return nil, model.UnavailableError(fmt.Errorf("agent updater is busy")) - } - // create a new version cfg := NewConfigversion(eleType) diff --git a/pkg/query-service/agentConf/version.go b/pkg/query-service/agentConf/version.go index 13be5f7cd2..d2bc4547b3 100644 --- a/pkg/query-service/agentConf/version.go +++ b/pkg/query-service/agentConf/version.go @@ -53,6 +53,8 @@ func NewConfigversion(typeDef ElementTypeDef) *ConfigVersion { IsValid: false, Disabled: false, DeployStatus: PendingDeploy, + LastHash: "", + LastConf: "{}", // todo: get user id from context? // CreatedBy } diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 8b42c9265a..c8f150cd85 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -43,6 +43,7 @@ import ( promModel "github.com/prometheus/common/model" "go.uber.org/zap" + "go.signoz.io/signoz/pkg/query-service/app/dashboards" "go.signoz.io/signoz/pkg/query-service/app/logs" "go.signoz.io/signoz/pkg/query-service/app/services" "go.signoz.io/signoz/pkg/query-service/auth" @@ -51,6 +52,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/interfaces" "go.signoz.io/signoz/pkg/query-service/model" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/telemetry" "go.signoz.io/signoz/pkg/query-service/utils" ) @@ -3421,6 +3423,100 @@ func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Contex return &tagsInfo, nil } +// GetDashboardsInfo returns analytics data for dashboards +func (r *ClickHouseReader) GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) { + dashboardsInfo := model.DashboardsInfo{} + // fetch dashboards from dashboard db + query := "SELECT data FROM dashboards" + var dashboardsData []dashboards.Dashboard + err := r.localDB.Select(&dashboardsData, query) + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return &dashboardsInfo, err + } + for _, dashboard := range dashboardsData { + dashboardsInfo = countPanelsInDashboard(dashboard.Data) + } + dashboardsInfo.TotalDashboards = len(dashboardsData) + + return &dashboardsInfo, nil +} + +func countPanelsInDashboard(data map[string]interface{}) model.DashboardsInfo { + var logsPanelCount, tracesPanelCount, metricsPanelCount int + // totalPanels := 0 + if data != nil && data["widgets"] != nil { + widgets, ok := data["widgets"].(interface{}) + if ok { + data, ok := widgets.([]interface{}) + if ok { + for _, widget := range data { + sData, ok := widget.(map[string]interface{}) + if ok && sData["query"] != nil { + // totalPanels++ + query, ok := sData["query"].(interface{}).(map[string]interface{}) + if ok && query["queryType"] == "builder" && query["builder"] != nil { + builderData, ok := query["builder"].(interface{}).(map[string]interface{}) + if ok && builderData["queryData"] != nil { + builderQueryData, ok := builderData["queryData"].([]interface{}) + if ok { + for _, queryData := range builderQueryData { + data, ok := queryData.(map[string]interface{}) + if ok { + if data["dataSource"] == "traces" { + tracesPanelCount++ + } else if data["dataSource"] == "metrics" { + metricsPanelCount++ + } else if data["dataSource"] == "logs" { + logsPanelCount++ + } + } + } + } + } + } + } + } + } + } + } + return model.DashboardsInfo{ + LogsBasedPanels: logsPanelCount, + TracesBasedPanels: tracesPanelCount, + MetricBasedPanels: metricsPanelCount, + } +} + +func (r *ClickHouseReader) GetAlertsInfo(ctx context.Context) (*model.AlertsInfo, error) { + alertsInfo := model.AlertsInfo{} + // fetch alerts from rules db + query := "SELECT data FROM rules" + var alertsData []string + err := r.localDB.Select(&alertsData, query) + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return &alertsInfo, err + } + for _, alert := range alertsData { + var rule rules.GettableRule + err = json.Unmarshal([]byte(alert), &rule) + if err != nil { + zap.S().Errorf("msg:", "invalid rule data", "\t err:", err) + continue + } + if rule.AlertType == "LOGS_BASED_ALERT" { + alertsInfo.LogsBasedAlerts = alertsInfo.LogsBasedAlerts + 1 + } else if rule.AlertType == "METRIC_BASED_ALERT" { + alertsInfo.MetricBasedAlerts = alertsInfo.MetricBasedAlerts + 1 + } else if rule.AlertType == "TRACES_BASED_ALERT" { + alertsInfo.TracesBasedAlerts = alertsInfo.TracesBasedAlerts + 1 + } + alertsInfo.TotalAlerts = alertsInfo.TotalAlerts + 1 + } + + return &alertsInfo, nil +} + func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) { // response will contain top level fields from the otel log model response := model.GetFieldsResponse{ diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 973e23328c..1d01267860 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -314,6 +314,7 @@ func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMid subRouter.HandleFunc("/autocomplete/attribute_values", am.ViewAccess( withCacheControl(AutoCompleteCacheControlAge, aH.autoCompleteAttributeValues))).Methods(http.MethodGet) subRouter.HandleFunc("/query_range", am.ViewAccess(aH.QueryRangeV3)).Methods(http.MethodPost) + subRouter.HandleFunc("/query_range/format", am.ViewAccess(aH.QueryRangeV3Format)).Methods(http.MethodPost) // live logs subRouter.HandleFunc("/logs/livetail", am.ViewAccess(aH.liveTailLogs)).Methods(http.MethodGet) @@ -3001,6 +3002,18 @@ func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.Qu return data, nil } +func (aH *APIHandler) QueryRangeV3Format(w http.ResponseWriter, r *http.Request) { + queryRangeParams, apiErrorObj := ParseQueryRangeParams(r) + + if apiErrorObj != nil { + zap.S().Errorf(apiErrorObj.Err.Error()) + RespondError(w, apiErrorObj, nil) + return + } + + aH.Respond(w, queryRangeParams) +} + func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3, w http.ResponseWriter, r *http.Request) { var result []*v3.Result diff --git a/pkg/query-service/app/logparsingpipeline/controller.go b/pkg/query-service/app/logparsingpipeline/controller.go index 7880ac27b7..066123c416 100644 --- a/pkg/query-service/app/logparsingpipeline/controller.go +++ b/pkg/query-service/app/logparsingpipeline/controller.go @@ -73,12 +73,6 @@ func (ic *LogParsingPipelineController) ApplyPipelines( } - if !agentConf.Ready() { - return nil, model.UnavailableError(fmt.Errorf( - "agent updater unavailable at the moment. Please try in sometime", - )) - } - // prepare config elements elements := make([]string, len(pipelines)) for i, p := range pipelines { diff --git a/pkg/query-service/app/metrics/v4/cumulative/helper.go b/pkg/query-service/app/metrics/v4/cumulative/helper.go new file mode 100644 index 0000000000..5914ee495d --- /dev/null +++ b/pkg/query-service/app/metrics/v4/cumulative/helper.go @@ -0,0 +1,57 @@ +package cumulative + +import ( + "fmt" + "strings" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +// groupingSets returns a string of comma separated tags for group by clause +// `ts` is always added to the group by clause +func groupingSets(tags ...string) string { + withTs := append(tags, "ts") + return fmt.Sprintf(`GROUPING SETS ( (%s), (%s) )`, strings.Join(withTs, ", "), strings.Join(tags, ", ")) +} + +// groupingSetsByAttributeKeyTags returns a string of comma separated tags for group by clause +func groupingSetsByAttributeKeyTags(tags ...v3.AttributeKey) string { + groupTags := []string{} + for _, tag := range tags { + groupTags = append(groupTags, tag.Key) + } + return groupingSets(groupTags...) +} + +// groupBy returns a string of comma separated tags for group by clause +func groupByAttributeKeyTags(tags ...v3.AttributeKey) string { + groupTags := []string{} + for _, tag := range tags { + groupTags = append(groupTags, tag.Key) + } + groupTags = append(groupTags, "ts") + return strings.Join(groupTags, ", ") +} + +// orderBy returns a string of comma separated tags for order by clause +// if the order is not specified, it defaults to ASC +func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string { + var orderBy []string + for _, tag := range tags { + found := false + for _, item := range items { + if item.ColumnName == tag.Key { + found = true + orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) + break + } + } + if !found { + orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag.Key)) + } + } + + orderBy = append(orderBy, "ts ASC") + + return strings.Join(orderBy, ", ") +} diff --git a/pkg/query-service/app/metrics/v4/cumulative/timeseries.go b/pkg/query-service/app/metrics/v4/cumulative/timeseries.go new file mode 100644 index 0000000000..78d22be4aa --- /dev/null +++ b/pkg/query-service/app/metrics/v4/cumulative/timeseries.go @@ -0,0 +1,220 @@ +package cumulative + +import ( + "fmt" + + v4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +// See https://clickhouse.com/docs/en/sql-reference/window-functions for more details on `lagInFrame` function +// +// Calculating the rate of change of a metric is a common use case. +// Requests and errors are two examples of metrics that are often expressed as a rate of change. +// The rate of change is the difference between the current value and the previous value divided by +// the time difference between the current and previous values (i.e. the time interval). +// +// The value of a cumulative counter always increases. However, the rate of change can be negative +// if the value decreases between two samples. This can happen if the counter is reset when the +// application restarts or if the counter is reset manually. In this case, the rate of change is +// not meaningful and should be ignored. +// +// The condition `(per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0` +// checks if the rate of change is negative. If it is negative, the value is replaced with `nan`. +// +// The condition `ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400` checks +// if the time difference between the current and previous values is greater than or equal to 1 day. +// The first sample of a metric is always `nan` because there is no previous value to compare it to. +// When the first sample is encountered, the previous value for the time is set to default i.e `1970-01-01`. +// Since any difference between the first sample timestamp and the previous value timestamp will be +// greater than or equal to 1 day, the rate of change for the first sample will be `nan`. +// +// If neither of the above conditions are true, the rate of change is calculated as +// `(per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window)` +// where `rate_window` is a window function that partitions the data by fingerprint and orders it by timestamp. +// We want to calculate the rate of change for each time series, so we partition the data by fingerprint. +// +// The `increase` function is similar to the `rate` function, except that it does not divide by the time interval. +const ( + rateWithoutNegative = `If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window)))` + increaseWithoutNegative = `If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window)))` +) + +// prepareTimeAggregationSubQueryTimeSeries prepares the sub-query to be used for temporal aggregation +// of time series data + +// The following example illustrates how the sub-query is used to calculate the sume of values for each +// time series in a 15 seconds interval: + +// ``` +// timestamp 01.00 01.05 01.10 01.15 01.20 01.25 01.30 01.35 01.40 +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | | +// | v1 | v2 | v3 | v4 | v5 | v6 | v7 | v8 | v9 | +// | | | | | | | | | | +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | +// | | | | | | | | | +// | | | +// +------+ +------+ +------+ +// | v1+ | | v4+ | | v7+ | +// | v2+ | | v5+ | | v8+ | +// | v3 | | v6 | | v9 | +// +------+ +------+ +------+ +// 01.00 01.15 01.30 +// ``` + +// Calculating the rate/increase involves an additional step. We first calculate the maximum value for each time series +// in a 15 seconds interval. Then, we calculate the difference between the current maximum value and the previous +// maximum value + +// The following example illustrates how the sub-query is used to calculate the rate of change for each time series +// in a 15 seconds interval: + +// ``` +// timestamp 01.00 01.05 01.10 01.15 01.20 01.25 01.30 01.35 01.40 +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | | +// | v1 | v2 | v3 | v4 | v5 | v6 | v7 | v8 | v9 | +// | | | | | | | | | | +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | +// | | | | | | | | | +// | | | +// +------+ +------+ +------+ +// max(| v1, | max(| v4, | max(| v7, | +// | v2, | | v5, | | v8, | +// | v3 |) | v6 |) | v9 |) +// +------+ +------+ +------+ +// 01.00 01.15 01.30 + +// +-------+ +--------+ +// | V6-V2 | | V9-V6 | +// | | | | +// | | | | +// +------+ +--------+ +// 01.00 01.15 +// ``` + +// The rate of change is calculated as (Vy - Vx) / (Ty - Tx) where Vx and Vy are the values at time Tx and Ty respectively. +// In an ideal scenario, the last value of each interval could be used to calculate the rate of change. Instead, we use +// the maximum value of each interval to calculate the rate of change. This is because any process restart can cause the +// value to be reset to 0. This will produce an inaccurate result. The max is the best approximation we can get. +// We don't expect the process to restart very often, so this should be a good approximation. + +func prepareTimeAggregationSubQueryTimeSeries(start, end, step int64, mq *v3.BuilderQuery) (string, error) { + var subQuery string + + timeSeriesSubQuery, err := v4.PrepareTimeseriesFilterQuery(mq) + if err != nil { + return "", err + } + + samplesTableFilter := fmt.Sprintf("metric_name = %s AND timestamp_ms >= %d AND timestamp_ms <= %d", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key), start, end) + + // Select the aggregate value for interval + queryTmpl := + "SELECT fingerprint, %s" + + " toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL %d SECOND) as ts," + + " %s as per_series_value" + + " FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + constants.SIGNOZ_SAMPLES_TABLENAME + + " INNER JOIN" + + " (%s) as filtered_time_series" + + " USING fingerprint" + + " WHERE " + samplesTableFilter + + " GROUP BY fingerprint, ts" + + " ORDER BY fingerprint, ts" + + var selectLabelsAny string + for _, tag := range mq.GroupBy { + selectLabelsAny += fmt.Sprintf("any(%s) as %s,", tag.Key, tag.Key) + } + + var selectLabels string + for _, tag := range mq.GroupBy { + selectLabels += tag.Key + "," + } + + switch mq.TimeAggregation { + case v3.TimeAggregationAvg: + op := "avg(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationSum: + op := "sum(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationMin: + op := "min(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationMax: + op := "max(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationCount: + op := "count(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationCountDistinct: + op := "count(distinct(value))" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationAnyLast: + op := "anyLast(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationRate: + op := "max(value)" + innerSubQuery := fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + rateQueryTmpl := + "SELECT %s ts, " + rateWithoutNegative + + " as per_series_value FROM (%s) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)" + subQuery = fmt.Sprintf(rateQueryTmpl, selectLabels, innerSubQuery) + case v3.TimeAggregationIncrease: + op := "max(value)" + innerSubQuery := fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + rateQueryTmpl := + "SELECT %s ts, " + increaseWithoutNegative + + " as per_series_value FROM (%s) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)" + subQuery = fmt.Sprintf(rateQueryTmpl, selectLabels, innerSubQuery) + } + return subQuery, nil +} + +// prepareMetricQueryCumulativeTimeSeries prepares the query to be used for fetching metrics +func prepareMetricQueryCumulativeTimeSeries(start, end, step int64, mq *v3.BuilderQuery) (string, error) { + var query string + + temporalAggSubQuery, err := prepareTimeAggregationSubQueryTimeSeries(start, end, step, mq) + if err != nil { + return "", err + } + + groupBy := groupingSetsByAttributeKeyTags(mq.GroupBy...) + orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) + selectLabels := groupByAttributeKeyTags(mq.GroupBy...) + + queryTmpl := + "SELECT %s," + + " %s as value" + + " FROM (%s)" + + " WHERE isNaN(per_series_value) = 0" + + " GROUP BY %s" + + " ORDER BY %s" + + switch mq.SpaceAggregation { + case v3.SpaceAggregationAvg: + op := "avg(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationSum: + op := "sum(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationMin: + op := "min(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationMax: + op := "max(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationCount: + op := "count(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + } + + return query, nil +} diff --git a/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go b/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go new file mode 100644 index 0000000000..70f2e1d3ef --- /dev/null +++ b/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go @@ -0,0 +1,229 @@ +package cumulative + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func TestPrepareTimeAggregationSubQuery(t *testing.T) { + // The time aggregation is performed for each unique series - since the fingerprint represents the + // unique hash of label set, we always group by fingerprint regardless of the GroupBy + // This sub result is then aggregated on dimensions using the provided GroupBy clause keys + testCases := []struct { + name string + builderQuery *v3.BuilderQuery + start int64 + end int64 + expectedQueryContains string + }{ + { + name: "test time aggregation = avg, temporality = cumulative", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorNotEqual, + Value: "payment_service", + }, + { + Key: v3.AttributeKey{ + Key: "endpoint", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorIn, + Value: []interface{}{"/paycallback", "/payme", "/paypal"}, + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationAvg, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts", + }, + { + name: "test time aggregation = rate, temporality = cumulative", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorContains, + Value: "payment_service", + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationRate, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + query, err := prepareTimeAggregationSubQueryTimeSeries( + testCase.start, + testCase.end, + testCase.builderQuery.StepInterval, + testCase.builderQuery, + ) + assert.Nil(t, err) + assert.Contains(t, query, testCase.expectedQueryContains) + }) + } +} +func TestPrepareTimeseriesQuery(t *testing.T) { + testCases := []struct { + name string + builderQuery *v3.BuilderQuery + start int64 + end int64 + expectedQueryContains string + }{ + { + name: "test time aggregation = avg, space aggregation = sum, temporality = unspecified", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "system_memory_usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Unspecified, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "state", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorNotEqual, + Value: "idle", + }, + }, + }, + GroupBy: []v3.AttributeKey{}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationSum, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'system_memory_usage' AND temporality = 'Unspecified' AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name = 'system_memory_usage' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY GROUPING SETS ( (ts), () ) ORDER BY ts ASC", + }, + { + name: "test time aggregation = rate, space aggregation = sum, temporality = cumulative", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorContains, + Value: "payment_service", + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationRate, + SpaceAggregation: v3.SpaceAggregationSum, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY GROUPING SETS ( (service_name, ts), (service_name) ) ORDER BY service_name ASC, ts ASC", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + query, err := prepareMetricQueryCumulativeTimeSeries( + testCase.start, + testCase.end, + testCase.builderQuery.StepInterval, + testCase.builderQuery, + ) + assert.Nil(t, err) + assert.Contains(t, query, testCase.expectedQueryContains) + }) + } +} diff --git a/pkg/query-service/app/metrics/v4/query_builder.go b/pkg/query-service/app/metrics/v4/query_builder.go new file mode 100644 index 0000000000..70d35e8e08 --- /dev/null +++ b/pkg/query-service/app/metrics/v4/query_builder.go @@ -0,0 +1,86 @@ +package v4 + +import ( + "fmt" + "strings" + + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +// PrepareTimeseriesFilterQuery builds the sub-query to be used for filtering timeseries based on the search criteria +func PrepareTimeseriesFilterQuery(mq *v3.BuilderQuery) (string, error) { + var conditions []string + var fs *v3.FilterSet = mq.Filters + var groupTags []v3.AttributeKey = mq.GroupBy + + conditions = append(conditions, fmt.Sprintf("metric_name = %s", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key))) + conditions = append(conditions, fmt.Sprintf("temporality = '%s'", mq.Temporality)) + + if fs != nil && len(fs.Items) != 0 { + for _, item := range fs.Items { + toFormat := item.Value + op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) + if op == v3.FilterOperatorContains || op == v3.FilterOperatorNotContains { + toFormat = fmt.Sprintf("%%%s%%", toFormat) + } + fmtVal := utils.ClickHouseFormattedValue(toFormat) + switch op { + case v3.FilterOperatorEqual: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') = %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotEqual: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') != %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorIn: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') IN %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotIn: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') NOT IN %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorLike: + conditions = append(conditions, fmt.Sprintf("like(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotLike: + conditions = append(conditions, fmt.Sprintf("notLike(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorRegex: + conditions = append(conditions, fmt.Sprintf("match(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotRegex: + conditions = append(conditions, fmt.Sprintf("not match(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorGreaterThan: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') > %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorGreaterThanOrEq: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') >= %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorLessThan: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') < %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorLessThanOrEq: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') <= %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorContains: + conditions = append(conditions, fmt.Sprintf("like(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotContains: + conditions = append(conditions, fmt.Sprintf("notLike(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorExists: + conditions = append(conditions, fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", item.Key.Key)) + case v3.FilterOperatorNotExists: + conditions = append(conditions, fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", item.Key.Key)) + default: + return "", fmt.Errorf("unsupported filter operator") + } + } + } + whereClause := strings.Join(conditions, " AND ") + + var selectLabels string + for _, tag := range groupTags { + selectLabels += fmt.Sprintf("JSONExtractString(labels, '%s') as %s, ", tag.Key, tag.Key) + } + + // The table JOIN key always exists + selectLabels += "fingerprint" + + filterSubQuery := fmt.Sprintf( + "SELECT DISTINCT %s FROM %s.%s WHERE %s", + selectLabels, + constants.SIGNOZ_METRIC_DBNAME, + constants.SIGNOZ_TIMESERIES_LOCAL_TABLENAME, + whereClause, + ) + + return filterSubQuery, nil +} diff --git a/pkg/query-service/app/metrics/v4/query_builder_test.go b/pkg/query-service/app/metrics/v4/query_builder_test.go new file mode 100644 index 0000000000..eb071ecb2f --- /dev/null +++ b/pkg/query-service/app/metrics/v4/query_builder_test.go @@ -0,0 +1,150 @@ +package v4 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func TestPrepareTimeseriesFilterQuery(t *testing.T) { + testCases := []struct { + name string + builderQuery *v3.BuilderQuery + expectedQueryContains string + }{ + { + name: "test prepare time series with no filters and no group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Delta, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Delta'", + }, + { + name: "test prepare time series with no filters and group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative'", + }, + { + name: "test prepare time series with no filters and multiple group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + GroupBy: []v3.AttributeKey{ + { + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + { + Key: "endpoint", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'endpoint') as endpoint, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative'", + }, + { + name: "test prepare time series with filters and multiple group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorNotEqual, + Value: "payment_service", + }, + { + Key: v3.AttributeKey{ + Key: "endpoint", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorIn, + Value: []interface{}{"/paycallback", "/payme", "/paypal"}, + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + query, err := PrepareTimeseriesFilterQuery(testCase.builderQuery) + assert.Nil(t, err) + assert.Contains(t, query, testCase.expectedQueryContains) + }) + } +} diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go index a6f9dd66ef..758531f345 100644 --- a/pkg/query-service/app/opamp/model/agent.go +++ b/pkg/query-service/app/opamp/model/agent.go @@ -259,7 +259,7 @@ func (agent *Agent) processStatusUpdate( // send the new remote config to the Agent. if configChanged || (agent.Status.RemoteConfigStatus != nil && - bytes.Compare(agent.Status.RemoteConfigStatus.LastRemoteConfigHash, agent.remoteConfig.ConfigHash) != 0) { + !bytes.Equal(agent.Status.RemoteConfigStatus.LastRemoteConfigHash, agent.remoteConfig.ConfigHash)) { // The new status resulted in a change in the config of the Agent or the Agent // does not have this config (hash is different). Send the new config the Agent. response.RemoteConfig = agent.remoteConfig @@ -352,7 +352,7 @@ func isEqualConfigFile(f1, f2 *protobufs.AgentConfigFile) bool { if f1 == nil || f2 == nil { return false } - return bytes.Compare(f1.Body, f2.Body) == 0 && f1.ContentType == f2.ContentType + return bytes.Equal(f1.Body, f2.Body) && f1.ContentType == f2.ContentType } func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index e4e43f1648..f7fa328b9f 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -417,7 +417,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { } // if telemetry.GetInstance().IsSampled() { - if _, ok := telemetry.IgnoredPaths()[path]; !ok { + if _, ok := telemetry.EnabledPaths()[path]; ok { userEmail, err := auth.GetEmailFromJwt(r.Context()) if err == nil { telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data, userEmail) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index b25888f607..e2b2b49481 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -71,6 +71,8 @@ type Reader interface { GetListResultV3(ctx context.Context, query string) ([]*v3.Row, error) LiveTailLogsV3(ctx context.Context, query string, timestampStart uint64, idStart string, client *v3.LogsLiveTailClient) + GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) + GetAlertsInfo(ctx context.Context) (*model.AlertsInfo, error) GetTotalSpans(ctx context.Context) (uint64, error) GetSpansInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetTimeSeriesInfo(ctx context.Context) (map[string]interface{}, error) diff --git a/pkg/query-service/model/featureSet.go b/pkg/query-service/model/featureSet.go index 92aa7b2426..26cd70b908 100644 --- a/pkg/query-service/model/featureSet.go +++ b/pkg/query-service/model/featureSet.go @@ -55,14 +55,14 @@ var BasicPlan = FeatureSet{ Name: QueryBuilderPanels, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 20, Route: "", }, Feature{ Name: QueryBuilderAlerts, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 10, Route: "", }, Feature{ diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 9bb1d985c0..6d5e65d732 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -615,6 +615,20 @@ type TagsInfo struct { Env string `json:"env"` } +type AlertsInfo struct { + TotalAlerts int `json:"totalAlerts"` + LogsBasedAlerts int `json:"logsBasedAlerts"` + MetricBasedAlerts int `json:"metricBasedAlerts"` + TracesBasedAlerts int `json:"tracesBasedAlerts"` +} + +type DashboardsInfo struct { + TotalDashboards int `json:"totalDashboards"` + LogsBasedPanels int `json:"logsBasedPanels"` + MetricBasedPanels int `json:"metricBasedPanels"` + TracesBasedPanels int `json:"tracesBasedPanels"` +} + type TagTelemetryData struct { ServiceName string `json:"serviceName" ch:"serviceName"` Env string `json:"env" ch:"env"` diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index e04be217cf..453c6475a8 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -447,6 +447,38 @@ const ( Cumulative Temporality = "Cumulative" ) +type TimeAggregation string + +const ( + TimeAggregationUnspecified TimeAggregation = "" + TimeAggregationAnyLast TimeAggregation = "latest" + TimeAggregationSum TimeAggregation = "sum" + TimeAggregationAvg TimeAggregation = "avg" + TimeAggregationMin TimeAggregation = "min" + TimeAggregationMax TimeAggregation = "max" + TimeAggregationCount TimeAggregation = "count" + TimeAggregationCountDistinct TimeAggregation = "count_distinct" + TimeAggregationRate TimeAggregation = "rate" + TimeAggregationIncrease TimeAggregation = "increase" +) + +type SpaceAggregation string + +const ( + SpaceAggregationUnspecified SpaceAggregation = "" + SpaceAggregationSum SpaceAggregation = "sum" + SpaceAggregationAvg SpaceAggregation = "avg" + SpaceAggregationMin SpaceAggregation = "min" + SpaceAggregationMax SpaceAggregation = "max" + SpaceAggregationCount SpaceAggregation = "count" +) + +type Function struct { + Category string `json:"category"` + Name string `json:"name"` + Args []interface{} `json:"args,omitempty"` +} + type BuilderQuery struct { QueryName string `json:"queryName"` StepInterval int64 `json:"stepInterval"` @@ -466,6 +498,9 @@ type BuilderQuery struct { OrderBy []OrderBy `json:"orderBy,omitempty"` ReduceTo ReduceToOperator `json:"reduceTo,omitempty"` SelectColumns []AttributeKey `json:"selectColumns,omitempty"` + TimeAggregation TimeAggregation `json:"timeAggregation,omitempty"` + SpaceAggregation SpaceAggregation `json:"spaceAggregation,omitempty"` + Functions []Function `json:"functions,omitempty"` } func (b *BuilderQuery) Validate() error { diff --git a/pkg/query-service/rules/promRule.go b/pkg/query-service/rules/promRule.go index 6d0cafa930..94ace4137b 100644 --- a/pkg/query-service/rules/promRule.go +++ b/pkg/query-service/rules/promRule.go @@ -3,7 +3,6 @@ package rules import ( "context" "fmt" - "strconv" "sync" "time" @@ -367,7 +366,10 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) ( l[lbl.Name] = lbl.Value } - tmplData := AlertTemplateData(l, valueFormatter.Format(smpl.F, r.Unit()), strconv.FormatFloat(r.targetVal(), 'f', 2, 64)+converter.UnitToName(r.ruleCondition.TargetUnit)) + thresholdFormatter := formatter.FromUnit(r.ruleCondition.TargetUnit) + threshold := thresholdFormatter.Format(r.targetVal(), r.ruleCondition.TargetUnit) + + tmplData := AlertTemplateData(l, valueFormatter.Format(smpl.F, r.Unit()), threshold) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}" diff --git a/pkg/query-service/rules/thresholdRule_test.go b/pkg/query-service/rules/thresholdRule_test.go index 27ad6611f5..031a19b70a 100644 --- a/pkg/query-service/rules/thresholdRule_test.go +++ b/pkg/query-service/rules/thresholdRule_test.go @@ -14,7 +14,7 @@ import ( func TestThresholdRuleCombinations(t *testing.T) { postableRule := PostableRule{ Alert: "Tricky Condition Tests", - AlertType: "METRICS_BASED_ALERT", + AlertType: "METRIC_BASED_ALERT", RuleType: RuleTypeThreshold, EvalWindow: Duration(5 * time.Minute), Frequency: Duration(1 * time.Minute), diff --git a/pkg/query-service/telemetry/ignored.go b/pkg/query-service/telemetry/ignored.go index 29c06fe1ac..c0a739e9ee 100644 --- a/pkg/query-service/telemetry/ignored.go +++ b/pkg/query-service/telemetry/ignored.go @@ -1,16 +1,11 @@ package telemetry -func IgnoredPaths() map[string]struct{} { - ignoredPaths := map[string]struct{}{ - "/api/v1/tags": {}, - "/api/v1/version": {}, - "/api/v1/query_range": {}, - "/api/v2/metrics/query_range": {}, - "/api/v1/health": {}, - "/api/v1/featureFlags": {}, +func EnabledPaths() map[string]struct{} { + enabledPaths := map[string]struct{}{ + "/api/v1/channels": {}, } - return ignoredPaths + return enabledPaths } func ignoreEvents(event string, attributes map[string]interface{}) bool { diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 24a2dbc4e2..4797e7d740 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -38,6 +38,7 @@ const ( TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" TELEMETRY_EVENT_DISTRIBUTED = "Distributed" TELEMETRY_EVENT_QUERY_RANGE_V3 = "Query Range V3 Metadata" + TELEMETRY_EVENT_DASHBOARDS_ALERTS = "Dashboards/Alerts Info" TELEMETRY_EVENT_ACTIVE_USER = "Active User" TELEMETRY_EVENT_ACTIVE_USER_PH = "Active User V2" TELEMETRY_EVENT_USER_INVITATION_SENT = "User Invitation Sent" @@ -53,6 +54,7 @@ var SAAS_EVENTS_LIST = map[string]struct{}{ TELEMETRY_EVENT_ENVIRONMENT: {}, TELEMETRY_EVENT_USER_INVITATION_SENT: {}, TELEMETRY_EVENT_USER_INVITATION_ACCEPTED: {}, + TELEMETRY_EVENT_DASHBOARDS_ALERTS: {}, } const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" @@ -61,9 +63,9 @@ const ph_api_key = "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w" const IP_NOT_FOUND_PLACEHOLDER = "NA" const DEFAULT_NUMBER_OF_SERVICES = 6 -const HEART_BEAT_DURATION = 6 * time.Hour +const HEART_BEAT_DURATION = 12 * time.Hour -const ACTIVE_USER_DURATION = 30 * time.Minute +const ACTIVE_USER_DURATION = 6 * time.Hour // const HEART_BEAT_DURATION = 30 * time.Second // const ACTIVE_USER_DURATION = 30 * time.Second @@ -241,9 +243,30 @@ func createTelemetry() { } telemetry.SendEvent(TELEMETRY_EVENT_HEART_BEAT, data, "") + alertsInfo, err := telemetry.reader.GetAlertsInfo(context.Background()) + if err != nil { + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, map[string]interface{}{"error": err.Error()}, "") + } else { + dashboardsInfo, err := telemetry.reader.GetDashboardsInfo(context.Background()) + if err == nil { + dashboardsAlertsData := map[string]interface{}{ + "totalDashboards": dashboardsInfo.TotalDashboards, + "logsBasedPanels": dashboardsInfo.LogsBasedPanels, + "metricBasedPanels": dashboardsInfo.MetricBasedPanels, + "tracesBasedPanels": dashboardsInfo.TracesBasedPanels, + "totalAlerts": alertsInfo.TotalAlerts, + "logsBasedAlerts": alertsInfo.LogsBasedAlerts, + "metricBasedAlerts": alertsInfo.MetricBasedAlerts, + "tracesBasedAlerts": alertsInfo.TracesBasedAlerts, + } + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, dashboardsAlertsData, "") + } else { + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, map[string]interface{}{"error": err.Error()}, "") + } + } + getDistributedInfoInLastHeartBeatInterval, _ := telemetry.reader.GetDistributedInfoInLastHeartBeatInterval(context.Background()) telemetry.SendEvent(TELEMETRY_EVENT_DISTRIBUTED, getDistributedInfoInLastHeartBeatInterval, "") - } } }() diff --git a/pkg/query-service/tests/integration/logparsingpipeline_test.go b/pkg/query-service/tests/integration/logparsingpipeline_test.go index 0b4d22973c..4c260596e5 100644 --- a/pkg/query-service/tests/integration/logparsingpipeline_test.go +++ b/pkg/query-service/tests/integration/logparsingpipeline_test.go @@ -367,17 +367,70 @@ func TestLogPipelinesValidation(t *testing.T) { } } +func TestCanSavePipelinesWithoutConnectedAgents(t *testing.T) { + require := require.New(t) + testbed := NewTestbedWithoutOpamp(t) + + getPipelinesResp := testbed.GetPipelinesFromQS() + require.Equal(0, len(getPipelinesResp.Pipelines)) + require.Equal(0, len(getPipelinesResp.History)) + + postablePipelines := logparsingpipeline.PostablePipelines{ + Pipelines: []logparsingpipeline.PostablePipeline{ + { + OrderId: 1, + Name: "pipeline1", + Alias: "pipeline1", + Enabled: true, + Filter: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "method", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: "=", + Value: "GET", + }, + }, + }, + Config: []logparsingpipeline.PipelineOperator{ + { + OrderId: 1, + ID: "add", + Type: "add", + Field: "attributes.test", + Value: "val", + Enabled: true, + Name: "test add", + }, + }, + }, + }, + } + + testbed.PostPipelinesToQS(postablePipelines) + getPipelinesResp = testbed.GetPipelinesFromQS() + require.Equal(1, len(getPipelinesResp.Pipelines)) + require.Equal(1, len(getPipelinesResp.History)) + +} + // LogPipelinesTestBed coordinates and mocks components involved in // configuring log pipelines and provides test helpers. type LogPipelinesTestBed struct { t *testing.T + testDBFilePath string testUser *model.User apiHandler *app.APIHandler + agentConfMgr *agentConf.Manager opampServer *opamp.Server opampClientConn *opamp.MockOpAmpConnection } -func NewLogPipelinesTestBed(t *testing.T) *LogPipelinesTestBed { +func NewTestbedWithoutOpamp(t *testing.T) *LogPipelinesTestBed { // Create a tmp file based sqlite db for testing. testDBFile, err := os.CreateTemp("", "test-signoz-db-*") if err != nil { @@ -408,22 +461,61 @@ func NewLogPipelinesTestBed(t *testing.T) *LogPipelinesTestBed { t.Fatalf("could not create a new ApiHandler: %v", err) } - opampServer, clientConn := mockOpampAgent(t, testDBFilePath, controller) - user, apiErr := createTestUser() if apiErr != nil { t.Fatalf("could not create a test user: %v", apiErr) } + // Mock an available opamp agent + testDB, err = opampModel.InitDB(testDBFilePath) + require.Nil(t, err, "failed to init opamp model") + + agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{ + DB: testDB, + DBEngine: "sqlite", + AgentFeatures: []agentConf.AgentFeature{ + apiHandler.LogsParsingPipelineController, + }}) + require.Nil(t, err, "failed to init agentConf") + return &LogPipelinesTestBed{ - t: t, - testUser: user, - apiHandler: apiHandler, - opampServer: opampServer, - opampClientConn: clientConn, + t: t, + testDBFilePath: testDBFilePath, + testUser: user, + apiHandler: apiHandler, + agentConfMgr: agentConfMgr, } } +func NewLogPipelinesTestBed(t *testing.T) *LogPipelinesTestBed { + testbed := NewTestbedWithoutOpamp(t) + + opampServer := opamp.InitializeServer(nil, testbed.agentConfMgr) + err := opampServer.Start(opamp.GetAvailableLocalAddress()) + require.Nil(t, err, "failed to start opamp server") + + t.Cleanup(func() { + opampServer.Stop() + }) + + opampClientConnection := &opamp.MockOpAmpConnection{} + opampServer.OnMessage( + opampClientConnection, + &protobufs.AgentToServer{ + InstanceUid: "test", + EffectiveConfig: &protobufs.EffectiveConfig{ + ConfigMap: newInitialAgentConfigMap(), + }, + }, + ) + + testbed.opampServer = opampServer + testbed.opampClientConn = opampClientConnection + + return testbed + +} + func (tb *LogPipelinesTestBed) PostPipelinesToQSExpectingStatusCode( postablePipelines logparsingpipeline.PostablePipelines, expectedStatusCode int, @@ -668,43 +760,6 @@ func assertPipelinesResponseMatchesPostedPipelines( } } -func mockOpampAgent( - t *testing.T, - testDBFilePath string, - pipelinesController *logparsingpipeline.LogParsingPipelineController, -) (*opamp.Server, *opamp.MockOpAmpConnection) { - // Mock an available opamp agent - testDB, err := opampModel.InitDB(testDBFilePath) - require.Nil(t, err, "failed to init opamp model") - - agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{ - DB: testDB, - DBEngine: "sqlite", - AgentFeatures: []agentConf.AgentFeature{pipelinesController}, - }) - require.Nil(t, err, "failed to init agentConf") - - opampServer := opamp.InitializeServer(nil, agentConfMgr) - err = opampServer.Start(opamp.GetAvailableLocalAddress()) - require.Nil(t, err, "failed to start opamp server") - - t.Cleanup(func() { - opampServer.Stop() - }) - - opampClientConnection := &opamp.MockOpAmpConnection{} - opampServer.OnMessage( - opampClientConnection, - &protobufs.AgentToServer{ - InstanceUid: "test", - EffectiveConfig: &protobufs.EffectiveConfig{ - ConfigMap: newInitialAgentConfigMap(), - }, - }, - ) - return opampServer, opampClientConnection -} - func newInitialAgentConfigMap() *protobufs.AgentConfigMap { return &protobufs.AgentConfigMap{ ConfigMap: map[string]*protobufs.AgentConfigFile{ diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index 1f6c718b1f..caed48440b 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -192,7 +192,7 @@ services: <<: *db-depend otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.3} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -205,7 +205,7 @@ services: # condition: service_healthy otel-collector: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 container_name: signoz-otel-collector command: [ @@ -245,7 +245,7 @@ services: condition: service_healthy otel-collector-metrics: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 container_name: signoz-otel-collector-metrics command: [