Merge branch 'develop' of github.com:SigNoz/signoz into pranshuchittora/feat/ttl-s3

This commit is contained in:
Pranshu Chittora 2022-04-04 15:06:33 +05:30
commit 8367c106bc
No known key found for this signature in database
GPG Key ID: 3A9E57A016CC0626
18 changed files with 314 additions and 103 deletions

View File

@ -0,0 +1,28 @@
import axios from 'api';
import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
import { AxiosError } from 'axios';
import { ErrorResponse, SuccessResponse } from 'types/api';
import { PayloadProps, Props } from 'types/api/trace/getTagValue';
const getTagValue = async (
props: Props,
): Promise<SuccessResponse<PayloadProps> | ErrorResponse> => {
try {
const response = await axios.post<PayloadProps>(`/getTagValues`, {
start: props.start.toString(),
end: props.end.toString(),
tagKey: props.tagKey,
});
return {
statusCode: 200,
error: null,
message: 'Success',
payload: response.data,
};
} catch (error) {
return ErrorResponseHandler(error as AxiosError);
}
};
export default getTagValue;

View File

@ -27,6 +27,12 @@ function AppLayout(props: AppLayoutProps): JSX.Element {
}
}, [isLoggedIn, isSignUpPage]);
useEffect(() => {
if (isLoggedIn && pathname === ROUTES.SIGN_UP) {
history.push(ROUTES.APPLICATION);
}
}, [isLoggedIn, pathname]);
return (
<Layout>
{!isSignUpPage && <SideNav />}

View File

@ -157,7 +157,7 @@ function ListOfAllDashboard(): JSX.Element {
<TextToolTip
{...{
text: `More details on how to create dashboards`,
url: 'https://signoz.io/docs/userguide/metrics-dashboard',
url: 'https://signoz.io/docs/userguide/dashboards',
}}
/>

View File

@ -114,7 +114,7 @@ function Query({
<TextToolTip
{...{
text: `More details on how to plot metrics graphs`,
url: 'https://signoz.io/docs/userguide/prometheus-metrics/',
url: 'https://signoz.io/docs/userguide/send-metrics/#related-videos',
}}
/>
</ButtonContainer>

View File

@ -0,0 +1,69 @@
import { Select } from 'antd';
import { DefaultOptionType } from 'antd/lib/select';
import getTagValue from 'api/trace/getTagValue';
import useFetch from 'hooks/useFetch';
import React from 'react';
import { useSelector } from 'react-redux';
import { AppState } from 'store/reducers';
import { PayloadProps, Props } from 'types/api/trace/getTagValue';
import { GlobalReducer } from 'types/reducer/globalTime';
import { TraceReducer } from 'types/reducer/trace';
import { Value } from '.';
import { SelectComponent } from './styles';
function TagValue(props: TagValueProps): JSX.Element {
const { tag, setLocalSelectedTags, index, tagKey } = props;
const {
Key: selectedKey,
Operator: selectedOperator,
Values: selectedValues,
} = tag;
const globalReducer = useSelector<AppState, GlobalReducer>(
(state) => state.globalTime,
);
const valueSuggestion = useFetch<PayloadProps, Props>(getTagValue, {
end: globalReducer.maxTime,
start: globalReducer.minTime,
tagKey,
});
return (
<SelectComponent
onSelect={(value: unknown): void => {
if (typeof value === 'string') {
setLocalSelectedTags((tags) => [
...tags.slice(0, index),
{
Key: selectedKey,
Operator: selectedOperator,
Values: [...selectedValues, value],
},
...tags.slice(index + 1, tags.length),
]);
}
}}
loading={valueSuggestion.loading || false}
>
{valueSuggestion.payload &&
valueSuggestion.payload.map((suggestion) => (
<Select.Option key={suggestion.tagValues} value={suggestion.tagValues}>
{suggestion.tagValues}
</Select.Option>
))}
</SelectComponent>
);
}
interface TagValueProps {
index: number;
tag: FlatArray<TraceReducer['selectedTags'], 1>;
setLocalSelectedTags: React.Dispatch<
React.SetStateAction<TraceReducer['selectedTags']>
>;
tagKey: string;
}
export default TagValue;

View File

@ -5,13 +5,9 @@ import { useSelector } from 'react-redux';
import { AppState } from 'store/reducers';
import { TraceReducer } from 'types/reducer/trace';
import {
Container,
IconContainer,
SelectComponent,
ValueSelect,
} from './styles';
import { Container, IconContainer, SelectComponent } from './styles';
import TagsKey from './TagKey';
import TagValue from './TagValue';
const { Option } = Select;
@ -68,7 +64,6 @@ function SingleTags(props: AllTagsProps): JSX.Element {
tag={tag}
setLocalSelectedTags={setLocalSelectedTags}
/>
<SelectComponent
onChange={onChangeOperatorHandler}
value={AllMenu.find((e) => e.key === selectedOperator)?.value || ''}
@ -80,21 +75,16 @@ function SingleTags(props: AllTagsProps): JSX.Element {
))}
</SelectComponent>
<ValueSelect
value={selectedValues}
onChange={(value): void => {
setLocalSelectedTags((tags) => [
...tags.slice(0, index),
{
Key: selectedKey,
Operator: selectedOperator,
Values: value as string[],
},
...tags.slice(index + 1, tags.length),
]);
}}
mode="tags"
/>
{selectedKey[0] ? (
<TagValue
index={index}
tag={tag}
setLocalSelectedTags={setLocalSelectedTags}
tagKey={selectedKey[0]}
/>
) : (
<SelectComponent />
)}
<IconContainer role="button" onClick={(): void => onDeleteTagHandler(index)}>
<CloseOutlined />
@ -112,4 +102,10 @@ interface AllTagsProps {
>;
}
export interface Value {
key: string;
label: string;
value: string;
}
export default SingleTags;

View File

@ -15,12 +15,6 @@ export const SelectComponent = styled(Select)`
}
`;
export const ValueSelect = styled(Select)`
&&& {
width: 100%;
}
`;
export const Container = styled.div`
&&& {
display: flex;

View File

@ -1,12 +1,13 @@
/* eslint-disable */
//@ts-nocheck
import { Card } from 'antd';
import Spinner from 'components/Spinner';
import React, { useEffect, useRef } from 'react';
import { ForceGraph2D } from 'react-force-graph';
import { connect } from 'react-redux';
import { RouteComponentProps, withRouter } from 'react-router-dom';
import { getDetailedServiceMapItems, getServiceMapItems } from 'store/actions';
import { getDetailedServiceMapItems, ServiceMapStore } from 'store/actions';
import { AppState } from 'store/reducers';
import styled from 'styled-components';
import { GlobalTime } from 'types/actions/globalTime';
@ -31,9 +32,8 @@ const Container = styled.div`
`;
interface ServiceMapProps extends RouteComponentProps<any> {
serviceMap: serviceMapStore;
serviceMap: ServiceMapStore;
globalTime: GlobalTime;
getServiceMapItems: (time: GlobalTime) => void;
getDetailedServiceMapItems: (time: GlobalTime) => void;
}
interface graphNode {
@ -53,29 +53,32 @@ export interface graphDataType {
function ServiceMap(props: ServiceMapProps): JSX.Element {
const fgRef = useRef();
const {
getDetailedServiceMapItems,
getServiceMapItems,
globalTime,
serviceMap,
} = props;
const { getDetailedServiceMapItems, globalTime, serviceMap } = props;
useEffect(() => {
/*
Call the apis only when the route is loaded.
Check this issue: https://github.com/SigNoz/signoz/issues/110
*/
getServiceMapItems(globalTime);
getDetailedServiceMapItems(globalTime);
}, [globalTime, getServiceMapItems, getDetailedServiceMapItems]);
}, [globalTime, getDetailedServiceMapItems]);
useEffect(() => {
fgRef.current && fgRef.current.d3Force('charge').strength(-400);
});
if (!serviceMap.items.length || !serviceMap.services.length) {
if (serviceMap.loading) {
return <Spinner size="large" tip="Loading..." />;
}
if (!serviceMap.loading && serviceMap.items.length === 0) {
return (
<Container>
<Card>No Service Found</Card>
</Container>
);
}
const zoomToService = (value: string): void => {
fgRef &&
fgRef.current &&
@ -149,7 +152,6 @@ const mapStateToProps = (
export default withRouter(
connect(mapStateToProps, {
getServiceMapItems,
getDetailedServiceMapItems,
})(ServiceMap),
);

View File

@ -7,6 +7,7 @@ import { ActionTypes } from './types';
export interface ServiceMapStore {
items: ServicesMapItem[];
services: ServicesItem[];
loading: boolean;
}
export interface ServicesItem {
@ -37,38 +38,39 @@ export interface ServicesAction {
payload: ServicesItem[];
}
export const getServiceMapItems = (globalTime: GlobalTime) => {
return async (dispatch: Dispatch): Promise<void> => {
dispatch<ServiceMapItemAction>({
type: ActionTypes.getServiceMapItems,
payload: [],
});
const requestString = `/serviceMapDependencies?start=${globalTime.minTime}&end=${globalTime.maxTime}`;
const response = await api.get<ServicesMapItem[]>(requestString);
dispatch<ServiceMapItemAction>({
type: ActionTypes.getServiceMapItems,
payload: response.data,
});
export interface ServiceMapLoading {
type: ActionTypes.serviceMapLoading;
payload: {
loading: ServiceMapStore['loading'];
};
};
}
export const getDetailedServiceMapItems = (globalTime: GlobalTime) => {
return async (dispatch: Dispatch): Promise<void> => {
dispatch<ServicesAction>({
type: ActionTypes.getServices,
payload: [],
});
const requestString = `/services?start=${globalTime.minTime}&end=${globalTime.maxTime}`;
const response = await api.get<ServicesItem[]>(requestString);
const serviceMapDependencies = `/serviceMapDependencies?start=${globalTime.minTime}&end=${globalTime.maxTime}`;
const [serviceMapDependenciesResponse, response] = await Promise.all([
api.get<ServicesMapItem[]>(serviceMapDependencies),
api.get<ServicesItem[]>(requestString),
]);
dispatch<ServicesAction>({
type: ActionTypes.getServices,
payload: response.data,
});
dispatch<ServiceMapItemAction>({
type: ActionTypes.getServiceMapItems,
payload: serviceMapDependenciesResponse.data,
});
dispatch<ServiceMapLoading>({
type: ActionTypes.serviceMapLoading,
payload: {
loading: false,
},
});
};
};

View File

@ -1,14 +1,22 @@
import { ServiceMapItemAction, ServicesAction } from './serviceMap';
import {
ServiceMapItemAction,
ServiceMapLoading,
ServicesAction,
} from './serviceMap';
import { GetUsageDataAction } from './usage';
export enum ActionTypes {
updateTraceFilters = 'UPDATE_TRACES_FILTER',
updateTimeInterval = 'UPDATE_TIME_INTERVAL',
getServiceMapItems = 'GET_SERVICE_MAP_ITEMS',
getServices = 'GET_SERVICES',
getUsageData = 'GET_USAGE_DATE',
fetchTraces = 'FETCH_TRACES',
fetchTraceItem = 'FETCH_TRACE_ITEM',
serviceMapLoading = 'UPDATE_SERVICE_MAP_LOADING',
}
export type Action = GetUsageDataAction | ServicesAction | ServiceMapItemAction;
export type Action =
| GetUsageDataAction
| ServicesAction
| ServiceMapItemAction
| ServiceMapLoading;

View File

@ -3,6 +3,7 @@ import { Action, ActionTypes, ServiceMapStore } from 'store/actions';
const initialState: ServiceMapStore = {
items: [],
services: [],
loading: true,
};
export const ServiceMapReducer = (
@ -20,6 +21,12 @@ export const ServiceMapReducer = (
...state,
services: action.payload,
};
case ActionTypes.serviceMapLoading: {
return {
...state,
loading: action.payload.loading,
};
}
default:
return state;
}

View File

@ -0,0 +1,13 @@
import { GlobalReducer } from 'types/reducer/globalTime';
export interface Props {
start: GlobalReducer['minTime'];
end: GlobalReducer['maxTime'];
tagKey: string;
}
interface Value {
tagValues: string;
}
export type PayloadProps = Value[];

View File

@ -13,6 +13,7 @@ import (
"net/http"
"net/url"
"os"
"regexp"
"sort"
"strconv"
"strings"
@ -44,8 +45,8 @@ import (
"github.com/prometheus/prometheus/util/strutil"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
am "go.signoz.io/query-service/integrations/alertManager"
"go.signoz.io/query-service/model"
"go.uber.org/zap"
)
@ -75,7 +76,7 @@ type ClickHouseReader struct {
remoteStorage *remote.Storage
ruleManager *rules.Manager
promConfig *config.Config
alertManager am.Manager
alertManager am.Manager
}
// NewTraceReader returns a TraceReader for the database
@ -95,7 +96,7 @@ func NewReader(localDB *sqlx.DB) *ClickHouseReader {
return &ClickHouseReader{
db: db,
localDB: localDB,
alertManager: alertManager,
alertManager: alertManager,
operationsTable: options.primary.OperationsTable,
indexTable: options.primary.IndexTable,
errorTable: options.primary.ErrorTable,
@ -850,7 +851,6 @@ func (r *ClickHouseReader) EditChannel(receiver *am.Receiver, id string) (*am.Re
}
func (r *ClickHouseReader) CreateChannel(receiver *am.Receiver) (*am.Receiver, *model.ApiError) {
tx, err := r.localDB.Begin()
@ -860,8 +860,8 @@ func (r *ClickHouseReader) CreateChannel(receiver *am.Receiver) (*am.Receiver, *
channel_type := getChannelType(receiver)
receiverString, _ := json.Marshal(receiver)
// todo: check if the channel name already exists, raise an error if so
// todo: check if the channel name already exists, raise an error if so
{
stmt, err := tx.Prepare(`INSERT INTO notification_channels (created_at, updated_at, name, type, data) VALUES($1,$2,$3,$4,$5);`)
@ -884,7 +884,7 @@ func (r *ClickHouseReader) CreateChannel(receiver *am.Receiver) (*am.Receiver, *
tx.Rollback()
return nil, apiError
}
err = tx.Commit()
if err != nil {
zap.S().Errorf("Error in commiting transaction for INSERT to notification_channels\n", err)
@ -2602,7 +2602,6 @@ func (r *ClickHouseReader) GetDisks(ctx context.Context) (*[]model.DiskItem, *mo
fmt.Errorf("error while getting disks. Err=%v", err)}
}
zap.S().Infof("Got response: %+v\n", diskItems)
return &diskItems, nil
@ -2610,29 +2609,33 @@ func (r *ClickHouseReader) GetDisks(ctx context.Context) (*[]model.DiskItem, *mo
func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError) {
parseTTL := func(queryResp string) int {
values := strings.Split(queryResp, " ")
N := len(values)
ttlIdx := -1
parseTTL := func(queryResp string) (int, int) {
for i := 0; i < N; i++ {
if strings.Contains(values[i], "toIntervalSecond") {
ttlIdx = i
break
zap.S().Debugf("Parsing TTL from: %s", queryResp)
deleteTTLExp := regexp.MustCompile(`toIntervalSecond\(([0-9]*)\)`)
moveTTLExp := regexp.MustCompile(`toIntervalSecond\(([0-9]*)\) TO VOLUME`)
var delTTL, moveTTL int = -1, -1
m := deleteTTLExp.FindStringSubmatch(queryResp)
if len(m) > 1 {
seconds_int, err := strconv.Atoi(m[1])
if err != nil {
return -1, -1
}
}
if ttlIdx == -1 {
return ttlIdx
delTTL = seconds_int / 3600
}
output := strings.SplitN(values[ttlIdx], "(", 2)
timePart := strings.Trim(output[1], ")")
seconds_int, err := strconv.Atoi(timePart)
if err != nil {
return -1
m = moveTTLExp.FindStringSubmatch(queryResp)
if len(m) > 1 {
seconds_int, err := strconv.Atoi(m[1])
if err != nil {
return -1, -1
}
moveTTL = seconds_int / 3600
}
ttl_hrs := seconds_int / 3600
return ttl_hrs
return delTTL, moveTTL
}
getMetricsTTL := func() (*model.DBResponseTTL, *model.ApiError) {
@ -2671,7 +2674,8 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa
return nil, err
}
return &model.GetTTLResponseItem{TracesTime: parseTTL(dbResp.EngineFull)}, nil
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
return &model.GetTTLResponseItem{TracesTime: delTTL, TracesMoveTime: moveTTL}, nil
case constants.MetricsTTL:
dbResp, err := getMetricsTTL()
@ -2679,7 +2683,9 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa
return nil, err
}
return &model.GetTTLResponseItem{MetricsTime: parseTTL(dbResp.EngineFull)}, nil
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
return &model.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL}, nil
}
db1, err := getTracesTTL()
if err != nil {
@ -2690,9 +2696,15 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa
if err != nil {
return nil, err
}
tracesDelTTL, tracesMoveTTL := parseTTL(db1.EngineFull)
metricsDelTTL, metricsMoveTTL := parseTTL(db2.EngineFull)
return &model.GetTTLResponseItem{TracesTime: parseTTL(db1.EngineFull), MetricsTime: parseTTL(db2.EngineFull)}, nil
return &model.GetTTLResponseItem{
TracesTime: tracesDelTTL,
TracesMoveTime: tracesMoveTTL,
MetricsTime: metricsDelTTL,
MetricsMoveTime: metricsMoveTTL,
}, nil
}
func (r *ClickHouseReader) GetErrors(ctx context.Context, queryParams *model.GetErrorsParams) (*[]model.Error, *model.ApiError) {

View File

@ -914,7 +914,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) {
if err != nil {
return nil, fmt.Errorf("Not a valid toCold TTL duration %v", toColdDuration)
}
if toColdParsed.Seconds() >= durationParsed.Seconds() {
if toColdParsed.Seconds() != 0 && toColdParsed.Seconds() >= durationParsed.Seconds() {
return nil, fmt.Errorf("Delete TTL should be greater than cold storage move TTL.")
}
}

View File

@ -62,7 +62,7 @@ type QueryScan struct {
Limit int64 `json:"limit,omitempty"`
Offset int64 `json:"offset,omitempty"`
BatchSize int64 `json:"batchSize,omitempty"`
Order string `json:"order",omitempty`
Order string `json:"order,omitempty"`
ResultFormat string `json:"resultFormat"`
Context map[string]interface{} `json:"context,omitempty"`
@ -189,7 +189,7 @@ type TimeBoundaryItem struct {
type TimeBoundary struct {
MinTime string `json:"minTime"`
MaxTime string `json:"minTime"`
MaxTime string `json:"maxTime"`
}
func (q *QueryTimeBoundary) setup() { q.QueryType = "timeBoundary" }

View File

@ -283,8 +283,10 @@ type DBResponseTTL struct {
}
type GetTTLResponseItem struct {
MetricsTime int `json:"metrics_ttl_duration_hrs"`
TracesTime int `json:"traces_ttl_duration_hrs"`
MetricsTime int `json:"metrics_ttl_duration_hrs,omitempty"`
MetricsMoveTime int `json:"metrics_move_ttl_duration_hrs,omitempty"`
TracesTime int `json:"traces_ttl_duration_hrs,omitempty"`
TracesMoveTime int `json:"traces_move_ttl_duration_hrs,omitempty"`
}
type DBResponseMinMaxDuration struct {

View File

@ -1,6 +1,7 @@
package tests
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
@ -8,6 +9,7 @@ import (
"time"
"github.com/stretchr/testify/require"
"go.signoz.io/query-service/model"
)
const (
@ -102,6 +104,76 @@ func TestSetTTL(t *testing.T) {
fmt.Printf("=== Found %d objects in Minio\n", count)
}
func getTTL(t *testing.T, table string) *model.GetTTLResponseItem {
req := endpoint + fmt.Sprintf("/api/v1/settings/ttl?type=%s", table)
if len(table) == 0 {
req = endpoint + "/api/v1/settings/ttl"
}
resp, err := client.Get(req)
require.NoError(t, err)
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
res := &model.GetTTLResponseItem{}
require.NoError(t, json.Unmarshal(b, res))
return res
}
func TestGetTTL(t *testing.T) {
r, err := setTTL("traces", "s3", "3600s", "7200s")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp := getTTL(t, "traces")
require.Equal(t, 1, resp.TracesMoveTime)
require.Equal(t, 2, resp.TracesTime)
r, err = setTTL("metrics", "s3", "3600s", "7200s")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "metrics")
require.Equal(t, 1, resp.MetricsMoveTime)
require.Equal(t, 2, resp.MetricsTime)
r, err = setTTL("traces", "s3", "36000s", "72000s")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "")
require.Equal(t, 10, resp.TracesMoveTime)
require.Equal(t, 20, resp.TracesTime)
require.Equal(t, 1, resp.MetricsMoveTime)
require.Equal(t, 2, resp.MetricsTime)
r, err = setTTL("metrics", "s3", "15h", "50h")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "")
require.Equal(t, 10, resp.TracesMoveTime)
require.Equal(t, 20, resp.TracesTime)
require.Equal(t, 15, resp.MetricsMoveTime)
require.Equal(t, 50, resp.MetricsTime)
r, err = setTTL("metrics", "s3", "0s", "0s")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
r, err = setTTL("traces", "s3", "0s", "0s")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "")
require.Equal(t, 0, resp.TracesMoveTime)
require.Equal(t, 0, resp.TracesTime)
require.Equal(t, 0, resp.MetricsMoveTime)
require.Equal(t, 0, resp.MetricsTime)
}
func TestMain(m *testing.M) {
if err := startCluster(); err != nil {
fmt.Println(err)

Binary file not shown.