feat: add ability to import Grafana dashboards (#1700)

* feat: add ability to import Grafana dashboards

* chore: remove unnecessary file

* chore: more 9XX support

* chore: some more hacks

* chore: update deps

* chore: arrange equal spaced widgets instead of inheriting from grafana
This commit is contained in:
Srikanth Chekuri 2022-11-10 16:49:54 +05:30 committed by GitHub
parent 674883cd18
commit 9735a6e5ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 514 additions and 5 deletions

View File

@ -1,6 +1,7 @@
{
"create_dashboard": "Create Dashboard",
"import_json": "Import JSON",
"import_grafana_json": "Import Grafana JSON",
"copy_to_clipboard": "Copy To ClipBoard",
"download_json": "Download JSON",
"view_json": "View JSON",

View File

@ -1,6 +1,7 @@
{
"create_dashboard": "Create Dashboard",
"import_json": "Import JSON",
"import_grafana_json": "Import Grafana JSON",
"copy_to_clipboard": "Copy To ClipBoard",
"download_json": "Download JSON",
"view_json": "View JSON",

View File

@ -7,8 +7,9 @@ import { PayloadProps, Props } from 'types/api/dashboard/create';
const create = async (
props: Props,
): Promise<SuccessResponse<PayloadProps> | ErrorResponse> => {
const url = props.uploadedGrafana ? '/dashboards/grafana' : '/dashboards';
try {
const response = await axios.post('/dashboards', {
const response = await axios.post(url, {
...props,
});

View File

@ -26,6 +26,7 @@ import { EditorContainer, FooterContainer } from './styles';
function ImportJSON({
isImportJSONModalVisible,
uploadedGrafana,
onModalHandler,
}: ImportJSONProps): JSX.Element {
const [jsonData, setJsonData] = useState<Record<string, unknown>>();
@ -89,6 +90,7 @@ function ImportJSON({
const response = await createDashboard({
...parsedWidgets,
uploadedGrafana,
});
if (response.statusCode === 200) {
@ -186,6 +188,7 @@ function ImportJSON({
interface ImportJSONProps {
isImportJSONModalVisible: boolean;
onModalHandler: VoidFunction;
uploadedGrafana: boolean;
}
export default ImportJSON;

View File

@ -57,6 +57,7 @@ function ListOfAllDashboard(): JSX.Element {
isImportJSONModalVisible,
setIsImportJSONModalVisible,
] = useState<boolean>(false);
const [uploadedGrafana, setUploadedGrafana] = useState<boolean>(false);
const [filteredDashboards, setFilteredDashboards] = useState<Dashboard[]>();
@ -137,6 +138,7 @@ function ListOfAllDashboard(): JSX.Element {
title: t('new_dashboard_title', {
ns: 'dashboard',
}),
uploadedGrafana: false,
});
if (response.statusCode === 200) {
@ -182,8 +184,9 @@ function ListOfAllDashboard(): JSX.Element {
newDashboardState.loading,
]);
const onModalHandler = (): void => {
const onModalHandler = (uploadedGrafana: boolean): void => {
setIsImportJSONModalVisible((state) => !state);
setUploadedGrafana(uploadedGrafana);
};
const menu = useMemo(
@ -198,9 +201,18 @@ function ListOfAllDashboard(): JSX.Element {
{t('create_dashboard')}
</Menu.Item>
)}
<Menu.Item onClick={onModalHandler} key={t('import_json').toString()}>
<Menu.Item
onClick={(): void => onModalHandler(false)}
key={t('import_json').toString()}
>
{t('import_json')}
</Menu.Item>
<Menu.Item
onClick={(): void => onModalHandler(true)}
key={t('import_grafana_json').toString()}
>
{t('import_grafana_json')}
</Menu.Item>
</Menu>
),
[createNewDashboard, loading, onNewDashboardHandler, t],
@ -256,7 +268,8 @@ function ListOfAllDashboard(): JSX.Element {
<TableContainer>
<ImportJSON
isImportJSONModalVisible={isImportJSONModalVisible}
onModalHandler={onModalHandler}
uploadedGrafana={uploadedGrafana}
onModalHandler={(): void => onModalHandler(false)}
/>
<Table
pagination={{

View File

@ -3,7 +3,8 @@ import { Dashboard, DashboardData } from './getAll';
export type Props =
| {
title: Dashboard['data']['title'];
uploadedGrafana: boolean;
}
| DashboardData;
| { DashboardData: DashboardData; uploadedGrafana: boolean };
export type PayloadProps = Dashboard;

1
go.mod
View File

@ -14,6 +14,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/mattn/go-sqlite3 v1.14.8
github.com/minio/minio-go/v6 v6.0.57
github.com/mitchellh/mapstructure v1.5.0
github.com/oklog/oklog v0.3.2
github.com/pkg/errors v0.9.1
github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f

2
go.sum
View File

@ -345,6 +345,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=

View File

@ -4,12 +4,15 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/gosimple/slug"
"github.com/jmoiron/sqlx"
"github.com/mitchellh/mapstructure"
"go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)
@ -260,3 +263,199 @@ func SlugifyTitle(title string) string {
}
return s
}
func TransformGrafanaJSONToSignoz(grafanaJSON model.GrafanaJSON) model.DashboardData {
var toReturn model.DashboardData
toReturn.Title = grafanaJSON.Title
toReturn.Tags = grafanaJSON.Tags
toReturn.Variables = make(map[string]model.Variable)
for templateIdx, template := range grafanaJSON.Templating.List {
var sort, typ, textboxValue, customValue, queryValue string
if template.Sort == 1 {
sort = "ASC"
} else if template.Sort == 2 {
sort = "DESC"
} else {
sort = "DISABLED"
}
if template.Type == "query" {
if template.Datasource == nil {
zap.S().Warnf("Skipping panel %d as it has no datasource", templateIdx)
continue
}
// Skip if the source is not prometheus
source, stringOk := template.Datasource.(string)
if stringOk && !strings.Contains(strings.ToLower(source), "prometheus") {
zap.S().Warnf("Skipping template %d as it is not prometheus", templateIdx)
continue
}
var result model.Datasource
var structOk bool
if reflect.TypeOf(template.Datasource).Kind() == reflect.Map {
err := mapstructure.Decode(template.Datasource, &result)
if err == nil {
structOk = true
}
}
if result.Type != "prometheus" && result.Type != "" {
zap.S().Warnf("Skipping template %d as it is not prometheus", templateIdx)
continue
}
if !stringOk && !structOk {
zap.S().Warnf("Didn't recognize source, skipping")
continue
}
typ = "QUERY"
} else if template.Type == "custom" {
typ = "CUSTOM"
} else if template.Type == "textbox" {
typ = "TEXTBOX"
text, ok := template.Current.Text.(string)
if ok {
textboxValue = text
}
array, ok := template.Current.Text.([]string)
if ok {
textboxValue = strings.Join(array, ",")
}
} else {
continue
}
var selectedValue string
text, ok := template.Current.Value.(string)
if ok {
selectedValue = text
}
array, ok := template.Current.Value.([]string)
if ok {
selectedValue = strings.Join(array, ",")
}
toReturn.Variables[template.Name] = model.Variable{
AllSelected: false,
CustomValue: customValue,
Description: template.Label,
MultiSelect: template.Multi,
QueryValue: queryValue,
SelectedValue: selectedValue,
ShowALLOption: template.IncludeAll,
Sort: sort,
TextboxValue: textboxValue,
Type: typ,
}
}
row := 0
for idx, panel := range grafanaJSON.Panels {
if panel.Datasource == nil {
zap.S().Warnf("Skipping panel %d as it has no datasource", idx)
continue
}
// Skip if the datasource is not prometheus
source, stringOk := panel.Datasource.(string)
if stringOk && !strings.Contains(strings.ToLower(source), "prometheus") {
zap.S().Warnf("Skipping panel %d as it is not prometheus", idx)
continue
}
var result model.Datasource
var structOk bool
if reflect.TypeOf(panel.Datasource).Kind() == reflect.Map {
err := mapstructure.Decode(panel.Datasource, &result)
if err == nil {
structOk = true
}
}
if result.Type != "prometheus" && result.Type != "" {
zap.S().Warnf("Skipping panel %d as it is not prometheus", idx)
continue
}
if !stringOk && !structOk {
zap.S().Warnf("Didn't recognize source, skipping")
continue
}
// Create a panel from "gridPos"
if idx%3 == 0 {
row++
}
toReturn.Layout = append(
toReturn.Layout,
model.Layout{
X: idx % 3 * 4,
Y: row * 3,
W: 4,
H: 3,
I: strconv.Itoa(idx),
},
)
widget := model.Widget{
Description: panel.Description,
ID: strconv.Itoa(idx),
IsStacked: false,
NullZeroValues: "zero",
Opacity: "1",
PanelTypes: "TIME_SERIES", // TODO: Need to figure out how to get this
Query: model.Query{
ClickHouse: []model.ClickHouseQueryDashboard{
{
Disabled: false,
Legend: "",
Name: "A",
Query: "",
},
},
MetricsBuilder: model.MetricsBuilder{
Formulas: []string{},
QueryBuilder: []model.QueryBuilder{
{
AggregateOperator: 1,
Disabled: false,
GroupBy: []string{},
Legend: "",
MetricName: "",
Name: "A",
ReduceTo: 1,
},
},
},
PromQL: []model.PromQueryDashboard{},
QueryType: int(model.PROM),
},
QueryData: model.QueryDataDashboard{
Data: model.Data{
QueryData: []interface{}{},
},
},
Title: panel.Title,
YAxisUnit: panel.FieldConfig.Defaults.Unit,
QueryType: int(model.PROM), // TODO: Supprot for multiple query types
}
for _, target := range panel.Targets {
if target.Expr != "" {
for name := range toReturn.Variables {
target.Expr = strings.ReplaceAll(target.Expr, "$"+name, "{{"+"."+name+"}}")
target.Expr = strings.ReplaceAll(target.Expr, "$"+"__rate_interval", "5m")
}
widget.Query.PromQL = append(
widget.Query.PromQL,
model.PromQueryDashboard{
Disabled: false,
Legend: target.LegendFormat,
Name: target.RefID,
Query: target.Expr,
},
)
}
}
toReturn.Widgets = append(toReturn.Widgets, widget)
}
return toReturn
}

View File

@ -339,6 +339,7 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router) {
router.HandleFunc("/api/v1/dashboards", ViewAccess(aH.getDashboards)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/dashboards", EditAccess(aH.createDashboards)).Methods(http.MethodPost)
router.HandleFunc("/api/v1/dashboards/grafana", EditAccess(aH.createDashboardsTransform)).Methods(http.MethodPost)
router.HandleFunc("/api/v1/dashboards/{uuid}", ViewAccess(aH.getDashboard)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/dashboards/{uuid}", EditAccess(aH.updateDashboard)).Methods(http.MethodPut)
router.HandleFunc("/api/v1/dashboards/{uuid}", EditAccess(aH.deleteDashboard)).Methods(http.MethodDelete)
@ -823,6 +824,40 @@ func (aH *APIHandler) getDashboard(w http.ResponseWriter, r *http.Request) {
}
func (aH *APIHandler) saveAndReturn(w http.ResponseWriter, signozDashboard model.DashboardData) {
toSave := make(map[string]interface{})
toSave["title"] = signozDashboard.Title
toSave["description"] = signozDashboard.Description
toSave["tags"] = signozDashboard.Tags
toSave["layout"] = signozDashboard.Layout
toSave["widgets"] = signozDashboard.Widgets
toSave["variables"] = signozDashboard.Variables
dashboard, apiError := dashboards.CreateDashboard(toSave)
if apiError != nil {
RespondError(w, apiError, nil)
return
}
aH.Respond(w, dashboard)
return
}
func (aH *APIHandler) createDashboardsTransform(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
var importData model.GrafanaJSON
err = json.Unmarshal(b, &importData)
if err == nil {
signozDashboard := dashboards.TransformGrafanaJSONToSignoz(importData)
aH.saveAndReturn(w, signozDashboard)
return
}
RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, "Error while creating dashboard from grafana json")
}
func (aH *APIHandler) createDashboards(w http.ResponseWriter, r *http.Request) {
var postData map[string]interface{}

View File

@ -0,0 +1,252 @@
package model
type Datasource struct {
Type string `json:"type"`
UID string `json:"uid"`
}
type GrafanaJSON struct {
Inputs []struct {
Name string `json:"name"`
Label string `json:"label"`
Description string `json:"description"`
Type string `json:"type"`
PluginID string `json:"pluginId"`
PluginName string `json:"pluginName"`
} `json:"__inputs"`
Requires []struct {
Type string `json:"type"`
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
} `json:"__requires"`
Annotations struct {
List []struct {
HashKey string `json:"$$hashKey"`
BuiltIn int `json:"builtIn"`
Datasource interface{} `json:"datasource"`
Enable bool `json:"enable"`
Hide bool `json:"hide"`
IconColor string `json:"iconColor"`
Name string `json:"name"`
Target struct {
Limit int `json:"limit"`
MatchAny bool `json:"matchAny"`
Tags []interface{} `json:"tags"`
Type string `json:"type"`
} `json:"target"`
Type string `json:"type"`
} `json:"list"`
} `json:"annotations"`
Editable bool `json:"editable"`
FiscalYearStartMonth int `json:"fiscalYearStartMonth"`
GnetID int `json:"gnetId"`
GraphTooltip int `json:"graphTooltip"`
ID interface{} `json:"id"`
Links []struct {
Icon string `json:"icon"`
Tags []interface{} `json:"tags"`
TargetBlank bool `json:"targetBlank"`
Title string `json:"title"`
Type string `json:"type"`
URL string `json:"url"`
} `json:"links"`
LiveNow bool `json:"liveNow"`
Panels []struct {
Datasource interface{} `json:"datasource"`
Description string `json:"description,omitempty"`
FieldConfig struct {
Defaults struct {
Color struct {
Mode string `json:"mode"`
} `json:"color"`
Max float64 `json:"max"`
Min float64 `json:"min"`
Thresholds struct {
Mode string `json:"mode"`
Steps []struct {
Color string `json:"color"`
Value interface{} `json:"value"`
} `json:"steps"`
} `json:"thresholds"`
Unit string `json:"unit"`
} `json:"defaults"`
Overrides []interface{} `json:"overrides"`
} `json:"fieldConfig,omitempty"`
GridPos struct {
H int `json:"h"`
W int `json:"w"`
X int `json:"x"`
Y int `json:"y"`
} `json:"gridPos"`
ID int `json:"id"`
Links []interface{} `json:"links,omitempty"`
Options struct {
Orientation string `json:"orientation"`
ReduceOptions struct {
Calcs []string `json:"calcs"`
Fields string `json:"fields"`
Values bool `json:"values"`
} `json:"reduceOptions"`
ShowThresholdLabels bool `json:"showThresholdLabels"`
ShowThresholdMarkers bool `json:"showThresholdMarkers"`
} `json:"options,omitempty"`
PluginVersion string `json:"pluginVersion,omitempty"`
Targets []struct {
Datasource interface{} `json:"datasource"`
EditorMode string `json:"editorMode"`
Expr string `json:"expr"`
Hide bool `json:"hide"`
IntervalFactor int `json:"intervalFactor"`
LegendFormat string `json:"legendFormat"`
Range bool `json:"range"`
RefID string `json:"refId"`
Step int `json:"step"`
} `json:"targets"`
Title string `json:"title"`
Type string `json:"type"`
HideTimeOverride bool `json:"hideTimeOverride,omitempty"`
MaxDataPoints int `json:"maxDataPoints,omitempty"`
Collapsed bool `json:"collapsed,omitempty"`
Panels []interface{} `json:"panels,omitempty"`
} `json:"panels"`
SchemaVersion int `json:"schemaVersion"`
Style string `json:"style"`
Tags []string `json:"tags"`
Templating struct {
List []struct {
Current struct {
Selected bool `json:"selected"`
Text interface{} `json:"text"`
Value interface{} `json:"value"`
} `json:"current"`
Hide int `json:"hide"`
IncludeAll bool `json:"includeAll"`
Label string `json:"label,omitempty"`
Multi bool `json:"multi"`
Name string `json:"name"`
Options []interface{} `json:"options"`
Query interface{} `json:"query"`
Refresh int `json:"refresh,omitempty"`
Regex string `json:"regex,omitempty"`
SkipURLSync bool `json:"skipUrlSync"`
Type string `json:"type"`
Datasource interface{} `json:"datasource,omitempty"`
Definition string `json:"definition,omitempty"`
Sort int `json:"sort,omitempty"`
TagValuesQuery string `json:"tagValuesQuery,omitempty"`
TagsQuery string `json:"tagsQuery,omitempty"`
UseTags bool `json:"useTags,omitempty"`
} `json:"list"`
} `json:"templating"`
Time struct {
From string `json:"from"`
To string `json:"to"`
} `json:"time"`
Timepicker struct {
RefreshIntervals []string `json:"refresh_intervals"`
TimeOptions []string `json:"time_options"`
} `json:"timepicker"`
Timezone string `json:"timezone"`
Title string `json:"title"`
UID string `json:"uid"`
Version int `json:"version"`
WeekStart string `json:"weekStart"`
}
type Layout struct {
H int `json:"h"`
I string `json:"i"`
Moved bool `json:"moved"`
Static bool `json:"static"`
W int `json:"w"`
X int `json:"x"`
Y int `json:"y"`
}
type Variable struct {
AllSelected bool `json:"allSelected"`
CustomValue string `json:"customValue"`
Description string `json:"description"`
ModificationUUID string `json:"modificationUUID"`
MultiSelect bool `json:"multiSelect"`
QueryValue string `json:"queryValue"`
SelectedValue string `json:"selectedValue"`
ShowALLOption bool `json:"showALLOption"`
Sort string `json:"sort"`
TextboxValue string `json:"textboxValue"`
Type string `json:"type"`
}
type Data struct {
Legend string `json:"legend"`
Query string `json:"query"`
QueryData []interface{} `json:"queryData"`
}
type QueryDataDashboard struct {
Data Data `json:"data"`
Error bool `json:"error"`
ErrorMessage string `json:"errorMessage"`
Loading bool `json:"loading"`
}
type ClickHouseQueryDashboard struct {
Legend string `json:"legend"`
Name string `json:"name"`
Query string `json:"rawQuery"`
Disabled bool `json:"disabled"`
}
type QueryBuilder struct {
AggregateOperator interface{} `json:"aggregateOperator"`
Disabled bool `json:"disabled"`
GroupBy []string `json:"groupBy"`
Legend string `json:"legend"`
MetricName string `json:"metricName"`
Name string `json:"name"`
TagFilters TagFilters `json:"tagFilters"`
ReduceTo interface{} `json:"reduceTo"`
}
type MetricsBuilder struct {
Formulas []string `json:"formulas"`
QueryBuilder []QueryBuilder `json:"queryBuilder"`
}
type PromQueryDashboard struct {
Query string `json:"query"`
Disabled bool `json:"disabled"`
Name string `json:"name"`
Legend string `json:"legend"`
}
type Query struct {
ClickHouse []ClickHouseQueryDashboard `json:"clickHouse"`
PromQL []PromQueryDashboard `json:"promQL"`
MetricsBuilder MetricsBuilder `json:"metricsBuilder"`
QueryType int `json:"queryType"`
}
type Widget struct {
Description string `json:"description"`
ID string `json:"id"`
IsStacked bool `json:"isStacked"`
NullZeroValues string `json:"nullZeroValues"`
Opacity string `json:"opacity"`
PanelTypes string `json:"panelTypes"`
Query Query `json:"query"`
QueryData QueryDataDashboard `json:"queryData"`
TimePreferance string `json:"timePreferance"`
Title string `json:"title"`
YAxisUnit string `json:"yAxisUnit"`
QueryType int `json:"queryType"`
}
type DashboardData struct {
Description string `json:"description"`
Tags []string `json:"tags"`
Layout []Layout `json:"layout"`
Title string `json:"title"`
Widgets []Widget `json:"widgets"`
Variables map[string]Variable `json:"variables"`
}