From 3c90d3c742cc24327880e6e2b68d23cbf1ee9d4a Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Thu, 7 Jan 2021 23:58:19 +0530 Subject: [PATCH 1/4] changed baseUrl for axios --- frontend/src/api/metricsAPI.js | 2 +- frontend/src/api/tracesAPI.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/api/metricsAPI.js b/frontend/src/api/metricsAPI.js index 8943c00b23..5c3954c693 100644 --- a/frontend/src/api/metricsAPI.js +++ b/frontend/src/api/metricsAPI.js @@ -9,7 +9,7 @@ export default axios.create({ // baseURL: 'http://104.211.113.204:8080/api/v1/', // baseURL: process.env.REACT_APP_QUERY_SERVICE_URL, - baseURL: 'api/v1/', + baseURL: '/api/v1/', // console.log('in metrics API', process.env.QUERY_SERVICE_URL) } diff --git a/frontend/src/api/tracesAPI.js b/frontend/src/api/tracesAPI.js index 8e63511731..0f475ff702 100644 --- a/frontend/src/api/tracesAPI.js +++ b/frontend/src/api/tracesAPI.js @@ -8,7 +8,7 @@ export default axios.create({ // baseURL: process.env.QUERY_SERVICE_URL, // console.log('in traces API', process.env.QUERY_SERVICE_URL) - baseURL: 'api/v1/', + baseURL: '/api/v1/', }); From b43b25dab0e559684a309220dbdabad8f2849f22 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Fri, 8 Jan 2021 00:40:19 +0530 Subject: [PATCH 2/4] updates to readme --- .gitignore | 3 ++- deploy/docker/README.md | 0 deploy/kubernetes/README.md | 0 deploy/kubernetes/config.yaml | 0 pkg/README.md | 0 pkg/processors/README.md | 0 pkg/processors/flattener/README.md | 24 +++++++++++++++++++++-- pkg/query-service/README.md | 31 ++++++++++++++++++++++++++++-- 8 files changed, 53 insertions(+), 5 deletions(-) delete mode 100644 deploy/docker/README.md delete mode 100644 deploy/kubernetes/README.md delete mode 100644 deploy/kubernetes/config.yaml delete mode 100644 pkg/README.md delete mode 100644 pkg/processors/README.md diff --git a/.gitignore b/.gitignore index 9d62e2609f..35ddcfd52f 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ frontend/yarn-debug.log* frontend/yarn-error.log* **/.vscode -*.tgz \ No newline at end of file +*.tgz +**/build \ No newline at end of file diff --git a/deploy/docker/README.md b/deploy/docker/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deploy/kubernetes/config.yaml b/deploy/kubernetes/config.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pkg/README.md b/pkg/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pkg/processors/README.md b/pkg/processors/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pkg/processors/flattener/README.md b/pkg/processors/flattener/README.md index d54194fee7..aaac997e71 100644 --- a/pkg/processors/flattener/README.md +++ b/pkg/processors/flattener/README.md @@ -1,3 +1,23 @@ -docker build -t signoz/flattener-processor:v0.1.0 . -docker push signoz/flattener-processor:v0.1.0 +# Flattener Processor +Flattener Processor is a spream processing application that reads spans from topic `otlp_spans` and writes to topic `flattened_spans`. It is written in **Golang**. Flattener Processor is responsible to convert the incoming spans into a flat model that can be ingested to Apache Druid. + + +#### Configuration +Flattener Processor needs below `env` variables to run: +``` + KAFKA_BROKER: signoz-kafka:9092 + KAFKA_INPUT_TOPIC: otlp_spans + KAFKA_OUTPUT_TOPIC: flattened_spans +``` +The above values are the default ones used by SigNoz and are kept at `deploy/kubernetes/platform/signoz-charts/flattener-processor/values.yaml` + +#### Build and Run locally +```console +cd pkg/processors/flattener +go build -o build/flattener-processor main.go +KAFKA_BROKER=xxxx KAFKA_INPUT_TOPIC=otlp_spans KAFKA_OUTPUT_TOPIC=flattened_spans build/flattener-processor +``` + +#### Docker Images +The docker images of flattener-processor is available at https://hub.docker.com/r/signoz/flattener-processor \ No newline at end of file diff --git a/pkg/query-service/README.md b/pkg/query-service/README.md index b996df41d4..c4c96a9757 100644 --- a/pkg/query-service/README.md +++ b/pkg/query-service/README.md @@ -1,2 +1,29 @@ -docker build -t signoz/query-service:v0.1.0 . -docker push signoz/query-service:v0.1.0 \ No newline at end of file +# Query Service + +Query service is the interface between forntend and databases. It is written in **Golang**. It will have modules for all supported databases. Query service is responsible to: +- parse the request from Frontend +- create relevant Druid queries (and all other supported database queries) +- parse response from databases and handle error if any +- build response in the format accepted by Frontend + + +#### Druid Queries +Internally we use both native and sql queries to Druid. + +#### Configuration +Query Service needs below `env` variables to run: +``` + DruidClientUrl: http://signoz-druid-router:8888 + DruidDatasource: flattened_spans +``` +The above values are the default ones used by SigNoz and are kept at `deploy/kubernetes/platform/signoz-charts/query-service/values.yaml` + +#### Build and Run locally +```console +cd pkg/query-service +go build -o build/query-service main.go +DruidClientUrl=xxxx DruidDatasource=flattened_spans build/query-service +``` + +#### Docker Images +The docker images of query-service is available at https://hub.docker.com/r/signoz/query-service \ No newline at end of file From 02dd7c513e09e43b7bdc6b8cdfc47425af5fe2e0 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Sat, 9 Jan 2021 19:10:34 +0530 Subject: [PATCH 3/4] basic telemetry added --- pkg/query-service/app/http_handler.go | 52 ++++++++++++++++++--- pkg/query-service/app/server.go | 39 +++++++++++++++- pkg/query-service/druidQuery/mysql-query.go | 2 +- pkg/query-service/go.mod | 3 ++ pkg/query-service/go.sum | 5 ++ 5 files changed, 92 insertions(+), 9 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 5ecd914107..b0b6c88d00 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -6,6 +6,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/posthog/posthog-go" "go.signoz.io/query-service/druidQuery" "go.signoz.io/query-service/godruid" "go.uber.org/zap" @@ -20,17 +21,21 @@ func NewRouter() *mux.Router { type APIHandler struct { // queryService *querysvc.QueryService // queryParser queryParser - basePath string - apiPrefix string - client *godruid.Client - sqlClient *druidQuery.SqlClient + basePath string + apiPrefix string + client *godruid.Client + sqlClient *druidQuery.SqlClient + pc *posthog.Client + distinctId string } // NewAPIHandler returns an APIHandler -func NewAPIHandler(client *godruid.Client, sqlClient *druidQuery.SqlClient) *APIHandler { +func NewAPIHandler(client *godruid.Client, sqlClient *druidQuery.SqlClient, pc *posthog.Client, distinctId string) *APIHandler { aH := &APIHandler{ - client: client, - sqlClient: sqlClient, + client: client, + sqlClient: sqlClient, + pc: pc, + distinctId: distinctId, } return aH @@ -53,6 +58,7 @@ type structuredError struct { // RegisterRoutes registers routes for this handler on the given router func (aH *APIHandler) RegisterRoutes(router *mux.Router) { + router.HandleFunc("/api/v1/user", aH.user).Methods(http.MethodPost) router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet) router.HandleFunc("/api/v1/services", aH.getServices).Methods(http.MethodGet) router.HandleFunc("/api/v1/services/list", aH.getServicesList).Methods(http.MethodGet) @@ -66,6 +72,31 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router) { router.HandleFunc("/api/v1/usage", aH.getUsage).Methods(http.MethodGet) } +func (aH *APIHandler) user(w http.ResponseWriter, r *http.Request) { + email := r.URL.Query().Get("email") + + var err error + if len(email) == 0 { + err = fmt.Errorf("Email param is missing") + } + if aH.handleError(w, err, http.StatusBadRequest) { + return + } + + (*aH.pc).Enqueue(posthog.Identify{ + DistinctId: aH.distinctId, + Properties: posthog.NewProperties(). + Set("email", email), + }) + + _, err = http.Get(fmt.Sprintf("https://api.telegram.org/bot1518273960:AAHcgVvym9a0Qkl-PKiCI84X1VZaVbkTud0/sendMessage?chat_id=351813222&text=%s", email)) + + if err != nil { + zap.S().Debug(err) + } + +} + func (aH *APIHandler) getOperations(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -171,6 +202,13 @@ func (aH *APIHandler) getServices(w http.ResponseWriter, r *http.Request) { if aH.handleError(w, err, http.StatusBadRequest) { return } + if len(*result) != 4 { + (*aH.pc).Enqueue(posthog.Capture{ + DistinctId: distinctId, + Event: "Different Number of Services", + Properties: posthog.NewProperties().Set("number", len(*result)), + }) + } aH.writeJSON(w, r, result) } diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 6e38f9ab26..527427db6b 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -3,8 +3,12 @@ package app import ( "net" "net/http" + "time" + "github.com/google/uuid" "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/posthog/posthog-go" "github.com/soheilhy/cmux" "go.signoz.io/query-service/druidQuery" "go.signoz.io/query-service/godruid" @@ -74,8 +78,14 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { }, nil } +var posthogClient posthog.Client +var distinctId string + func createHTTPServer(druidClientUrl string) *http.Server { + posthogClient = posthog.New("H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w") + distinctId = uuid.New().String() + client := godruid.Client{ Url: druidClientUrl, Debug: true, @@ -86,9 +96,12 @@ func createHTTPServer(druidClientUrl string) *http.Server { Debug: true, } - apiHandler := NewAPIHandler(&client, &sqlClient) + apiHandler := NewAPIHandler(&client, &sqlClient, &posthogClient, distinctId) r := NewRouter() + r.Use(analyticsMiddleware) + r.Use(loggingMiddleware) + apiHandler.RegisterRoutes(r) // c := cors.New(cors.Options{ @@ -107,6 +120,30 @@ func createHTTPServer(druidClientUrl string) *http.Server { } } +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + route := mux.CurrentRoute(r) + path, _ := route.GetPathTemplate() + startTime := time.Now() + next.ServeHTTP(w, r) + zap.S().Info(path, "\ttimeTaken: ", time.Now().Sub(startTime)) + }) +} + +func analyticsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + route := mux.CurrentRoute(r) + path, _ := route.GetPathTemplate() + + posthogClient.Enqueue(posthog.Capture{ + DistinctId: distinctId, + Event: path, + }) + + next.ServeHTTP(w, r) + }) +} + // initListener initialises listeners of the server func (s *Server) initListener() (cmux.CMux, error) { if s.separatePorts { // use separate ports and listeners each for gRPC and HTTP requests diff --git a/pkg/query-service/druidQuery/mysql-query.go b/pkg/query-service/druidQuery/mysql-query.go index bf359825b5..dfff5050f5 100644 --- a/pkg/query-service/druidQuery/mysql-query.go +++ b/pkg/query-service/druidQuery/mysql-query.go @@ -92,7 +92,7 @@ func GetServicesList(client *SqlClient) (*[]string, error) { return nil, fmt.Errorf("Something went wrong in druid query") } - zap.S().Info(string(response)) + // zap.S().Info(string(response)) res := new([][]string) err = json.Unmarshal(response, res) diff --git a/pkg/query-service/go.mod b/pkg/query-service/go.mod index 5beca0eaac..2cf5d0eccf 100644 --- a/pkg/query-service/go.mod +++ b/pkg/query-service/go.mod @@ -3,15 +3,18 @@ module go.signoz.io/query-service go 1.14 require ( + github.com/google/uuid v1.1.1 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/jaegertracing/jaeger v1.21.0 github.com/ory/viper v1.7.5 + github.com/posthog/posthog-go v0.0.0-20200525173953-e46dc8e6b89b github.com/rs/cors v1.7.0 github.com/shunfei/godruid v0.0.0-20171207111340-296a59dd69bd github.com/sirupsen/logrus v1.7.0 github.com/smartystreets/goconvey v1.6.4 github.com/soheilhy/cmux v0.1.4 github.com/spf13/viper v1.6.2 + github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.uber.org/zap v1.16.0 ) diff --git a/pkg/query-service/go.sum b/pkg/query-service/go.sum index e1474af29a..4cce05fccc 100644 --- a/pkg/query-service/go.sum +++ b/pkg/query-service/go.sum @@ -221,6 +221,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -403,6 +404,8 @@ github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6J github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posthog/posthog-go v0.0.0-20200525173953-e46dc8e6b89b h1:a8lLvAV+8OQXnG18ZOV5ctFQY7jLHa3brA9BBhe1SVs= +github.com/posthog/posthog-go v0.0.0-20200525173953-e46dc8e6b89b/go.mod h1:s7IZAf1WuSPTb/R/agnboYa+gDnoKGdqIk7p2aFHDYs= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -512,6 +515,8 @@ github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHM github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= +github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= From 014c8c1c0f80628c99457462d41d24bf04fd10d6 Mon Sep 17 00:00:00 2001 From: Ankit Nayan Date: Sat, 9 Jan 2021 19:59:41 +0530 Subject: [PATCH 4/4] updated query-service chart version --- .../kubernetes/platform/signoz-charts/query-service/Chart.yaml | 2 +- .../kubernetes/platform/signoz-charts/query-service/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/kubernetes/platform/signoz-charts/query-service/Chart.yaml b/deploy/kubernetes/platform/signoz-charts/query-service/Chart.yaml index 5a7e010b75..56e0159f0d 100644 --- a/deploy/kubernetes/platform/signoz-charts/query-service/Chart.yaml +++ b/deploy/kubernetes/platform/signoz-charts/query-service/Chart.yaml @@ -18,4 +18,4 @@ version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.1.2 +appVersion: 0.1.3 diff --git a/deploy/kubernetes/platform/signoz-charts/query-service/values.yaml b/deploy/kubernetes/platform/signoz-charts/query-service/values.yaml index 55f76d2ef7..a145c18809 100644 --- a/deploy/kubernetes/platform/signoz-charts/query-service/values.yaml +++ b/deploy/kubernetes/platform/signoz-charts/query-service/values.yaml @@ -16,6 +16,7 @@ fullnameOverride: "" configVars: DruidClientUrl: http://signoz-druid-router:8888 DruidDatasource: flattened_spans + POSTHOG_API_KEY: "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w" serviceAccount: