From 210c5fd7f23556187b688e99427ee385c67d5c5a Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 15 Mar 2023 15:09:15 +0530 Subject: [PATCH] feat: opamp server application (#1787) * feat: opamp server application * chore: opamp * chore: refactor server implementation * chore: add Stop * chore: merged opamp updates * chore: removed all errorf * chore: added a comment about zero version * feat: added user context for created by * chore: changed debugf to debug * chore: removed lb from opamp + added config parser * fix: added userid to ConfigNewVersion() * chore: removed user id from contxt and added config parser * fix: removed lock inside re-deploy * chore: added config db fix * fix: merged app/server.go from develop * fix: restored extract jwt * Update pkg/query-service/app/server.go Co-authored-by: Nityananda Gohain * fix: dependency version fix and import added --------- Co-authored-by: Pranay Prateek Co-authored-by: Palash Gupta Co-authored-by: mindhash Co-authored-by: Nityananda Gohain --- ee/query-service/app/server.go | 61 +++- go.mod | 13 +- go.sum | 141 +++++++- pkg/query-service/agentConf/db.go | 236 +++++++++++++ pkg/query-service/agentConf/manager.go | 212 +++++++++++ pkg/query-service/agentConf/sqlite/init.go | 65 ++++ pkg/query-service/agentConf/version.go | 69 ++++ pkg/query-service/app/opamp/config.yaml | 76 ++++ .../app/opamp/configure_ingestionRules.go | 141 ++++++++ pkg/query-service/app/opamp/logger.go | 15 + pkg/query-service/app/opamp/model/agent.go | 334 ++++++++++++++++++ pkg/query-service/app/opamp/model/agents.go | 117 ++++++ .../app/opamp/model/coordinator.go | 66 ++++ pkg/query-service/app/opamp/opamp_server.go | 204 +++++++++++ .../app/opamp/otelconfig/config_parser.go | 195 ++++++++++ .../opamp/otelconfig/config_parser_test.go | 58 +++ .../otelconfig/filterprocessor/config.go | 11 + .../opamp/otelconfig/otlpreceiver/config.go | 6 + .../otelconfig/otlpreceiver/grpcSettings.go | 14 + .../otelconfig/otlpreceiver/httpSettings.go | 9 + .../app/opamp/otelconfig/otlpreceiver/tls.go | 38 ++ .../opamp/otelconfig/tailsampler/config.go | 82 +++++ .../app/opamp/otelconfig/testdata/basic.yaml | 76 ++++ .../opamp/otelconfig/testdata/service.yaml | 11 + .../app/opamp/pipeline_builder.go | 196 ++++++++++ pkg/query-service/app/opamp/signal.go | 9 + pkg/query-service/app/server.go | 43 +++ pkg/query-service/constants/constants.go | 1 + pkg/query-service/main.go | 8 +- 29 files changed, 2494 insertions(+), 13 deletions(-) create mode 100644 pkg/query-service/agentConf/db.go create mode 100644 pkg/query-service/agentConf/manager.go create mode 100644 pkg/query-service/agentConf/sqlite/init.go create mode 100644 pkg/query-service/agentConf/version.go create mode 100644 pkg/query-service/app/opamp/config.yaml create mode 100644 pkg/query-service/app/opamp/configure_ingestionRules.go create mode 100644 pkg/query-service/app/opamp/logger.go create mode 100644 pkg/query-service/app/opamp/model/agent.go create mode 100644 pkg/query-service/app/opamp/model/agents.go create mode 100644 pkg/query-service/app/opamp/model/coordinator.go create mode 100644 pkg/query-service/app/opamp/opamp_server.go create mode 100644 pkg/query-service/app/opamp/otelconfig/config_parser.go create mode 100644 pkg/query-service/app/opamp/otelconfig/config_parser_test.go create mode 100644 pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go create mode 100644 pkg/query-service/app/opamp/otelconfig/tailsampler/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml create mode 100644 pkg/query-service/app/opamp/otelconfig/testdata/service.yaml create mode 100644 pkg/query-service/app/opamp/pipeline_builder.go create mode 100644 pkg/query-service/app/opamp/signal.go diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index af4a90f885..72091cc1e3 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -25,16 +25,18 @@ import ( licensepkg "go.signoz.io/signoz/ee/query-service/license" "go.signoz.io/signoz/ee/query-service/usage" + "go.signoz.io/signoz/pkg/query-service/agentConf" baseapp "go.signoz.io/signoz/pkg/query-service/app" "go.signoz.io/signoz/pkg/query-service/app/dashboards" - "go.signoz.io/signoz/pkg/query-service/app/explorer" + baseexplorer "go.signoz.io/signoz/pkg/query-service/app/explorer" + "go.signoz.io/signoz/pkg/query-service/app/opamp" + opAmpModel "go.signoz.io/signoz/pkg/query-service/app/opamp/model" baseauth "go.signoz.io/signoz/pkg/query-service/auth" - "go.signoz.io/signoz/pkg/query-service/constants" baseconst "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/healthcheck" basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager" baseint "go.signoz.io/signoz/pkg/query-service/interfaces" - "go.signoz.io/signoz/pkg/query-service/model" + basemodel "go.signoz.io/signoz/pkg/query-service/model" pqle "go.signoz.io/signoz/pkg/query-service/pqlEngine" rules "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/telemetry" @@ -42,6 +44,8 @@ import ( "go.uber.org/zap" ) +const AppDbEngine = "sqlite" + type ServerOptions struct { PromConfigPath string HTTPHostPort string @@ -85,8 +89,9 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { return nil, err } + baseexplorer.InitWithDSN(baseconst.RELATIONAL_DATASOURCE_PATH) + localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH) - explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH) if err != nil { return nil, err @@ -127,6 +132,17 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { return nil, err } + // initiate opamp + _, err = opAmpModel.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH) + if err != nil { + return nil, err + } + + // initiate agent config handler + if err := agentConf.Initiate(localDB, AppDbEngine); err != nil { + return nil, err + } + // start the usagemanager usageManager, err := usage.New("sqlite", localDB, lm.GetRepo(), reader.GetConn()) if err != nil { @@ -208,7 +224,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e r := mux.NewRouter() - getUserFromRequest := func(r *http.Request) (*model.UserPayload, error) { + getUserFromRequest := func(r *http.Request) (*basemodel.UserPayload, error) { patToken := r.Header.Get("SIGNOZ-API-KEY") if len(patToken) > 0 { zap.S().Debugf("Received a non-zero length PAT token") @@ -299,7 +315,7 @@ func extractDashboardMetaData(path string, r *http.Request) (map[string]interfac pathToExtractBodyFrom := "/api/v2/metrics/query_range" data := map[string]interface{}{} - var postData *model.QueryRangeParamsV2 + var postData *basemodel.QueryRangeParamsV2 if path == pathToExtractBodyFrom && (r.Method == "POST") { if r.Body != nil { @@ -472,7 +488,7 @@ func (s *Server) Start() error { if port, err := utils.GetPort(s.privateConn.Addr()); err == nil { privatePort = port } - fmt.Println("starting private http") + go func() { zap.S().Info("Starting Private HTTP server", zap.Int("port", privatePort), zap.String("addr", s.serverOptions.PrivateHostPort)) @@ -488,6 +504,37 @@ func (s *Server) Start() error { }() + go func() { + zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", baseconst.OpAmpWsEndpoint)) + err := opamp.InitalizeServer(baseconst.OpAmpWsEndpoint, &opAmpModel.AllAgents) + if err != nil { + zap.S().Info("opamp ws server failed to start", err) + s.unavailableChannel <- healthcheck.Unavailable + } + }() + + return nil +} + +func (s *Server) Stop() error { + if s.httpServer != nil { + if err := s.httpServer.Shutdown(context.Background()); err != nil { + return err + } + } + + if s.privateHTTP != nil { + if err := s.privateHTTP.Shutdown(context.Background()); err != nil { + return err + } + } + + opamp.StopServer() + + if s.ruleManager != nil { + s.ruleManager.Stop() + } + return nil } diff --git a/go.mod b/go.mod index 490ecb53d7..cccc971615 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,13 @@ require ( github.com/gosimple/slug v1.10.0 github.com/jmoiron/sqlx v1.3.4 github.com/json-iterator/go v1.1.12 + github.com/knadh/koanf v1.5.0 github.com/mailru/easyjson v0.7.7 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/minio/minio-go/v6 v6.0.57 github.com/mitchellh/mapstructure v1.5.0 github.com/oklog/oklog v0.3.2 + github.com/open-telemetry/opamp-go v0.5.0 github.com/pkg/errors v0.9.1 github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f github.com/prometheus/common v0.39.0 @@ -28,8 +30,10 @@ require ( github.com/sethvargo/go-password v0.2.0 github.com/smartystreets/goconvey v1.6.4 github.com/soheilhy/cmux v0.1.5 + go.opentelemetry.io/collector/confmap v0.70.0 go.uber.org/zap v1.24.0 gopkg.in/segmentio/analytics-go.v3 v3.1.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.26.0 ) @@ -66,17 +70,20 @@ require ( github.com/beevik/etree v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid v1.2.3 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/minio/md5-simd v1.1.0 // indirect github.com/minio/sha256-simd v0.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect + go.opentelemetry.io/collector/featuregate v0.70.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -116,7 +123,7 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/smartystreets/assertions v1.1.0 github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.opentelemetry.io/otel v1.11.2 // indirect go.opentelemetry.io/otel/trace v1.11.2 // indirect @@ -131,7 +138,7 @@ require ( golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.51.0 - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index f437e8cf4c..19ef4f20ff 100644 --- a/go.sum +++ b/go.sum @@ -89,13 +89,27 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q= github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/auth0/go-jwt-middleware v1.0.1 h1:/fsQ4vRr4zod1wKReUH+0A3ySRjGiT9G34kypO/EKwI= github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.44.159 h1:9odtuHAYQE9tQKyuX6ny1U1MHeH5/yzeCJi96g9H4DU= github.com/aws/aws-sdk-go v1.44.159/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= @@ -104,6 +118,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -130,6 +145,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc h1:PYXxkRUBGUMa5xgMVMDl62vEklZvKpVaxQeN9ie7Hfk= github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g= github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -159,12 +176,16 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/go-control-plane v0.10.3 h1:xdCVXxEe0Y3FQith+0cj2irwZudqGYvecuLB1HtdexY= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= @@ -181,6 +202,7 @@ github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEai github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -201,7 +223,9 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -239,6 +263,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -304,30 +329,63 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosimple/slug v1.10.0 h1:3XbiQua1IpCdrvuntWvGBxVm+K99wCSxJjlxkP49GGQ= github.com/gosimple/slug v1.10.0/go.mod h1:MICb3w495l9KNdZm+Xn5b6T2Hn831f9DMxiJ1r+bAjw= github.com/gosimple/unidecode v1.0.0 h1:kPdvM+qy0tnk4/BrnkrbdJ82xe88xn7c9hcaipDz4dQ= github.com/gosimple/unidecode v1.0.0/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul v1.1.1-0.20180615161029-bed22a81e9fd h1:u6o+bd6FHxDKoCSa8PJ5vrHhAYSKgJtAHQtLO1EYgos= github.com/hashicorp/consul v1.1.1-0.20180615161029-bed22a81e9fd/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/nomad/api v0.0.0-20221214074818-7dbbf6bc584d h1:kEWrUx7mld3c6HRcO2KhfD1MYBkofuZfEfDwCRQ9aMU= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hetznercloud/hcloud-go v1.38.0 h1:K6Pd/mMdcLfBhvwG39qyAaacp4pCS3dKa8gChmLKxLg= +github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= +github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -338,6 +396,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -362,11 +422,14 @@ github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexD github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs= +github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -382,12 +445,22 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= @@ -395,10 +468,23 @@ github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/ github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -410,21 +496,29 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= github.com/oklog/oklog v0.3.2 h1:wVfs8F+in6nTBMkA7CbRw+zZMIB7nNM825cM1wuzoTk= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/open-telemetry/opamp-go v0.5.0 h1:2YFbb6G4qBkq3yTRdVb5Nfz9hKHW/ldUyex352e1J7g= +github.com/open-telemetry/opamp-go v0.5.0/go.mod h1:IMdeuHGVc5CjKSu5/oNV0o+UmiXuahoHvoZ4GOmAI9M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/ovh/go-ovh v1.3.0 h1:mvZaddk4E4kLcXhzb+cxBsMPYp2pHqiQpWYkInsuZPQ= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulmach/orb v0.8.0 h1:W5XAt5yNPNnhaMNEf0xNSkBMJ1LzOzdk2MRlB6EN0Vs= github.com/paulmach/orb v0.8.0/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -434,6 +528,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f h1:h0p1aZ9F5d6IXOygysob3g4B07b+HuVUQC0VJKD8wA4= github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f/go.mod h1:oa2sAs9tGai3VldabTV0eWejt/O4/OOD7azP8GaikqU= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -441,6 +537,7 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -465,6 +562,7 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -477,7 +575,11 @@ github.com/russellhaering/gosaml2 v0.8.0/go.mod h1:byViER/1YPUa0Puj9ROZblpoq2jsE github.com/russellhaering/goxmldsig v1.2.0 h1:Y6GTTc9Un5hCxSzVz4UIWQ/zuVwDvzJk80guqzwx6Vg= github.com/russellhaering/goxmldsig v1.2.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 h1:wsfMs0iv+MJiViM37qh5VEKISi3/ZUq2nNKNdqmumAs= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= @@ -506,6 +608,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -518,8 +621,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= @@ -533,6 +636,9 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -540,6 +646,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/collector/confmap v0.70.0 h1:GJDaM7c3yFyT7Zv6l2/5ahwaqPCvtC92Ii8Bg2AVdjU= +go.opentelemetry.io/collector/confmap v0.70.0/go.mod h1:8//JWR2TMChLH35Az0mGFrCskEIP6POgZJK6iRRhzeM= +go.opentelemetry.io/collector/featuregate v0.70.0 h1:Xr6hrMT/++SjTm06nreex8WlpgFhYJ7S0yRVn1OvVf8= +go.opentelemetry.io/collector/featuregate v0.70.0/go.mod h1:ih+oCwrHW3bLac/qnPUzes28yDCDmh8WzsAKKauwCYI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK7Vyo6h0+X8BA4FpreZQTlVEIarnsBP/H5mzs= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U= go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= @@ -549,12 +659,15 @@ go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2b go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -562,6 +675,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -622,6 +736,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -645,6 +760,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -697,11 +813,15 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -709,12 +829,19 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -736,10 +863,12 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -776,6 +905,7 @@ golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -804,6 +934,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -902,6 +1033,7 @@ google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -979,9 +1111,11 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1030,6 +1164,7 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1042,6 +1177,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/segmentio/analytics-go.v3 v3.1.0 h1:UzxH1uaGZRpMKDhJyBz0pexz6yUoBU3x8bJsRk/HV6U= gopkg.in/segmentio/analytics-go.v3 v3.1.0/go.mod h1:4QqqlTlSSpVlWA9/9nDcPw+FkM2yv1NQoYjUbL9/JAw= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1078,4 +1214,5 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go new file mode 100644 index 0000000000..591a7ee799 --- /dev/null +++ b/pkg/query-service/agentConf/db.go @@ -0,0 +1,236 @@ +package agentConf + +import ( + "context" + "database/sql" + "fmt" + "math/rand" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/agentConf/sqlite" + "go.signoz.io/signoz/pkg/query-service/model" + "go.uber.org/zap" +) + +func init() { + rand.Seed(2000) +} + +// Repo handles DDL and DML ops on ingestion rules +type Repo struct { + db *sqlx.DB +} + +func (r *Repo) initDB(engine string) error { + switch engine { + case "sqlite3", "sqlite": + return sqlite.InitDB(r.db) + default: + return fmt.Errorf("unsupported db") + } +} + +func (r *Repo) GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]ConfigVersion, error) { + var c []ConfigVersion + err := r.db.SelectContext(ctx, &c, `SELECT + id, + version, + element_type, + COALESCE(created_by, -1) as created_by, + created_at, + COALESCE((SELECT NAME FROM users + WHERE id = v.created_by), "unknown") created_by_name, + active, + is_valid, + disabled, + deploy_status, + deploy_result + FROM agent_config_versions AS v + WHERE element_type = $1`, typ) + + return c, err +} + +func (r *Repo) GetConfigVersion(ctx context.Context, typ ElementTypeDef, v int) (*ConfigVersion, error) { + var c ConfigVersion + err := r.db.GetContext(ctx, &c, `SELECT + id, + version, + element_type, + COALESCE(created_by, -1) as created_by, + created_at, + COALESCE((SELECT NAME FROM users + WHERE id = v.created_by), "unknown") created_by_name, + active, + is_valid, + disabled, + deploy_status, + deploy_result, + last_hash, + last_config + FROM agent_config_versions v + WHERE element_type = $1 + AND version = $2`, typ, v) + + return &c, err + +} + +func (r *Repo) GetLatestVersion(ctx context.Context, typ ElementTypeDef) (*ConfigVersion, error) { + var c ConfigVersion + err := r.db.GetContext(ctx, &c, `SELECT + id, + version, + element_type, + COALESCE(created_by, -1) as created_by, + created_at, + COALESCE((SELECT NAME FROM users + WHERE id = v.created_by), "unknown") created_by_name, + active, + is_valid, + disabled, + deploy_status, + deploy_result + FROM agent_config_versions AS v + WHERE element_type = $1 + AND version = ( + SELECT MAX(version) + FROM agent_config_versions + WHERE element_type=$2)`, typ, typ) + if err != nil { + zap.S().Error("failed get latest config version for element:", typ, err) + } + return &c, err +} + +func (r *Repo) insertConfig(ctx context.Context, userId string, c *ConfigVersion, elements []string) (fnerr error) { + + if string(c.ElementType) == "" { + return fmt.Errorf("element type is required for creating agent config version") + } + + if len(elements) == 0 { + zap.S().Error("insert config called with no elements", c.ElementType) + return fmt.Errorf("config must have atleast one element") + } + + if c.Version != 0 { + // the version can not be set by the user, we want to auto-assign the versions + // in a monotonically increasing order starting with 1. hence, we reject insert + // requests with version anything other than 0. here, 0 indicates un-assigned + zap.S().Error("invalid version assignment while inserting agent config", c.Version, c.ElementType) + return fmt.Errorf("user defined versions are not supported in the agent config") + } + + configVersion, err := r.GetLatestVersion(ctx, c.ElementType) + if err != nil { + if err != sql.ErrNoRows { + zap.S().Error("failed to fetch latest config version", err) + return fmt.Errorf("failed to fetch latest config version") + } + } + + c.Version = updateVersion(configVersion.Version) + + defer func() { + if fnerr != nil { + // remove all the damage (invalid rows from db) + r.db.Exec("DELETE FROM agent_config_versions WHERE id = $1", c.ID) + r.db.Exec("DELETE FROM agent_config_elements WHERE version_id=$1", c.ID) + } + }() + + // insert config + configQuery := `INSERT INTO agent_config_versions( + id, + version, + created_by, + element_type, + active, + is_valid, + disabled, + deploy_status, + deploy_result) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + + _, err = r.db.ExecContext(ctx, + configQuery, + c.ID, + c.Version, + userId, + c.ElementType, + false, + false, + false, + c.DeployStatus, + c.DeployResult) + + if err != nil { + zap.S().Error("error in inserting config version: ", zap.Error(err)) + return fmt.Errorf("failed to insert ingestion rule") + } + + elementsQuery := `INSERT INTO agent_config_elements( + id, + version_id, + element_type, + element_id) + VALUES ($1, $2, $3, $4)` + + for _, e := range elements { + + _, err = r.db.ExecContext(ctx, + elementsQuery, + uuid.NewString(), + c.ID, + c.ElementType, + e) + if err != nil { + return err + } + } + + return nil +} + +func (r *Repo) updateDeployStatus(ctx context.Context, + elementType ElementTypeDef, + version int, + status string, + result string, + lastHash string, + lastconf string) error { + + updateQuery := `UPDATE agent_config_versions + set deploy_status = $1, + deploy_result = $2, + last_hash = COALESCE($3, last_hash), + last_config = $4 + WHERE version=$5 + AND element_type = $6` + + _, err := r.db.ExecContext(ctx, updateQuery, status, result, lastHash, lastconf, version, string(elementType)) + if err != nil { + zap.S().Error("failed to update deploy status", err) + return model.BadRequestStr("failed to update deploy status") + } + + return nil +} + +func (r *Repo) updateDeployStatusByHash(ctx context.Context, confighash string, status string, result string) error { + + updateQuery := `UPDATE agent_config_versions + set deploy_status = $1, + deploy_result = $2 + WHERE last_hash=$4` + + _, err := r.db.ExecContext(ctx, updateQuery, status, result, confighash) + if err != nil { + zap.S().Error("failed to update deploy status", err) + return model.BadRequestStr("failed to update deploy status") + } + + return nil +} diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go new file mode 100644 index 0000000000..27478b332a --- /dev/null +++ b/pkg/query-service/agentConf/manager.go @@ -0,0 +1,212 @@ +package agentConf + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/app/opamp" + filterprocessor "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/filterprocessor" + tsp "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/tailsampler" + "go.uber.org/zap" + yaml "gopkg.in/yaml.v3" +) + +var m *Manager + +func init() { + m = &Manager{} +} + +type Manager struct { + Repo + // lock to make sure only one update is sent to remote agents at a time + lock uint32 +} + +// Ready indicates if Manager can accept new config update requests +func (mgr *Manager) Ready() bool { + if atomic.LoadUint32(&mgr.lock) != 0 { + return false + } + return opamp.Ready() +} + +func Initiate(db *sqlx.DB, engine string) error { + m.Repo = Repo{db} + return m.initDB(engine) +} + +// Ready indicates if Manager can accept new config update requests +func Ready() bool { + return m.Ready() +} + +func GetLatestVersion(ctx context.Context, elementType ElementTypeDef) (*ConfigVersion, error) { + return m.GetLatestVersion(ctx, elementType) +} + +func GetConfigVersion(ctx context.Context, elementType ElementTypeDef, version int) (*ConfigVersion, error) { + return m.GetConfigVersion(ctx, elementType, version) +} + +func GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]ConfigVersion, error) { + return m.GetConfigHistory(ctx, typ) +} + +// StartNewVersion launches a new config version for given set of elements +func StartNewVersion(ctx context.Context, userId string, eleType ElementTypeDef, elementIds []string) (*ConfigVersion, error) { + + if !m.Ready() { + // agent is already being updated, ask caller to wait and re-try after sometime + return nil, fmt.Errorf("agent updater is busy") + } + + // create a new version + cfg := NewConfigversion(eleType) + + // insert new config and elements into database + err := m.insertConfig(ctx, userId, cfg, elementIds) + if err != nil { + return nil, err + } + + return cfg, nil +} + +func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { + + configVersion, err := GetConfigVersion(ctx, typ, version) + if err != nil { + zap.S().Debug("failed to fetch config version during redeploy", err) + return fmt.Errorf("failed to fetch details of the config version") + } + + if configVersion == nil || (configVersion != nil && configVersion.LastConf == "") { + zap.S().Debug("config version has no conf yaml", configVersion) + return fmt.Errorf("the config version can not be redeployed") + } + switch typ { + case ElementTypeSamplingRules: + var config *tsp.Config + if err := yaml.Unmarshal([]byte(configVersion.LastConf), &config); err != nil { + zap.S().Error("failed to read last conf correctly", err) + return fmt.Errorf("failed to read the stored config correctly") + } + + // merge current config with new filter params + processorConf := map[string]interface{}{ + "signoz_tail_sampling": config, + } + + opamp.AddToTracePipelineSpec("signoz_tail_sampling") + configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Error("failed to call agent config update for trace processor:", err) + return fmt.Errorf("failed to deploy the config") + } + + m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf) + case ElementTypeDropRules: + var filterConfig *filterprocessor.Config + if err := yaml.Unmarshal([]byte(configVersion.LastConf), &filterConfig); err != nil { + zap.S().Error("failed to read last conf correctly", err) + return fmt.Errorf("failed to read the stored config correctly") + } + processorConf := map[string]interface{}{ + "filter": filterConfig, + } + + opamp.AddToMetricsPipelineSpec("filter") + configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Error("failed to call agent config update for trace processor:", err) + return err + } + + m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf) + } + + return nil +} + +// UpsertFilterProcessor updates the agent config with new filter processor params +func UpsertFilterProcessor(ctx context.Context, version int, config *filterprocessor.Config) error { + if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { + return fmt.Errorf("agent updater is busy") + } + defer atomic.StoreUint32(&m.lock, 0) + + // merge current config with new filter params + // merge current config with new filter params + processorConf := map[string]interface{}{ + "filter": config, + } + + opamp.AddToMetricsPipelineSpec("filter") + configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Error("failed to call agent config update for trace processor:", err) + return err + } + + processorConfYaml, err := yaml.Marshal(config) + if err != nil { + zap.S().Warnf("unexpected error while transforming processor config to yaml", err) + } + + m.updateDeployStatus(ctx, ElementTypeDropRules, version, string(DeployInitiated), "Deployment started", configHash, string(processorConfYaml)) + return nil +} + +// OnConfigUpdate is a callback function passed to opamp server. +// It receives a config hash with error status. We assume +// successful deployment if no error is received. +// this method is currently expected to be called only once in the lifecycle +// but can be improved in future to accept continuous request status updates from opamp +func (m *Manager) OnConfigUpdate(agentId string, hash string, err error) { + + status := string(Deployed) + + message := "deploy successful" + + defer func() { + zap.S().Info(status, zap.String("agentId", agentId), zap.String("agentResponse", message)) + }() + + if err != nil { + status = string(DeployFailed) + message = fmt.Sprintf("%s: %s", agentId, err.Error()) + } + + m.updateDeployStatusByHash(context.Background(), hash, status, message) +} + +// UpsertSamplingProcessor updates the agent config with new filter processor params +func UpsertSamplingProcessor(ctx context.Context, version int, config *tsp.Config) error { + if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { + return fmt.Errorf("agent updater is busy") + } + defer atomic.StoreUint32(&m.lock, 0) + + // merge current config with new filter params + processorConf := map[string]interface{}{ + "signoz_tail_sampling": config, + } + + opamp.AddToTracePipelineSpec("signoz_tail_sampling") + configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Error("failed to call agent config update for trace processor:", err) + return err + } + + processorConfYaml, err := yaml.Marshal(config) + if err != nil { + zap.S().Warnf("unexpected error while transforming processor config to yaml", err) + } + + m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, string(processorConfYaml)) + return nil +} diff --git a/pkg/query-service/agentConf/sqlite/init.go b/pkg/query-service/agentConf/sqlite/init.go new file mode 100644 index 0000000000..b844fc6a62 --- /dev/null +++ b/pkg/query-service/agentConf/sqlite/init.go @@ -0,0 +1,65 @@ +package sqlite + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/jmoiron/sqlx" +) + +func InitDB(db *sqlx.DB) error { + var err error + if db == nil { + return fmt.Errorf("invalid db connection") + } + + table_schema := `CREATE TABLE IF NOT EXISTS agent_config_versions( + id TEXT PRIMARY KEY, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_by TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + version INTEGER DEFAULT 1, + active int, + is_valid int, + disabled int, + element_type VARCHAR(120) NOT NULL, + deploy_status VARCHAR(80) NOT NULL DEFAULT 'DIRTY', + deploy_sequence INTEGER, + deploy_result TEXT, + last_hash TEXT, + last_config TEXT, + UNIQUE(element_type, version) + ); + + + CREATE UNIQUE INDEX IF NOT EXISTS agent_config_versions_u1 + ON agent_config_versions(element_type, version); + + CREATE INDEX IF NOT EXISTS agent_config_versions_nu1 + ON agent_config_versions(last_hash); + + + CREATE TABLE IF NOT EXISTS agent_config_elements( + id TEXT PRIMARY KEY, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_by TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + element_id TEXT NOT NULL, + element_type VARCHAR(120) NOT NULL, + version_id TEXT NOT NULL + ); + + CREATE UNIQUE INDEX IF NOT EXISTS agent_config_elements_u1 + ON agent_config_elements(version_id, element_id, element_type); + + ` + + _, err = db.Exec(table_schema) + if err != nil { + return errors.Wrap(err, "Error in creating agent config tables") + } + return nil +} diff --git a/pkg/query-service/agentConf/version.go b/pkg/query-service/agentConf/version.go new file mode 100644 index 0000000000..c6b8c01ee4 --- /dev/null +++ b/pkg/query-service/agentConf/version.go @@ -0,0 +1,69 @@ +package agentConf + +import ( + "time" + + "github.com/google/uuid" +) + +type ElementTypeDef string + +const ( + ElementTypeSamplingRules ElementTypeDef = "sampling_rules" + ElementTypeDropRules ElementTypeDef = "drop_rules" + ElementTypeLogPipelines ElementTypeDef = "log_pipelines" + ElementTypeLbExporter ElementTypeDef = "lb_exporter" +) + +type DeployStatus string + +const ( + PendingDeploy DeployStatus = "DIRTY" + Deploying DeployStatus = "DEPLOYING" + Deployed DeployStatus = "DEPLOYED" + DeployInitiated DeployStatus = "IN_PROGRESS" + DeployFailed DeployStatus = "FAILED" +) + +type ConfigVersion struct { + ID string `json:"id" db:"id"` + Version int `json:"version" db:"version"` + ElementType ElementTypeDef `json:"elementType" db:"element_type"` + Active bool `json:"active" db:"active"` + IsValid bool `json:"is_valid" db:"is_valid"` + Disabled bool `json:"disabled" db:"disabled"` + + DeployStatus DeployStatus `json:"deployStatus" db:"deploy_status"` + DeployResult string `json:"deployResult" db:"deploy_result"` + + LastHash string `json:"lastHash" db:"last_hash"` + LastConf string `json:"lastConf" db:"last_config"` + + CreatedBy string `json:"createdBy" db:"created_by"` + CreatedByName string `json:"createdByName" db:"created_by_name"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` +} + +func NewConfigversion(typeDef ElementTypeDef) *ConfigVersion { + return &ConfigVersion{ + ID: uuid.NewString(), + ElementType: typeDef, + Active: false, + IsValid: false, + Disabled: false, + DeployStatus: PendingDeploy, + // todo: get user id from context? + // CreatedBy + } +} + +func updateVersion(v int) int { + return v + 1 +} + +type ConfigElements struct { + VersionID string + Version int + ElementType ElementTypeDef + ElementId string +} diff --git a/pkg/query-service/app/opamp/config.yaml b/pkg/query-service/app/opamp/config.yaml new file mode 100644 index 0000000000..d5ef74e00f --- /dev/null +++ b/pkg/query-service/app/opamp/config.yaml @@ -0,0 +1,76 @@ +receivers: + otlp/spanmetrics: + protocols: + grpc: + endpoint: "localhost:12345" + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: + hostmetrics: + collection_interval: 30s + scrapers: + cpu: + load: + memory: + disk: + filesystem: + network: +processors: + batch: + send_batch_size: 1000 + timeout: 10s + signozspanmetrics/prometheus: + metrics_exporter: prometheus + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + dimensions_cache_size: 10000 + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + # memory_limiter: + # # 80% of maximum memory up to 2G + # limit_mib: 1500 + # # 25% of limit up to 2G + # spike_limit_mib: 512 + # check_interval: 5s + # + # # 50% of the maximum memory + # limit_percentage: 50 + # # 20% of max memory usage spike expected + # spike_limit_percentage: 20 + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + zpages: {} +exporters: + clickhousetraces: + datasource: tcp://localhost:9000/?database=signoz_traces + migrations: exporter/clickhousetracesexporter/migrations + clickhousemetricswrite: + endpoint: tcp://localhost:9000/?database=signoz_metrics + resource_to_telemetry_conversion: + enabled: true + prometheus: + endpoint: "0.0.0.0:8889" +service: + extensions: [zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhousetraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] + metrics/spanmetrics: + receivers: [otlp/spanmetrics] + exporters: [prometheus] \ No newline at end of file diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go new file mode 100644 index 0000000000..af04abb723 --- /dev/null +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -0,0 +1,141 @@ +package opamp + +import ( + "context" + "crypto/sha256" + "fmt" + + "github.com/knadh/koanf/parsers/yaml" + "github.com/open-telemetry/opamp-go/protobufs" + "go.opentelemetry.io/collector/confmap" + model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" + "go.uber.org/zap" +) + +// inserts or updates ingestion controller processors depending +// on the signal (metrics or traces) +func UpsertControlProcessors(ctx context.Context, signal string, processors map[string]interface{}, callback model.OnChangeCallback) (hash string, fnerr error) { + // note: only processors enabled through tracesPipelinePlan will be added + // to pipeline. To enable or disable processors from pipeline, call + // AddToTracePipeline() or RemoveFromTracesPipeline() prior to calling + // this method + + zap.S().Debug("initiating ingestion rules deployment config", signal, processors) + + if signal != string(Metrics) && signal != string(Traces) { + zap.S().Error("received invalid signal int UpsertControlProcessors", signal) + fnerr = fmt.Errorf("signal not supported in ingestion rules: %s", signal) + return + } + + if opAmpServer == nil { + fnerr = fmt.Errorf("opamp server is down, unable to push config to agent at this moment") + return + } + + agents := opAmpServer.agents.GetAllAgents() + if len(agents) == 0 { + fnerr = fmt.Errorf("no agents available at the moment") + return + } + + if len(agents) > 1 && signal == string(Traces) { + zap.S().Debug("found multiple agents. this feature is not supported for traces pipeline (sampling rules)") + fnerr = fmt.Errorf("multiple agents not supported in sampling rules") + return + } + + for _, agent := range agents { + + agenthash, err := addIngestionControlToAgent(agent, signal, processors, false) + if err != nil { + zap.S().Error("failed to push ingestion rules config to agent", agent.ID, err) + continue + } + + if agenthash != "" { + // subscribe callback + model.ListenToConfigUpdate(agent.ID, agenthash, callback) + } + + hash = agenthash + } + + return hash, nil +} + +// addIngestionControlToAgent adds ingestion contorl rules to agent config +func addIngestionControlToAgent(agent *model.Agent, signal string, processors map[string]interface{}, withLB bool) (string, error) { + confHash := "" + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + if err != nil { + return confHash, err + } + + agentConf := confmap.NewFromStringMap(c) + + // add ingestion control spec + err = makeIngestionControlSpec(agentConf, Signal(signal), processors) + if err != nil { + zap.S().Error("failed to prepare ingestion control processors for agent ", agent.ID, err) + return confHash, err + } + + // ------ complete adding processor + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + if err != nil { + return confHash, err + } + + zap.S().Debugf("sending new config", string(configR)) + hash := sha256.New() + _, err = hash.Write(configR) + if err != nil { + return confHash, err + } + confHash = string(hash.Sum(nil)) + agent.EffectiveConfig = string(configR) + err = agent.Upsert() + if err != nil { + return confHash, err + } + + agent.SendToAgent(&protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: configR, + ContentType: "application/x-yaml", + }, + }, + }, + ConfigHash: []byte(confHash), + }, + }) + + return string(confHash), nil +} + +// prepare spec to introduce ingestion control in agent conf +func makeIngestionControlSpec(agentConf *confmap.Conf, signal Signal, processors map[string]interface{}) error { + configParser := otelconfig.NewConfigParser(agentConf) + configParser.UpdateProcessors(processors) + + // edit pipeline if processor is missing + currentPipeline := configParser.PipelineProcessors(string(signal)) + + // merge tracesPipelinePlan with current pipeline + mergedPipeline, err := buildPipeline(signal, currentPipeline) + if err != nil { + zap.S().Error("failed to build pipeline", signal, err) + return err + } + + // add merged pipeline to the service + configParser.UpdateProcsInPipeline(string(signal), mergedPipeline) + + return nil +} diff --git a/pkg/query-service/app/opamp/logger.go b/pkg/query-service/app/opamp/logger.go new file mode 100644 index 0000000000..49f9e630fc --- /dev/null +++ b/pkg/query-service/app/opamp/logger.go @@ -0,0 +1,15 @@ +package opamp + +import "log" + +type Logger struct { + logger *log.Logger +} + +func (l *Logger) Debugf(format string, v ...interface{}) { + l.logger.Printf(format, v...) +} + +func (l *Logger) Errorf(format string, v ...interface{}) { + l.logger.Printf(format, v...) +} diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go new file mode 100644 index 0000000000..ba2ecfcddc --- /dev/null +++ b/pkg/query-service/app/opamp/model/agent.go @@ -0,0 +1,334 @@ +package model + +import ( + "bytes" + "context" + "crypto/sha256" + "sync" + "time" + + "google.golang.org/protobuf/proto" + + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/open-telemetry/opamp-go/server/types" +) + +type AgentStatus int + +const ( + AgentStatusUnknown AgentStatus = iota + AgentStatusConnected + AgentStatusDisconnected +) + +// set in agent description when agent is capable of supporting +// lb exporter configuration. values: 1 (true) or 0 (false) +const lbExporterFlag = "capabilities.lbexporter" + +type Agent struct { + ID string `json:"agentId" yaml:"agentId" db:"agent_id"` + StartedAt time.Time `json:"startedAt" yaml:"startedAt" db:"started_at"` + TerminatedAt time.Time `json:"terminatedAt" yaml:"terminatedAt" db:"terminated_at"` + EffectiveConfig string `json:"effectiveConfig" yaml:"effectiveConfig" db:"effective_config"` + CurrentStatus AgentStatus `json:"currentStatus" yaml:"currentStatus" db:"current_status"` + remoteConfig *protobufs.AgentRemoteConfig + Status *protobufs.AgentToServer + + // can this agent be load balancer + CanLB bool + + // is this agent setup as load balancer + IsLb bool + + conn types.Connection + connMutex sync.Mutex + mux sync.RWMutex +} + +func New(ID string, conn types.Connection) *Agent { + return &Agent{ID: ID, StartedAt: time.Now(), CurrentStatus: AgentStatusConnected, conn: conn} +} + +// Upsert inserts or updates the agent in the database. +func (agent *Agent) Upsert() error { + agent.mux.Lock() + defer agent.mux.Unlock() + + _, err := db.NamedExec(`INSERT OR REPLACE INTO agents ( + agent_id, + started_at, + effective_config, + current_status + ) VALUES ( + :agent_id, + :started_at, + :effective_config, + :current_status + )`, agent) + if err != nil { + return err + } + + return nil +} + +func (agent *Agent) UpdateStatus(statusMsg *protobufs.AgentToServer, response *protobufs.ServerToAgent) { + agent.mux.Lock() + defer agent.mux.Unlock() + agent.processStatusUpdate(statusMsg, response) +} + +// extracts lb exporter support flag from agent description. the flag +// is used to decide if lb exporter can be enabled on the agent. +func ExtractLbFlag(agentDescr *protobufs.AgentDescription) bool { + + if agentDescr == nil { + return false + } + + if len(agentDescr.NonIdentifyingAttributes) > 0 { + for _, kv := range agentDescr.NonIdentifyingAttributes { + anyvalue, ok := kv.Value.Value.(*protobufs.AnyValue_StringValue) + if !ok { + continue + } + if kv.Key == lbExporterFlag && anyvalue.StringValue == "1" { + // agent can support load balancer config + return true + } + } + } + return false +} + +func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) (agentDescrChanged bool) { + prevStatus := agent.Status + + if agent.Status == nil { + // First time this Agent reports a status, remember it. + agent.Status = newStatus + agentDescrChanged = true + } else { + // Not a new Agent. Update the Status. + agent.Status.SequenceNum = newStatus.SequenceNum + + // Check what's changed in the AgentDescription. + if newStatus.AgentDescription != nil { + // If the AgentDescription field is set it means the Agent tells us + // something is changed in the field since the last status report + // (or this is the first report). + // Make full comparison of previous and new descriptions to see if it + // really is different. + if prevStatus != nil && proto.Equal(prevStatus.AgentDescription, newStatus.AgentDescription) { + // Agent description didn't change. + agentDescrChanged = false + } else { + // Yes, the description is different, update it. + agent.Status.AgentDescription = newStatus.AgentDescription + agentDescrChanged = true + } + } else { + // AgentDescription field is not set, which means description didn't change. + agentDescrChanged = false + } + + // Update remote config status if it is included and is different from what we have. + if newStatus.RemoteConfigStatus != nil && + !proto.Equal(agent.Status.RemoteConfigStatus, newStatus.RemoteConfigStatus) { + agent.Status.RemoteConfigStatus = newStatus.RemoteConfigStatus + + // todo: need to address multiple agent scenario here + // for now, the first response will be sent back to the UI + if agent.Status.RemoteConfigStatus.Status == protobufs.RemoteConfigStatuses_RemoteConfigStatuses_APPLIED { + onConfigSuccess(agent.ID, string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash)) + } + + if agent.Status.RemoteConfigStatus.Status == protobufs.RemoteConfigStatuses_RemoteConfigStatuses_FAILED { + onConfigFailure(agent.ID, string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash), agent.Status.RemoteConfigStatus.ErrorMessage) + } + } + } + + if agentDescrChanged { + agent.CanLB = ExtractLbFlag(newStatus.AgentDescription) + } + + return agentDescrChanged +} + +func (agent *Agent) updateHealth(newStatus *protobufs.AgentToServer) { + if newStatus.Health == nil { + return + } + + agent.Status.Health = newStatus.Health + + if agent.Status != nil && agent.Status.Health != nil && agent.Status.Health.Healthy { + agent.StartedAt = time.Unix(0, int64(agent.Status.Health.StartTimeUnixNano)).UTC() + } +} + +func (agent *Agent) updateRemoteConfigStatus(newStatus *protobufs.AgentToServer) { + // Update remote config status if it is included and is different from what we have. + if newStatus.RemoteConfigStatus != nil { + agent.Status.RemoteConfigStatus = newStatus.RemoteConfigStatus + } +} + +func (agent *Agent) updateStatusField(newStatus *protobufs.AgentToServer) (agentDescrChanged bool) { + if agent.Status == nil { + // First time this Agent reports a status, remember it. + agent.Status = newStatus + agentDescrChanged = true + } + + agentDescrChanged = agent.updateAgentDescription(newStatus) || agentDescrChanged + agent.updateRemoteConfigStatus(newStatus) + agent.updateHealth(newStatus) + return agentDescrChanged +} + +func (agent *Agent) updateEffectiveConfig(newStatus *protobufs.AgentToServer, response *protobufs.ServerToAgent) { + // Update effective config if provided. + if newStatus.EffectiveConfig != nil { + if newStatus.EffectiveConfig.ConfigMap != nil { + agent.Status.EffectiveConfig = newStatus.EffectiveConfig + + // Convert to string for displaying purposes. + agent.EffectiveConfig = "" + // There should be only one config in the map. + for _, cfg := range newStatus.EffectiveConfig.ConfigMap.ConfigMap { + agent.EffectiveConfig = string(cfg.Body) + } + } + } +} + +func (agent *Agent) hasCapability(capability protobufs.AgentCapabilities) bool { + return agent.Status.Capabilities&uint64(capability) != 0 +} + +func (agent *Agent) processStatusUpdate( + newStatus *protobufs.AgentToServer, + response *protobufs.ServerToAgent, +) { + // We don't have any status for this Agent, or we lost the previous status update from the Agent, so our + // current status is not up-to-date. + lostPreviousUpdate := (agent.Status == nil) || (agent.Status != nil && agent.Status.SequenceNum+1 != newStatus.SequenceNum) + + agentDescrChanged := agent.updateStatusField(newStatus) + + // Check if any fields were omitted in the status report. + effectiveConfigOmitted := newStatus.EffectiveConfig == nil && + agent.hasCapability(protobufs.AgentCapabilities_AgentCapabilities_ReportsEffectiveConfig) + + remoteConfigStatusOmitted := newStatus.RemoteConfigStatus == nil && + agent.hasCapability(protobufs.AgentCapabilities_AgentCapabilities_ReportsRemoteConfig) + + healthOmitted := newStatus.Health == nil && + agent.hasCapability(protobufs.AgentCapabilities_AgentCapabilities_ReportsHealth) + + // True if the status was not fully reported. + statusIsCompressed := effectiveConfigOmitted || remoteConfigStatusOmitted || healthOmitted + + if statusIsCompressed && lostPreviousUpdate { + // The status message is not fully set in the message that we received, but we lost the previous + // status update. Request full status update from the agent. + response.Flags |= uint64(protobufs.ServerToAgentFlags_ServerToAgentFlags_ReportFullState) + } + + configChanged := false + if agentDescrChanged { + // Agent description is changed. + + // We need to recalculate the config. + configChanged = agent.updateRemoteConfig() + } + + // If remote config is changed and different from what the Agent has then + // send the new remote config to the Agent. + if configChanged || + (agent.Status.RemoteConfigStatus != nil && + bytes.Compare(agent.Status.RemoteConfigStatus.LastRemoteConfigHash, agent.remoteConfig.ConfigHash) != 0) { + // The new status resulted in a change in the config of the Agent or the Agent + // does not have this config (hash is different). Send the new config the Agent. + response.RemoteConfig = agent.remoteConfig + agent.SendToAgent(response) + } + + agent.updateEffectiveConfig(newStatus, response) +} + +func (agent *Agent) updateRemoteConfig() bool { + hash := sha256.New() + + cfg := protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{}, + }, + } + + // Calculate the hash. + for k, v := range cfg.Config.ConfigMap { + hash.Write([]byte(k)) + hash.Write(v.Body) + hash.Write([]byte(v.ContentType)) + } + + cfg.ConfigHash = hash.Sum(nil) + + configChanged := !isEqualRemoteConfig(agent.remoteConfig, &cfg) + + agent.remoteConfig = &cfg + + return configChanged +} + +func isEqualRemoteConfig(c1, c2 *protobufs.AgentRemoteConfig) bool { + if c1 == c2 { + return true + } + if c1 == nil || c2 == nil { + return false + } + return isEqualConfigSet(c1.Config, c2.Config) +} + +func isEqualConfigSet(c1, c2 *protobufs.AgentConfigMap) bool { + if c1 == c2 { + return true + } + if c1 == nil || c2 == nil { + return false + } + if len(c1.ConfigMap) != len(c2.ConfigMap) { + return false + } + for k, v1 := range c1.ConfigMap { + v2, ok := c2.ConfigMap[k] + if !ok { + return false + } + if !isEqualConfigFile(v1, v2) { + return false + } + } + return true +} + +func isEqualConfigFile(f1, f2 *protobufs.AgentConfigFile) bool { + if f1 == f2 { + return true + } + if f1 == nil || f2 == nil { + return false + } + return bytes.Compare(f1.Body, f2.Body) == 0 && f1.ContentType == f2.ContentType +} + +func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { + agent.connMutex.Lock() + defer agent.connMutex.Unlock() + + agent.conn.Send(context.Background(), msg) +} diff --git a/pkg/query-service/app/opamp/model/agents.go b/pkg/query-service/app/opamp/model/agents.go new file mode 100644 index 0000000000..18faddb48b --- /dev/null +++ b/pkg/query-service/app/opamp/model/agents.go @@ -0,0 +1,117 @@ +package model + +import ( + "fmt" + "sync" + "time" + + "github.com/jmoiron/sqlx" + "github.com/open-telemetry/opamp-go/server/types" +) + +var db *sqlx.DB + +var AllAgents = Agents{ + agentsById: map[string]*Agent{}, + connections: map[types.Connection]map[string]bool{}, +} + +type Agents struct { + mux sync.RWMutex + agentsById map[string]*Agent + connections map[types.Connection]map[string]bool +} + +func (a *Agents) Count() int { + return len(a.connections) +} + +// InitDB initializes the database and creates the agents table. +func InitDB(dataSourceName string) (*sqlx.DB, error) { + var err error + + db, err = sqlx.Open("sqlite3", dataSourceName) + if err != nil { + return nil, err + } + + tableSchema := `CREATE TABLE IF NOT EXISTS agents ( + agent_id TEXT PRIMARY KEY UNIQUE, + started_at datetime NOT NULL, + terminated_at datetime, + current_status TEXT NOT NULL, + effective_config TEXT NOT NULL + );` + + _, err = db.Exec(tableSchema) + if err != nil { + return nil, fmt.Errorf("Error in creating agents table: %s", err.Error()) + } + + AllAgents = Agents{ + agentsById: make(map[string]*Agent), + connections: make(map[types.Connection]map[string]bool), + mux: sync.RWMutex{}, + } + return db, nil +} + +// RemoveConnection removes the connection all Agent instances associated with the +// connection. +func (agents *Agents) RemoveConnection(conn types.Connection) { + agents.mux.Lock() + defer agents.mux.Unlock() + + for instanceId := range agents.connections[conn] { + agent := agents.agentsById[instanceId] + agent.CurrentStatus = AgentStatusDisconnected + agent.TerminatedAt = time.Now() + agent.Upsert() + delete(agents.agentsById, instanceId) + } + delete(agents.connections, conn) +} + +// FindAgent returns the Agent instance associated with the given agentID. +func (agents *Agents) FindAgent(agentID string) *Agent { + agents.mux.RLock() + defer agents.mux.RUnlock() + return agents.agentsById[agentID] +} + +// FindOrCreateAgent returns the Agent instance associated with the given agentID. +// If the Agent instance does not exist, it is created and added to the list of +// Agent instances. +func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection) (*Agent, bool, error) { + agents.mux.Lock() + defer agents.mux.Unlock() + var created bool + agent, ok := agents.agentsById[agentID] + var err error + if !ok || agent == nil { + agent = New(agentID, conn) + err = agent.Upsert() + if err != nil { + return nil, created, err + } + agents.agentsById[agentID] = agent + + if agents.connections[conn] == nil { + agents.connections[conn] = map[string]bool{} + } + agents.connections[conn][agentID] = true + created = true + } + return agent, created, nil +} + +func (agents *Agents) GetAllAgents() []*Agent { + agents.mux.RLock() + defer agents.mux.RUnlock() + + allAgents := []*Agent{} + for _, v := range agents.agentsById { + allAgents = append(allAgents, v) + } + return allAgents +} diff --git a/pkg/query-service/app/opamp/model/coordinator.go b/pkg/query-service/app/opamp/model/coordinator.go new file mode 100644 index 0000000000..a1f17f43a2 --- /dev/null +++ b/pkg/query-service/app/opamp/model/coordinator.go @@ -0,0 +1,66 @@ +package model + +import ( + "fmt" + "sync" +) + +// communicates with calling apis when config is applied or fails +var coordinator *Coordinator + +func init() { + subscribers := make(map[string][]OnChangeCallback, 0) + coordinator = &Coordinator{ + subscribers: subscribers, + } +} + +type OnChangeCallback func(agentId string, hash string, err error) + +// responsible for managing subscribers on config change +type Coordinator struct { + mutex sync.Mutex + + // hash wise list of subscribers + subscribers map[string][]OnChangeCallback +} + +func onConfigSuccess(agentId string, hash string) { + notifySubscribers(agentId, hash, nil) +} + +func onConfigFailure(agentId string, hash string, errorMessage string) { + notifySubscribers(agentId, hash, fmt.Errorf(errorMessage)) +} + +// OnSuccess listens to config changes and notifies subscribers +func notifySubscribers(agentId string, hash string, err error) { + // this method currently does not handle multi-agent scenario. + // as soon as a message is delivered, we release all the subscribers + // for a given hash + subs, ok := coordinator.subscribers[hash] + if !ok { + return + } + + for _, s := range subs { + s(agentId, hash, err) + } + + // delete all subscribers for this hash, assume future + // notifies will be disabled. the first response is processed + delete(coordinator.subscribers, hash) +} + +// callers subscribe to this function to listen on config change requests +func ListenToConfigUpdate(agentId string, hash string, ss OnChangeCallback) { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + if subs, ok := coordinator.subscribers[hash]; ok { + subs = append(subs, ss) + coordinator.subscribers[hash] = subs + } else { + coordinator.subscribers[hash] = []OnChangeCallback{ss} + } +} diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go new file mode 100644 index 0000000000..237b07f121 --- /dev/null +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -0,0 +1,204 @@ +package opamp + +import ( + "context" + "crypto/sha256" + + "strings" + + "github.com/knadh/koanf/parsers/yaml" + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/open-telemetry/opamp-go/server" + "github.com/open-telemetry/opamp-go/server/types" + "go.opentelemetry.io/collector/confmap" + model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + + "go.uber.org/zap" +) + +var opAmpServer *Server + +type Server struct { + server server.OpAMPServer + agents *model.Agents + logger *zap.Logger + capabilities int32 +} + +const capabilities = protobufs.ServerCapabilities_ServerCapabilities_AcceptsEffectiveConfig | + protobufs.ServerCapabilities_ServerCapabilities_OffersRemoteConfig | + protobufs.ServerCapabilities_ServerCapabilities_AcceptsStatus + +func InitalizeServer(listener string, agents *model.Agents) error { + + if agents == nil { + agents = &model.AllAgents + } + + opAmpServer = &Server{ + agents: agents, + } + opAmpServer.server = server.New(zap.S()) + + return opAmpServer.Start(listener) +} + +func StopServer() { + if opAmpServer != nil { + opAmpServer.Stop() + } +} + +func (srv *Server) Start(listener string) error { + settings := server.StartSettings{ + Settings: server.Settings{ + Callbacks: server.CallbacksStruct{ + OnMessageFunc: srv.onMessage, + OnConnectionCloseFunc: srv.onDisconnect, + }, + }, + ListenEndpoint: listener, + } + + return srv.server.Start(settings) +} + +func (srv *Server) Stop() { + srv.server.Stop(context.Background()) +} + +func (srv *Server) onDisconnect(conn types.Connection) { + srv.agents.RemoveConnection(conn) +} + +func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { + agentID := msg.InstanceUid + + agent, created, err := srv.agents.FindOrCreateAgent(agentID, conn) + if err != nil { + zap.S().Error("Failed to find or create agent %q: %v", agentID, err) + // TODO: handle error + } + + if created { + agent.CanLB = model.ExtractLbFlag(msg.AgentDescription) + zap.S().Debugf("New agent added:", zap.Bool("canLb", agent.CanLB), zap.String("ID", agent.ID), zap.Any("status", agent.CurrentStatus)) + } + + var response *protobufs.ServerToAgent + response = &protobufs.ServerToAgent{ + InstanceUid: agentID, + Capabilities: uint64(capabilities), + } + + agent.UpdateStatus(msg, response) + + return response +} + +// global var methods to support singleton pattern. we want to discourage +// allow multiple servers in one installation +func Ready() bool { + if opAmpServer == nil { + return false + } + if opAmpServer.agents.Count() == 0 { + zap.S().Warnf("no agents available, all agent config requests will be rejected") + return false + } + return true +} + +func Subscribe(agentId string, hash string, f model.OnChangeCallback) { + model.ListenToConfigUpdate(agentId, hash, f) +} + +func UpsertProcessor(ctx context.Context, processors map[string]interface{}, names []interface{}) error { + x := map[string]interface{}{ + "processors": processors, + } + + newConf := confmap.NewFromStringMap(x) + + agents := opAmpServer.agents.GetAllAgents() + for _, agent := range agents { + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + if err != nil { + return err + } + agentConf := confmap.NewFromStringMap(c) + + err = agentConf.Merge(newConf) + if err != nil { + return err + } + + service := agentConf.Get("service") + + logs := service.(map[string]interface{})["pipelines"].(map[string]interface{})["logs"] + processors := logs.(map[string]interface{})["processors"].([]interface{}) + userProcessors := []interface{}{} + // remove old ones + for _, v := range processors { + if !strings.HasPrefix(v.(string), "logstransform/pipeline_") { + userProcessors = append(userProcessors, v) + } + } + // all user processors are pushed after pipelines + processors = append(names, userProcessors...) + + service.(map[string]interface{})["pipelines"].(map[string]interface{})["logs"].(map[string]interface{})["processors"] = processors + + s := map[string]interface{}{ + "service": map[string]interface{}{ + "pipelines": map[string]interface{}{ + "logs": map[string]interface{}{ + "processors": processors, + }, + }, + }, + } + + serviceC := confmap.NewFromStringMap(s) + + err = agentConf.Merge(serviceC) + if err != nil { + return err + } + + // ------ complete adding processor + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + if err != nil { + return err + } + + zap.S().Infof("sending new config", string(configR)) + hash := sha256.New() + _, err = hash.Write(configR) + if err != nil { + return err + } + agent.EffectiveConfig = string(configR) + err = agent.Upsert() + if err != nil { + return err + } + + agent.SendToAgent(&protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: configR, + ContentType: "application/x-yaml", + }, + }, + }, + ConfigHash: hash.Sum(nil), + }, + }) + } + + return nil +} diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser.go b/pkg/query-service/app/opamp/otelconfig/config_parser.go new file mode 100644 index 0000000000..5d6c8adba4 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/config_parser.go @@ -0,0 +1,195 @@ +package otelconfig + +import ( + "sync" + + "go.opentelemetry.io/collector/confmap" +) + +type ConfigParser struct { + lock sync.Mutex + agentConf *confmap.Conf +} + +func NewConfigParser(agentConf *confmap.Conf) ConfigParser { + return ConfigParser{ + agentConf: agentConf, + } +} + +func toMap(i interface{}) map[string]interface{} { + return i.(map[string]interface{}) +} + +func toList(i interface{}) []interface{} { + return i.([]interface{}) +} + +// sent when key is not found in config +func emptyMap() map[string]interface{} { + return map[string]interface{}{} +} + +func emptyList() []interface{} { + return []interface{}{} +} + +func (cp *ConfigParser) Service() map[string]interface{} { + service := cp.agentConf.Get("service") + if service == nil { + return emptyMap() + } + return toMap(service) +} + +// components gets the high level parts like receivers, exporters, processors etc +func (cp *ConfigParser) components(partName, nameOptional string) map[string]interface{} { + parts := cp.agentConf.Get(partName) + if parts == nil { + return emptyMap() + } + + parsedParts := toMap(parts) + if nameOptional != "" { + if p, ok := parsedParts[nameOptional]; ok { + return p.(map[string]interface{}) + } else { + return emptyMap() + } + } + + return parsedParts +} + +func (cp *ConfigParser) Processors() map[string]interface{} { + return cp.components("processors", "") +} + +func (cp *ConfigParser) Processor(name string) map[string]interface{} { + return cp.components("processors", name) +} + +func (cp *ConfigParser) Exporters() map[string]interface{} { + return cp.components("exporters", "") +} + +func (cp *ConfigParser) Exporter(name string) map[string]interface{} { + return cp.components("exporters", name) +} + +func (cp *ConfigParser) Receivers() map[string]interface{} { + return cp.components("receivers", "") +} + +func (cp *ConfigParser) Receiver(name string) map[string]interface{} { + return cp.components("receivers", name) +} + +func (cp *ConfigParser) Pipelines(nameOptional string) map[string]interface{} { + services := cp.Service() + if p, ok := services["pipelines"]; ok { + pipelines := toMap(p) + if nameOptional != "" { + if namedPipeline, ok := pipelines[nameOptional]; ok { + return toMap(namedPipeline) + } else { + return emptyMap() + } + + } + return pipelines + } + return emptyMap() +} + +// component can be "recevers", "exporter" or "processors" +func (cp *ConfigParser) PipelineComponent(pipelineName, pipelineComponent string) []interface{} { + pipeline := cp.Pipelines(pipelineName) + if exporters, ok := pipeline[pipelineComponent]; ok { + exporters := toList(exporters) + return exporters + } + return emptyList() +} + +func (cp *ConfigParser) PipelineExporters(pipelineName string) []interface{} { + return cp.PipelineComponent(pipelineName, "exporters") +} + +func (cp *ConfigParser) PipelineReceivers(pipelineName string) []interface{} { + return cp.PipelineComponent(pipelineName, "receivers") +} + +func (cp *ConfigParser) PipelineProcessors(pipelineName string) []interface{} { + return cp.PipelineComponent(pipelineName, "processors") +} + +func (cp *ConfigParser) CheckPipelineExists(name string) bool { + if name == "" { + return false + } + pipelines := cp.Pipelines(name) + return len(pipelines) > 0 +} + +// CheckEntryInPipeline lets you look for an entry in pipeline by receiver, processor or exporter name +func (cp *ConfigParser) CheckEntryInPipeline(pipelineName, pipelineComponent, name string) bool { + if pipelineName == "" || pipelineComponent == "" || name == "" { + return false + } + + list := cp.PipelineComponent(pipelineName, pipelineComponent) + var found bool + for _, item := range list { + if item == name { + found = true + } + } + + return found +} + +func (cp *ConfigParser) CheckExporterInPipeline(pipelineName, name string) bool { + return cp.CheckEntryInPipeline(pipelineName, "exporters", name) +} + +func (cp *ConfigParser) CheckProcessorInPipeline(pipelineName, name string) bool { + return cp.CheckEntryInPipeline(pipelineName, "processors", name) +} + +func (cp *ConfigParser) Merge(c *confmap.Conf) { + cp.lock.Lock() + defer cp.lock.Unlock() + cp.agentConf.Merge(c) +} + +func (cp *ConfigParser) UpdateProcessors(processors map[string]interface{}) { + updates := cp.Processors() + + for key, params := range processors { + updates[key] = params + } + + updatedProcessors := map[string]interface{}{ + "processors": updates, + } + + updatedProcessorConf := confmap.NewFromStringMap(updatedProcessors) + + cp.Merge(updatedProcessorConf) +} + +func (cp *ConfigParser) UpdateProcsInPipeline(pipelineName string, list []interface{}) { + + serviceConf := map[string]interface{}{ + "service": map[string]interface{}{ + "pipelines": map[string]interface{}{ + pipelineName: map[string]interface{}{ + "processors": list, + }, + }, + }, + } + + cp.Merge(confmap.NewFromStringMap(serviceConf)) +} diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser_test.go b/pkg/query-service/app/opamp/otelconfig/config_parser_test.go new file mode 100644 index 0000000000..61ceb1f613 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/config_parser_test.go @@ -0,0 +1,58 @@ +package otelconfig + +import ( + "fmt" + "io/ioutil" + "testing" + + "github.com/knadh/koanf/parsers/yaml" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap" +) + +func TestServiceConfig(t *testing.T) { + yamlFile, err := ioutil.ReadFile("./testdata/service.yaml") + if err != nil { + fmt.Printf("yamlFile.Get err #%v ", err) + t.Fail() + return + } + + c, err := yaml.Parser().Unmarshal([]byte(yamlFile)) + if err != nil { + fmt.Println("failed to parse config file as yaml", err) + t.Fail() + return + } + + agentConf := confmap.NewFromStringMap(c) + configParser := NewConfigParser(agentConf) + + expected := map[string]interface{}{ + "extensions": []interface{}{"zpages"}, + "pipelines": map[string]interface{}{ + "traces": map[string]interface{}{ + "receivers": []interface{}{"jaeger", "otlp"}, + "processors": []interface{}{ + "signozspanmetrics/prometheus", "batch", + }, + "exporters": []interface{}{ + "clickhousetraces", + }, + }, + "metrics": map[string]interface{}{ + "receivers": []interface{}{ + "otlp", "hostmetrics", + }, + "processors": []interface{}{ + "batch", + }, + "exporters": []interface{}{ + "clickhousemetricswrite", + }, + }, + }, + } + + require.Equal(t, expected, configParser.Service(), "expected same service config after parsing") +} diff --git a/pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go b/pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go new file mode 100644 index 0000000000..1d4d93a959 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go @@ -0,0 +1,11 @@ +package filterprocessor + +type Config struct { + Metrics MetricFilters `mapstructure:"metrics"` +} + +// MetricFilters filters by Metric properties. +type MetricFilters struct { + MetricConditions []string `mapstructure:"metric" yaml:"metric,omitempty"` + DataPointConditions []string `mapstructure:"datapoint" yaml:"datapoint,omitempty"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go new file mode 100644 index 0000000000..869afc9f43 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go @@ -0,0 +1,6 @@ +package otlpreceiver + +type Protocols struct { + GRPC *GRPCServerSettings `mapstructure:"grpc"` + HTTP *HTTPServerSettings `mapstructure:"http"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go new file mode 100644 index 0000000000..f0f55eda10 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go @@ -0,0 +1,14 @@ +package otlpreceiver + +type GRPCServerSettings struct { + // Endpoint configures the address for this network connection. + // For TCP and UDP networks, the address has the form "host:port". The host must be a literal IP address, + // or a host name that can be resolved to IP addresses. The port must be a literal port number or a service name. + // If the host is a literal IPv6 address it must be enclosed in square brackets, as in "[2001:db8::1]:80" or + // "[fe80::1%zone]:80". The zone specifies the scope of the literal IPv6 address as defined in RFC 4007. + Endpoint string `mapstructure:"endpoint"` + + // Transport to use. Known protocols are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only), "udp", "udp4" (IPv4-only), + // "udp6" (IPv6-only), "ip", "ip4" (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and "unixpacket". + Transport string `mapstructure:"transport"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go new file mode 100644 index 0000000000..64c9a16fe5 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go @@ -0,0 +1,9 @@ +package otlpreceiver + +type HTTPServerSettings struct { + // Endpoint configures the listening address for the server. + Endpoint string `mapstructure:"endpoint" yaml:"endpoint"` + + // TLSSetting struct exposes TLS client configuration. + TLSSetting *TLSServerSetting `mapstructure:"tls" yaml:"tls"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go new file mode 100644 index 0000000000..39978fcb29 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go @@ -0,0 +1,38 @@ +package otlpreceiver + +import "time" + +type TLSSetting struct { + // Path to the CA cert. For a client this verifies the server certificate. + // For a server this verifies client certificates. If empty uses system root CA. + // (optional) + CAFile string `mapstructure:"ca_file"` + + // Path to the TLS cert to use for TLS required connections. (optional) + CertFile string `mapstructure:"cert_file"` + + // Path to the TLS key to use for TLS required connections. (optional) + KeyFile string `mapstructure:"key_file"` + + // MinVersion sets the minimum TLS version that is acceptable. + // If not set, TLS 1.2 will be used. (optional) + MinVersion string `mapstructure:"min_version"` + + // MaxVersion sets the maximum TLS version that is acceptable. + // If not set, refer to crypto/tls for defaults. (optional) + MaxVersion string `mapstructure:"max_version"` + + // ReloadInterval specifies the duration after which the certificate will be reloaded + // If not set, it will never be reloaded (optional) + ReloadInterval time.Duration `mapstructure:"reload_interval"` +} + +type TLSServerSetting struct { + // squash ensures fields are correctly decoded in embedded struct. + TLSSetting `mapstructure:",squash"` + + // Path to the TLS cert to use by the server to verify a client certificate. (optional) + // This sets the ClientCAs and ClientAuth to RequireAndVerifyClientCert in the TLSConfig. Please refer to + // https://godoc.org/crypto/tls#Config for more information. (optional) + ClientCAFile string `mapstructure:"client_ca_file"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/tailsampler/config.go b/pkg/query-service/app/opamp/otelconfig/tailsampler/config.go new file mode 100644 index 0000000000..d43a8eacb4 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/tailsampler/config.go @@ -0,0 +1,82 @@ +package tailsampler + +import "time" + +type PolicyType string + +type Config struct { + DecisionWait time.Duration `mapstructure:"decision_wait" yaml:"decision_wait"` + NumTraces uint64 `mapstructure:"num_traces" yaml:"num_traces"` + ExpectedNewTracesPerSec uint64 `mapstructure:"expected_new_traces_per_sec" yaml:"expected_new_traces_per_sec"` + PolicyCfgs []PolicyCfg `mapstructure:"policies" yaml:"policies"` + + // read only version number (optional) + Version int +} + +type ProbabilisticCfg struct { + // HashSalt allows one to configure the hashing salts. This is important in scenarios where multiple layers of collectors + // have different sampling rates: if they use the same salt all passing one layer may pass the other even if they have + // different sampling rates, configuring different salts avoids that. + HashSalt string `mapstructure:"hash_salt" yaml:"hash_salt"` + // SamplingPercentage is the percentage rate at which traces are going to be sampled. Defaults to zero, i.e.: no sample. + // Values greater or equal 100 are treated as "sample all traces". + SamplingPercentage float64 `mapstructure:"sampling_percentage" yaml:"sampling_percentage"` +} + +type NumericAttributeCfg struct { + // Tag that the filter is going to be matching against. + Key string `mapstructure:"key" yaml:"key"` + // MinValue is the minimum value of the attribute to be considered a match. + MinValue int64 `mapstructure:"min_value" yaml:"min_value"` + // MaxValue is the maximum value of the attribute to be considered a match. + MaxValue int64 `mapstructure:"max_value" yaml:"max_value"` +} + +type StringAttributeCfg struct { + // Tag that the filter is going to be matching against. + Key string `mapstructure:"key" yaml:"key"` + // Values indicate the set of values or regular expressions to use when matching against attribute values. + // StringAttribute Policy will apply exact value match on Values unless EnabledRegexMatching is true. + Values []string `mapstructure:"values" yaml:"values"` + // EnabledRegexMatching determines whether match attribute values by regexp string. + EnabledRegexMatching bool `mapstructure:"enabled_regex_matching" yaml:"enabled_regex_matching"` + // CacheMaxSize is the maximum number of attribute entries of LRU Cache that stores the matched result + // from the regular expressions defined in Values. + // CacheMaxSize will not be used if EnabledRegexMatching is set to false. + CacheMaxSize int `mapstructure:"cache_max_size" yaml:"cache_max_size"` + // InvertMatch indicates that values or regular expressions must not match against attribute values. + // If InvertMatch is true and Values is equal to 'acme', all other values will be sampled except 'acme'. + // Also, if the specified Key does not match on any resource or span attributes, data will be sampled. + InvertMatch bool `mapstructure:"invert_match" yaml:"invert_match"` +} + +type PolicyFilterCfg struct { + // values: AND | OR + FilterOp string `mapstructure:"filter_op" yaml:"filter_op"` + + StringAttributeCfgs []StringAttributeCfg `mapstructure:"string_attributes" yaml:"string_attributes"` + NumericAttributeCfgs []NumericAttributeCfg `mapstructure:"numeric_attributes" yaml:"numeric_attributes"` +} + +// PolicyCfg identifies policy rules in policy group +type PolicyCfg struct { + // name of the policy + Name string `mapstructure:"name" yaml:"name"` + + // Type of the policy this will be used to match the proper configuration of the policy. + Type PolicyType `mapstructure:"type" yaml:"type"` + + // Set to true for sampling rule (root) and false for conditions + Root bool `mapstructure:"root" yaml:"root"` + + Priority int `mapstructure:"priority" yaml:"priority"` + + // sampling applied when PolicyFilter matches + ProbabilisticCfg `mapstructure:",squash" yaml:"sampling"` + + // filter to activate policy + PolicyFilterCfg `mapstructure:",squash" yaml:"policy_filter"` + + SubPolicies []PolicyCfg `mapstructure:"sub_policies" yaml:"sub_policies"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml b/pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml new file mode 100644 index 0000000000..d5ef74e00f --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml @@ -0,0 +1,76 @@ +receivers: + otlp/spanmetrics: + protocols: + grpc: + endpoint: "localhost:12345" + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: + hostmetrics: + collection_interval: 30s + scrapers: + cpu: + load: + memory: + disk: + filesystem: + network: +processors: + batch: + send_batch_size: 1000 + timeout: 10s + signozspanmetrics/prometheus: + metrics_exporter: prometheus + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + dimensions_cache_size: 10000 + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + # memory_limiter: + # # 80% of maximum memory up to 2G + # limit_mib: 1500 + # # 25% of limit up to 2G + # spike_limit_mib: 512 + # check_interval: 5s + # + # # 50% of the maximum memory + # limit_percentage: 50 + # # 20% of max memory usage spike expected + # spike_limit_percentage: 20 + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + zpages: {} +exporters: + clickhousetraces: + datasource: tcp://localhost:9000/?database=signoz_traces + migrations: exporter/clickhousetracesexporter/migrations + clickhousemetricswrite: + endpoint: tcp://localhost:9000/?database=signoz_metrics + resource_to_telemetry_conversion: + enabled: true + prometheus: + endpoint: "0.0.0.0:8889" +service: + extensions: [zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhousetraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] + metrics/spanmetrics: + receivers: [otlp/spanmetrics] + exporters: [prometheus] \ No newline at end of file diff --git a/pkg/query-service/app/opamp/otelconfig/testdata/service.yaml b/pkg/query-service/app/opamp/otelconfig/testdata/service.yaml new file mode 100644 index 0000000000..dd562fba0d --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/testdata/service.yaml @@ -0,0 +1,11 @@ +service: + extensions: [zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhousetraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] diff --git a/pkg/query-service/app/opamp/pipeline_builder.go b/pkg/query-service/app/opamp/pipeline_builder.go new file mode 100644 index 0000000000..841a9ce5c6 --- /dev/null +++ b/pkg/query-service/app/opamp/pipeline_builder.go @@ -0,0 +1,196 @@ +package opamp + +import ( + "fmt" + "sync" + + "go.uber.org/zap" +) + +var lockTracesPipelineSpec sync.RWMutex +var lockMetricsPipelineSpec sync.RWMutex + +type pipelineStatus struct { + Name string + Enabled bool +} + +var tracesPipelineSpec = map[int]pipelineStatus{ + 0: { + Name: "signoz_tail_sampling", + Enabled: false, + }, + 1: { + Name: "batch", + Enabled: true, + }, +} + +var metricsPipelineSpec = map[int]pipelineStatus{ + 0: { + Name: "filter", + Enabled: false, + }, + 1: { + Name: "batch", + Enabled: true, + }, +} + +func updatePipelineSpec(signal string, name string, enabled bool) { + switch signal { + case "metrics": + lockMetricsPipelineSpec.Lock() + defer lockMetricsPipelineSpec.Unlock() + + for i := 0; i < len(metricsPipelineSpec); i++ { + p := metricsPipelineSpec[i] + if p.Name == name { + p.Enabled = enabled + metricsPipelineSpec[i] = p + } + } + case "traces": + lockTracesPipelineSpec.Lock() + defer lockTracesPipelineSpec.Unlock() + + for i := 0; i < len(tracesPipelineSpec); i++ { + p := tracesPipelineSpec[i] + if p.Name == name { + p.Enabled = enabled + tracesPipelineSpec[i] = p + } + } + default: + return + } + +} + +// AddToTracePipeline to enable processor in traces pipeline +func AddToTracePipelineSpec(processor string) { + updatePipelineSpec("traces", processor, true) +} + +// RemoveFromTracePipeline to remove processor from traces pipeline +func RemoveFromTracePipelineSpec(name string) { + updatePipelineSpec("traces", name, false) +} + +// AddToMetricsPipeline to enable processor in traces pipeline +func AddToMetricsPipelineSpec(processor string) { + updatePipelineSpec("metrics", processor, true) +} + +// RemoveFromMetricsPipeline to remove processor from traces pipeline +func RemoveFromMetricsPipelineSpec(name string) { + updatePipelineSpec("metrics", name, false) +} + +func checkDuplicates(pipeline []interface{}) bool { + exists := make(map[string]bool, len(pipeline)) + zap.S().Debugf("checking duplicate processors in the pipeline:", pipeline) + for _, processor := range pipeline { + name := processor.(string) + if _, ok := exists[name]; ok { + return true + } + + exists[name] = true + } + return false +} + +func buildPipeline(signal Signal, current []interface{}) ([]interface{}, error) { + var spec map[int]pipelineStatus + + switch signal { + case Metrics: + spec = metricsPipelineSpec + lockMetricsPipelineSpec.Lock() + defer lockMetricsPipelineSpec.Unlock() + case Traces: + spec = tracesPipelineSpec + lockTracesPipelineSpec.Lock() + defer lockTracesPipelineSpec.Unlock() + default: + return nil, fmt.Errorf("invalid signal") + } + + pipeline := current + // create a reverse map of existing config processors and their position + existing := map[string]int{} + for i, p := range current { + name := p.(string) + existing[name] = i + } + + // create mapping from our tracesPipelinePlan (processors managed by us) to position in existing processors (from current config) + // this means, if "batch" holds position 3 in the current effective config, and 2 in our config, the map will be [2]: 3 + specVsExistingMap := map[int]int{} + + // go through plan and map its elements to current positions in effective config + for i, m := range spec { + if loc, ok := existing[m.Name]; ok { + specVsExistingMap[i] = loc + } + } + + lastMatched := -1 + inserts := 0 + + // go through plan again in the increasing order + for i := 0; i < len(spec); i++ { + m := spec[i] + + if loc, ok := specVsExistingMap[i]; ok { + // element from plan already exists in current effective config. + + currentPos := loc + inserts + // if disabled then remove from the pipeline + if !m.Enabled { + zap.S().Debugf("build_pipeline: found a disabled item, removing from pipeline at position", currentPos-1, " ", m.Name) + if currentPos-1 <= 0 { + pipeline = pipeline[currentPos+1:] + } else { + pipeline = append(pipeline[:currentPos-1], pipeline[currentPos+1:]...) + } + } + + // capture last position where match was found, this will be used + // to insert missing elements next to it + lastMatched = currentPos + + } else { + if m.Enabled { + // track inserts as they shift the elements in pipeline + inserts++ + + // we use last matched to insert new item. This means, we keep inserting missing processors + // right after last matched processsor (e.g. insert filters after tail_sampling for existing list of [batch, tail_sampling]) + + if lastMatched <= 0 { + zap.S().Debugf("build_pipeline: found a new item to be inserted, inserting at position 0:", m.Name) + pipeline = append([]interface{}{m.Name}, pipeline[lastMatched+1:]...) + } else { + zap.S().Debugf("build_pipeline: found a new item to be inserted, inserting at position :", lastMatched, " ", m.Name) + prior := make([]interface{}, len(pipeline[:lastMatched])) + next := make([]interface{}, len(pipeline[lastMatched:])) + copy(prior, pipeline[:lastMatched]) + copy(next, pipeline[lastMatched:]) + + pipeline = append(prior, m.Name) + pipeline = append(pipeline, next...) + } + } + } + } + + if checkDuplicates(pipeline) { + // duplicates are most likely because the processor sequence in effective config conflicts + // with the planned sequence as per planned pipeline + return pipeline, fmt.Errorf("the effective config has an unexpected processor sequence: %v", pipeline) + } + + return pipeline, nil +} diff --git a/pkg/query-service/app/opamp/signal.go b/pkg/query-service/app/opamp/signal.go new file mode 100644 index 0000000000..754dfc33a8 --- /dev/null +++ b/pkg/query-service/app/opamp/signal.go @@ -0,0 +1,9 @@ +package opamp + +type Signal string + +const ( + Metrics Signal = "metrics" + Traces Signal = "traces" + Logs Signal = "logs" +) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 8088c2294e..c3142643f0 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -18,8 +18,12 @@ import ( "github.com/rs/cors" "github.com/soheilhy/cmux" + "go.signoz.io/signoz/pkg/query-service/agentConf" "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader" "go.signoz.io/signoz/pkg/query-service/app/dashboards" + opamp "go.signoz.io/signoz/pkg/query-service/app/opamp" + opAmpModel "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.signoz.io/signoz/pkg/query-service/app/explorer" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/constants" @@ -142,6 +146,14 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { s.privateHTTP = privateServer + _, err = opAmpModel.InitDB(constants.RELATIONAL_DATASOURCE_PATH) + if err != nil { + return nil, err + } + + if err := agentConf.Initiate(localDB, "sqlite"); err != nil { + return nil, err + } return s, nil } @@ -439,6 +451,37 @@ func (s *Server) Start() error { }() + go func() { + zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", constants.OpAmpWsEndpoint)) + err := opamp.InitalizeServer(constants.OpAmpWsEndpoint, &opAmpModel.AllAgents) + if err != nil { + zap.S().Info("opamp ws server failed to start", err) + s.unavailableChannel <- healthcheck.Unavailable + } + }() + + return nil +} + +func (s *Server) Stop() error { + if s.httpServer != nil { + if err := s.httpServer.Shutdown(context.Background()); err != nil { + return err + } + } + + if s.privateHTTP != nil { + if err := s.privateHTTP.Shutdown(context.Background()); err != nil { + return err + } + } + + opamp.StopServer() + + if s.ruleManager != nil { + s.ruleManager.Stop() + } + return nil } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 5bdc147c60..a18a69ac9a 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -11,6 +11,7 @@ const ( HTTPHostPort = "0.0.0.0:8080" // Address to serve http (query service) PrivateHostPort = "0.0.0.0:8085" // Address to server internal services like alert manager DebugHttpPort = "0.0.0.0:6060" // Address to serve http (pprof) + OpAmpWsEndpoint = "0.0.0.0:4320" // address for opamp websocket ) var ConfigSignozIo = "https://config.signoz.io/api/v1" diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index 24e9d06cb8..10bfe67306 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -84,7 +84,13 @@ func main() { case status := <-server.HealthCheckStatus(): logger.Info("Received HealthCheck status: ", zap.Int("status", int(status))) case <-signalsChannel: - logger.Fatal("Received OS Interrupt Signal ... ") + logger.Info("Received OS Interrupt Signal ... ") + err := server.Stop() + if err != nil { + logger.Fatal("Failed to stop server", zap.Error(err)) + } + logger.Info("Server stopped") + return } }