diff --git a/.gitignore b/.gitignore index 5107685692..080a49ae37 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ e2e/.auth # go vendor/ **/main/** +__debug_bin** # git-town .git-branches.toml diff --git a/conf/example.yaml b/conf/example.yaml index fbda035d8d..7c9f285da5 100644 --- a/conf/example.yaml +++ b/conf/example.yaml @@ -207,3 +207,11 @@ emailing: key_file_path: # The path to the certificate file. cert_file_path: + +##################### Sharder (experimental) ##################### +sharder: + # Specifies the sharder provider to use. + provider: noop + single: + # The org id to which this instance belongs to. + org_id: org_id diff --git a/ee/licensing/httplicensing/provider.go b/ee/licensing/httplicensing/provider.go index 52a162143c..5874df44d9 100644 --- a/ee/licensing/httplicensing/provider.go +++ b/ee/licensing/httplicensing/provider.go @@ -10,6 +10,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/licensing" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/licensetypes" @@ -19,23 +20,31 @@ import ( ) type provider struct { - store licensetypes.Store - zeus zeus.Zeus - config licensing.Config - settings factory.ScopedProviderSettings - stopChan chan struct{} + store licensetypes.Store + zeus zeus.Zeus + config licensing.Config + settings factory.ScopedProviderSettings + orgGetter organization.Getter + stopChan chan struct{} } -func NewProviderFactory(store sqlstore.SQLStore, zeus zeus.Zeus) factory.ProviderFactory[licensing.Licensing, licensing.Config] { +func NewProviderFactory(store sqlstore.SQLStore, zeus zeus.Zeus, orgGetter organization.Getter) factory.ProviderFactory[licensing.Licensing, licensing.Config] { return factory.NewProviderFactory(factory.MustNewName("http"), func(ctx context.Context, providerSettings factory.ProviderSettings, config licensing.Config) (licensing.Licensing, error) { - return New(ctx, providerSettings, config, store, zeus) + return New(ctx, providerSettings, config, store, zeus, orgGetter) }) } -func New(ctx context.Context, ps factory.ProviderSettings, config licensing.Config, sqlstore sqlstore.SQLStore, zeus zeus.Zeus) (licensing.Licensing, error) { +func New(ctx context.Context, ps factory.ProviderSettings, config licensing.Config, sqlstore sqlstore.SQLStore, zeus zeus.Zeus, orgGetter organization.Getter) (licensing.Licensing, error) { settings := factory.NewScopedProviderSettings(ps, "github.com/SigNoz/signoz/ee/licensing/httplicensing") licensestore := sqllicensingstore.New(sqlstore) - return &provider{store: licensestore, zeus: zeus, config: config, settings: settings, stopChan: make(chan struct{})}, nil + return &provider{ + store: licensestore, + zeus: zeus, + config: config, + settings: settings, + orgGetter: orgGetter, + stopChan: make(chan struct{}), + }, nil } func (provider *provider) Start(ctx context.Context) error { @@ -67,13 +76,13 @@ func (provider *provider) Stop(ctx context.Context) error { } func (provider *provider) Validate(ctx context.Context) error { - organizations, err := provider.store.ListOrganizations(ctx) + organizations, err := provider.orgGetter.ListByOwnedKeyRange(ctx) if err != nil { return err } - for _, organizationID := range organizations { - err := provider.Refresh(ctx, organizationID) + for _, organization := range organizations { + err := provider.Refresh(ctx, organization.ID) if err != nil { return err } diff --git a/ee/licensing/licensingstore/sqllicensingstore/store.go b/ee/licensing/licensingstore/sqllicensingstore/store.go index 5167a5fc00..9e8ea19d71 100644 --- a/ee/licensing/licensingstore/sqllicensingstore/store.go +++ b/ee/licensing/licensingstore/sqllicensingstore/store.go @@ -5,7 +5,6 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/sqlstore" - "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/licensetypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -82,31 +81,6 @@ func (store *store) Update(ctx context.Context, organizationID valuer.UUID, stor return nil } -func (store *store) ListOrganizations(ctx context.Context) ([]valuer.UUID, error) { - orgIDStrs := make([]string, 0) - err := store.sqlstore. - BunDB(). - NewSelect(). - Model(new(types.Organization)). - Column("id"). - Scan(ctx, &orgIDStrs) - if err != nil { - return nil, err - } - - orgIDs := make([]valuer.UUID, len(orgIDStrs)) - for idx, orgIDStr := range orgIDStrs { - orgID, err := valuer.NewUUID(orgIDStr) - if err != nil { - return nil, err - } - orgIDs[idx] = orgID - } - - return orgIDs, nil - -} - func (store *store) CreateFeature(ctx context.Context, storableFeature *featuretypes.StorableFeature) error { _, err := store. sqlstore. diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 84f3f4a7f4..cda311ec34 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -20,6 +20,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/http/middleware" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/signoz" "github.com/SigNoz/signoz/pkg/sqlstore" @@ -113,6 +114,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.SigNoz.SQLStore, serverOptions.SigNoz.TelemetryStore, serverOptions.SigNoz.Prometheus, + serverOptions.SigNoz.Modules.OrgGetter, ) if err != nil { @@ -157,7 +159,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } // start the usagemanager - usageManager, err := usage.New(serverOptions.SigNoz.Licensing, serverOptions.SigNoz.TelemetryStore.ClickhouseDB(), serverOptions.SigNoz.Zeus, serverOptions.SigNoz.Modules.Organization) + usageManager, err := usage.New(serverOptions.SigNoz.Licensing, serverOptions.SigNoz.TelemetryStore.ClickhouseDB(), serverOptions.SigNoz.Zeus, serverOptions.SigNoz.Modules.OrgGetter) if err != nil { return nil, err } @@ -225,7 +227,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { &opAmpModel.AllAgents, agentConfMgr, ) - orgs, err := apiHandler.Signoz.Modules.Organization.GetAll(context.Background()) + orgs, err := apiHandler.Signoz.Modules.OrgGetter.ListByOwnedKeyRange(context.Background()) if err != nil { return nil, err } @@ -240,11 +242,10 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } func (s *Server) createPrivateServer(apiHandler *api.APIHandler) (*http.Server, error) { - r := baseapp.NewRouter() - r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) - r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) + r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, s.serverOptions.SigNoz.Sharder, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) + r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.SigNoz.Sharder).Wrap) r.Use(middleware.NewTimeout(s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.Config.APIServer.Timeout.ExcludedRoutes, s.serverOptions.Config.APIServer.Timeout.Default, @@ -275,8 +276,8 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler, web web.Web) (*h r := baseapp.NewRouter() am := middleware.NewAuthZ(s.serverOptions.SigNoz.Instrumentation.Logger()) - r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) - r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) + r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, s.serverOptions.SigNoz.Sharder, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) + r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.SigNoz.Sharder).Wrap) r.Use(middleware.NewTimeout(s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.Config.APIServer.Timeout.ExcludedRoutes, s.serverOptions.Config.APIServer.Timeout.Default, @@ -450,6 +451,7 @@ func makeRulesManager( sqlstore sqlstore.SQLStore, telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, + orgGetter organization.Getter, ) (*baserules.Manager, error) { // create manager opts managerOpts := &baserules.ManagerOptions{ @@ -465,6 +467,7 @@ func makeRulesManager( PrepareTestRuleFunc: rules.TestNotification, Alertmanager: alertmanager, SQLStore: sqlstore, + OrgGetter: orgGetter, } // create Manager diff --git a/ee/query-service/main.go b/ee/query-service/main.go index 996b2ac0eb..6d9c3eeaf4 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -17,6 +17,7 @@ import ( "github.com/SigNoz/signoz/pkg/config/fileprovider" "github.com/SigNoz/signoz/pkg/factory" pkglicensing "github.com/SigNoz/signoz/pkg/licensing" + "github.com/SigNoz/signoz/pkg/modules/organization" baseconst "github.com/SigNoz/signoz/pkg/query-service/constants" "github.com/SigNoz/signoz/pkg/signoz" "github.com/SigNoz/signoz/pkg/sqlstore" @@ -133,8 +134,8 @@ func main() { zeus.Config(), httpzeus.NewProviderFactory(), licensing.Config(24*time.Hour, 3), - func(sqlstore sqlstore.SQLStore, zeus pkgzeus.Zeus) factory.ProviderFactory[pkglicensing.Licensing, pkglicensing.Config] { - return httplicensing.NewProviderFactory(sqlstore, zeus) + func(sqlstore sqlstore.SQLStore, zeus pkgzeus.Zeus, orgGetter organization.Getter) factory.ProviderFactory[pkglicensing.Licensing, pkglicensing.Config] { + return httplicensing.NewProviderFactory(sqlstore, zeus, orgGetter) }, signoz.NewEmailingProviderFactories(), signoz.NewCacheProviderFactories(), diff --git a/ee/query-service/usage/manager.go b/ee/query-service/usage/manager.go index c7ab151f80..ad8c8ec4ba 100644 --- a/ee/query-service/usage/manager.go +++ b/ee/query-service/usage/manager.go @@ -41,16 +41,16 @@ type Manager struct { zeus zeus.Zeus - organizationModule organization.Module + orgGetter organization.Getter } -func New(licenseService licensing.Licensing, clickhouseConn clickhouse.Conn, zeus zeus.Zeus, organizationModule organization.Module) (*Manager, error) { +func New(licenseService licensing.Licensing, clickhouseConn clickhouse.Conn, zeus zeus.Zeus, orgGetter organization.Getter) (*Manager, error) { m := &Manager{ - clickhouseConn: clickhouseConn, - licenseService: licenseService, - scheduler: gocron.NewScheduler(time.UTC).Every(1).Day().At("00:00"), // send usage every at 00:00 UTC - zeus: zeus, - organizationModule: organizationModule, + clickhouseConn: clickhouseConn, + licenseService: licenseService, + scheduler: gocron.NewScheduler(time.UTC).Every(1).Day().At("00:00"), // send usage every at 00:00 UTC + zeus: zeus, + orgGetter: orgGetter, } return m, nil } @@ -74,8 +74,7 @@ func (lm *Manager) Start(ctx context.Context) error { return nil } func (lm *Manager) UploadUsage(ctx context.Context) { - - organizations, err := lm.organizationModule.GetAll(context.Background()) + organizations, err := lm.orgGetter.ListByOwnedKeyRange(ctx) if err != nil { zap.L().Error("failed to get organizations", zap.Error(err)) return diff --git a/pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/config.go b/pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/config.go index 1175d9c369..c2bbc69876 100644 --- a/pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/config.go +++ b/pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/config.go @@ -67,23 +67,6 @@ func (store *config) Set(ctx context.Context, config *alertmanagertypes.Config, }, opts...) } -func (store *config) ListOrgs(ctx context.Context) ([]string, error) { - var orgIDs []string - - err := store. - sqlstore. - BunDB(). - NewSelect(). - Table("organizations"). - ColumnExpr("id"). - Scan(ctx, &orgIDs) - if err != nil { - return nil, err - } - - return orgIDs, nil -} - func (store *config) CreateChannel(ctx context.Context, channel *alertmanagertypes.Channel, opts ...alertmanagertypes.StoreOption) error { return store.wrap(ctx, func(ctx context.Context) error { if _, err := store. diff --git a/pkg/alertmanager/legacyalertmanager/provider.go b/pkg/alertmanager/legacyalertmanager/provider.go index b8fbba68e3..b4e1453b31 100644 --- a/pkg/alertmanager/legacyalertmanager/provider.go +++ b/pkg/alertmanager/legacyalertmanager/provider.go @@ -14,6 +14,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerbatcher" "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/types/alertmanagertypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -57,16 +58,17 @@ type provider struct { configStore alertmanagertypes.ConfigStore batcher *alertmanagerbatcher.Batcher url *url.URL + orgGetter organization.Getter orgID string } -func NewFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[alertmanager.Alertmanager, alertmanager.Config] { +func NewFactory(sqlstore sqlstore.SQLStore, orgGetter organization.Getter) factory.ProviderFactory[alertmanager.Alertmanager, alertmanager.Config] { return factory.NewProviderFactory(factory.MustNewName("legacy"), func(ctx context.Context, settings factory.ProviderSettings, config alertmanager.Config) (alertmanager.Alertmanager, error) { - return New(ctx, settings, config, sqlstore) + return New(ctx, settings, config, sqlstore, orgGetter) }) } -func New(ctx context.Context, providerSettings factory.ProviderSettings, config alertmanager.Config, sqlstore sqlstore.SQLStore) (*provider, error) { +func New(ctx context.Context, providerSettings factory.ProviderSettings, config alertmanager.Config, sqlstore sqlstore.SQLStore, orgGetter organization.Getter) (*provider, error) { settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/alertmanager/legacyalertmanager") configStore := sqlalertmanagerstore.NewConfigStore(sqlstore) @@ -92,7 +94,7 @@ func (provider *provider) Start(ctx context.Context) error { // For the first time, we need to get the orgID from the config store. // Since this is the legacy alertmanager, we get the first org from the store. if provider.orgID == "" { - orgIDs, err := provider.configStore.ListOrgs(ctx) + orgIDs, err := provider.orgGetter.ListByOwnedKeyRange(ctx) if err != nil { provider.settings.Logger().ErrorContext(ctx, "failed to send alerts to alertmanager", "error", err) continue @@ -103,7 +105,7 @@ func (provider *provider) Start(ctx context.Context) error { continue } - provider.orgID = orgIDs[0] + provider.orgID = orgIDs[0].ID.String() } if err := provider.putAlerts(ctx, provider.orgID, alerts); err != nil { diff --git a/pkg/alertmanager/service.go b/pkg/alertmanager/service.go index d8fdd74b28..c67c9e9edc 100644 --- a/pkg/alertmanager/service.go +++ b/pkg/alertmanager/service.go @@ -7,6 +7,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/types/alertmanagertypes" ) @@ -20,6 +21,9 @@ type Service struct { // configStore is the config store for the alertmanager service configStore alertmanagertypes.ConfigStore + // organization is the organization module for the alertmanager service + orgGetter organization.Getter + // settings is the settings for the alertmanager service settings factory.ScopedProviderSettings @@ -30,11 +34,19 @@ type Service struct { serversMtx sync.RWMutex } -func New(ctx context.Context, settings factory.ScopedProviderSettings, config alertmanagerserver.Config, stateStore alertmanagertypes.StateStore, configStore alertmanagertypes.ConfigStore) *Service { +func New( + ctx context.Context, + settings factory.ScopedProviderSettings, + config alertmanagerserver.Config, + stateStore alertmanagertypes.StateStore, + configStore alertmanagertypes.ConfigStore, + orgGetter organization.Getter, +) *Service { service := &Service{ config: config, stateStore: stateStore, configStore: configStore, + orgGetter: orgGetter, settings: settings, servers: make(map[string]*alertmanagerserver.Server), serversMtx: sync.RWMutex{}, @@ -44,38 +56,38 @@ func New(ctx context.Context, settings factory.ScopedProviderSettings, config al } func (service *Service) SyncServers(ctx context.Context) error { - orgIDs, err := service.configStore.ListOrgs(ctx) + orgs, err := service.orgGetter.ListByOwnedKeyRange(ctx) if err != nil { return err } service.serversMtx.Lock() - for _, orgID := range orgIDs { - config, err := service.getConfig(ctx, orgID) + for _, org := range orgs { + config, err := service.getConfig(ctx, org.ID.StringValue()) if err != nil { - service.settings.Logger().ErrorContext(ctx, "failed to get alertmanager config for org", "org_id", orgID, "error", err) + service.settings.Logger().ErrorContext(ctx, "failed to get alertmanager config for org", "org_id", org.ID.StringValue(), "error", err) continue } // If the server is not present, create it and sync the config - if _, ok := service.servers[orgID]; !ok { - server, err := service.newServer(ctx, orgID) + if _, ok := service.servers[org.ID.StringValue()]; !ok { + server, err := service.newServer(ctx, org.ID.StringValue()) if err != nil { - service.settings.Logger().ErrorContext(ctx, "failed to create alertmanager server", "org_id", orgID, "error", err) + service.settings.Logger().ErrorContext(ctx, "failed to create alertmanager server", "org_id", org.ID.StringValue(), "error", err) continue } - service.servers[orgID] = server + service.servers[org.ID.StringValue()] = server } - if service.servers[orgID].Hash() == config.StoreableConfig().Hash { - service.settings.Logger().DebugContext(ctx, "skipping alertmanager sync for org", "org_id", orgID, "hash", config.StoreableConfig().Hash) + if service.servers[org.ID.StringValue()].Hash() == config.StoreableConfig().Hash { + service.settings.Logger().DebugContext(ctx, "skipping alertmanager sync for org", "org_id", org.ID.StringValue(), "hash", config.StoreableConfig().Hash) continue } - err = service.servers[orgID].SetConfig(ctx, config) + err = service.servers[org.ID.StringValue()].SetConfig(ctx, config) if err != nil { - service.settings.Logger().ErrorContext(ctx, "failed to set config for alertmanager server", "org_id", orgID, "error", err) + service.settings.Logger().ErrorContext(ctx, "failed to set config for alertmanager server", "org_id", org.ID.StringValue(), "error", err) continue } } diff --git a/pkg/alertmanager/signozalertmanager/provider.go b/pkg/alertmanager/signozalertmanager/provider.go index c0b285358f..8c3c0872ae 100644 --- a/pkg/alertmanager/signozalertmanager/provider.go +++ b/pkg/alertmanager/signozalertmanager/provider.go @@ -8,6 +8,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/types/alertmanagertypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -22,13 +23,13 @@ type provider struct { stopC chan struct{} } -func NewFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[alertmanager.Alertmanager, alertmanager.Config] { +func NewFactory(sqlstore sqlstore.SQLStore, orgGetter organization.Getter) factory.ProviderFactory[alertmanager.Alertmanager, alertmanager.Config] { return factory.NewProviderFactory(factory.MustNewName("signoz"), func(ctx context.Context, settings factory.ProviderSettings, config alertmanager.Config) (alertmanager.Alertmanager, error) { - return New(ctx, settings, config, sqlstore) + return New(ctx, settings, config, sqlstore, orgGetter) }) } -func New(ctx context.Context, providerSettings factory.ProviderSettings, config alertmanager.Config, sqlstore sqlstore.SQLStore) (*provider, error) { +func New(ctx context.Context, providerSettings factory.ProviderSettings, config alertmanager.Config, sqlstore sqlstore.SQLStore, orgGetter organization.Getter) (*provider, error) { settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager") configStore := sqlalertmanagerstore.NewConfigStore(sqlstore) stateStore := sqlalertmanagerstore.NewStateStore(sqlstore) @@ -40,6 +41,7 @@ func New(ctx context.Context, providerSettings factory.ProviderSettings, config config.Signoz.Config, stateStore, configStore, + orgGetter, ), settings: settings, config: config, diff --git a/pkg/http/middleware/api_key.go b/pkg/http/middleware/api_key.go index 01e1981bd7..0d53b736bc 100644 --- a/pkg/http/middleware/api_key.go +++ b/pkg/http/middleware/api_key.go @@ -5,9 +5,15 @@ import ( "net/http" "time" + "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/authtypes" + "github.com/SigNoz/signoz/pkg/valuer" +) + +const ( + apiKeyCrossOrgMessage string = "::API-KEY-CROSS-ORG::" ) type APIKey struct { @@ -15,10 +21,11 @@ type APIKey struct { uuid *authtypes.UUID headers []string logger *slog.Logger + sharder sharder.Sharder } -func NewAPIKey(store sqlstore.SQLStore, headers []string, logger *slog.Logger) *APIKey { - return &APIKey{store: store, uuid: authtypes.NewUUID(), headers: headers, logger: logger} +func NewAPIKey(store sqlstore.SQLStore, headers []string, logger *slog.Logger, sharder sharder.Sharder) *APIKey { + return &APIKey{store: store, uuid: authtypes.NewUUID(), headers: headers, logger: logger, sharder: sharder} } func (a *APIKey) Wrap(next http.Handler) http.Handler { @@ -36,13 +43,20 @@ func (a *APIKey) Wrap(next http.Handler) http.Handler { next.ServeHTTP(w, r) return } + apiKeyToken, ok := authtypes.UUIDFromContext(ctx) if !ok { next.ServeHTTP(w, r) return } - err = a.store.BunDB().NewSelect().Model(&apiKey).Where("token = ?", apiKeyToken).Scan(r.Context()) + err = a. + store. + BunDB(). + NewSelect(). + Model(&apiKey). + Where("token = ?", apiKeyToken). + Scan(r.Context()) if err != nil { next.ServeHTTP(w, r) return @@ -71,6 +85,18 @@ func (a *APIKey) Wrap(next http.Handler) http.Handler { ctx = authtypes.NewContextWithClaims(ctx, jwt) + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + next.ServeHTTP(w, r) + return + } + + if err := a.sharder.IsMyOwnedKey(r.Context(), types.NewOrganizationKey(valuer.MustNewUUID(claims.OrgID))); err != nil { + a.logger.ErrorContext(r.Context(), apiKeyCrossOrgMessage, "claims", claims, "error", err) + next.ServeHTTP(w, r) + return + } + r = r.WithContext(ctx) next.ServeHTTP(w, r) diff --git a/pkg/http/middleware/auth.go b/pkg/http/middleware/auth.go index 491ccb93f1..8e6a4e3a03 100644 --- a/pkg/http/middleware/auth.go +++ b/pkg/http/middleware/auth.go @@ -1,18 +1,28 @@ package middleware import ( + "log/slog" "net/http" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/authtypes" + "github.com/SigNoz/signoz/pkg/valuer" +) + +const ( + authCrossOrgMessage string = "::AUTH-CROSS-ORG::" ) type Auth struct { jwt *authtypes.JWT headers []string + sharder sharder.Sharder + logger *slog.Logger } -func NewAuth(jwt *authtypes.JWT, headers []string) *Auth { - return &Auth{jwt: jwt, headers: headers} +func NewAuth(jwt *authtypes.JWT, headers []string, sharder sharder.Sharder, logger *slog.Logger) *Auth { + return &Auth{jwt: jwt, headers: headers, sharder: sharder, logger: logger} } func (a *Auth) Wrap(next http.Handler) http.Handler { @@ -28,6 +38,18 @@ func (a *Auth) Wrap(next http.Handler) http.Handler { return } + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + next.ServeHTTP(w, r) + return + } + + if err := a.sharder.IsMyOwnedKey(r.Context(), types.NewOrganizationKey(valuer.MustNewUUID(claims.OrgID))); err != nil { + a.logger.ErrorContext(r.Context(), authCrossOrgMessage, "claims", claims, "error", err) + next.ServeHTTP(w, r) + return + } + r = r.WithContext(ctx) next.ServeHTTP(w, r) diff --git a/pkg/modules/apdex/apdex.go b/pkg/modules/apdex/apdex.go index 9b23334f36..ed618f0670 100644 --- a/pkg/modules/apdex/apdex.go +++ b/pkg/modules/apdex/apdex.go @@ -4,13 +4,13 @@ import ( "context" "net/http" - "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/types/apdextypes" ) type Module interface { - Get(context.Context, string, []string) ([]*types.ApdexSettings, error) + Get(context.Context, string, []string) ([]*apdextypes.Settings, error) - Set(context.Context, string, *types.ApdexSettings) error + Set(context.Context, string, *apdextypes.Settings) error } type Handler interface { diff --git a/pkg/modules/apdex/implapdex/handler.go b/pkg/modules/apdex/implapdex/handler.go index effa4c537d..38030cda70 100644 --- a/pkg/modules/apdex/implapdex/handler.go +++ b/pkg/modules/apdex/implapdex/handler.go @@ -9,7 +9,7 @@ import ( "github.com/SigNoz/signoz/pkg/http/render" "github.com/SigNoz/signoz/pkg/modules/apdex" - "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/types/apdextypes" "github.com/SigNoz/signoz/pkg/types/authtypes" ) @@ -31,7 +31,7 @@ func (handler *handler) Set(rw http.ResponseWriter, req *http.Request) { return } - var apdexSettings types.ApdexSettings + var apdexSettings apdextypes.Settings if err := json.NewDecoder(req.Body).Decode(&apdexSettings); err != nil { render.Error(rw, err) return diff --git a/pkg/modules/apdex/implapdex/module.go b/pkg/modules/apdex/implapdex/module.go index 9eaa86048c..3012e4ba36 100644 --- a/pkg/modules/apdex/implapdex/module.go +++ b/pkg/modules/apdex/implapdex/module.go @@ -6,7 +6,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/modules/apdex" "github.com/SigNoz/signoz/pkg/sqlstore" - "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/types/apdextypes" "github.com/SigNoz/signoz/pkg/valuer" "github.com/uptrace/bun" ) @@ -25,8 +25,8 @@ func NewModule(sqlstore sqlstore.SQLStore) apdex.Module { } } -func (module *module) Get(ctx context.Context, orgID string, services []string) ([]*types.ApdexSettings, error) { - var apdexSettings []*types.ApdexSettings +func (module *module) Get(ctx context.Context, orgID string, services []string) ([]*apdextypes.Settings, error) { + var apdexSettings []*apdextypes.Settings err := module. sqlstore. @@ -51,7 +51,7 @@ func (module *module) Get(ctx context.Context, orgID string, services []string) } if !found { - apdexSettings = append(apdexSettings, &types.ApdexSettings{ + apdexSettings = append(apdexSettings, &apdextypes.Settings{ ServiceName: service, Threshold: defaultApdexThreshold, }) @@ -61,7 +61,7 @@ func (module *module) Get(ctx context.Context, orgID string, services []string) return apdexSettings, nil } -func (module *module) Set(ctx context.Context, orgID string, apdexSettings *types.ApdexSettings) error { +func (module *module) Set(ctx context.Context, orgID string, apdexSettings *apdextypes.Settings) error { apdexSettings.OrgID = orgID apdexSettings.Identifiable.ID = valuer.GenerateUUID() diff --git a/pkg/modules/organization/implorganization/getter.go b/pkg/modules/organization/implorganization/getter.go new file mode 100644 index 0000000000..78c684ece3 --- /dev/null +++ b/pkg/modules/organization/implorganization/getter.go @@ -0,0 +1,36 @@ +package implorganization + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/modules/organization" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/valuer" +) + +type getter struct { + store types.OrganizationStore + sharder sharder.Sharder +} + +func NewGetter(store types.OrganizationStore, sharder sharder.Sharder) organization.Getter { + return &getter{store: store, sharder: sharder} +} + +func (module *getter) Get(ctx context.Context, id valuer.UUID) (*types.Organization, error) { + return module.store.Get(ctx, id) +} + +func (module *getter) List(ctx context.Context) ([]*types.Organization, error) { + return module.store.GetAll(ctx) +} + +func (module *getter) ListByOwnedKeyRange(ctx context.Context) ([]*types.Organization, error) { + start, end, err := module.sharder.GetMyOwnedKeyRange(ctx) + if err != nil { + return nil, err + } + + return module.store.ListByKeyRange(ctx, start, end) +} diff --git a/pkg/modules/organization/implorganization/handler.go b/pkg/modules/organization/implorganization/handler.go index 9c5ca484ed..ce15529932 100644 --- a/pkg/modules/organization/implorganization/handler.go +++ b/pkg/modules/organization/implorganization/handler.go @@ -15,11 +15,12 @@ import ( ) type handler struct { - module organization.Module + orgGetter organization.Getter + orgSetter organization.Setter } -func NewHandler(module organization.Module) organization.Handler { - return &handler{module: module} +func NewHandler(orgGetter organization.Getter, orgSetter organization.Setter) organization.Handler { + return &handler{orgGetter: orgGetter, orgSetter: orgSetter} } func (handler *handler) Get(rw http.ResponseWriter, r *http.Request) { @@ -38,7 +39,7 @@ func (handler *handler) Get(rw http.ResponseWriter, r *http.Request) { return } - organization, err := handler.module.Get(ctx, orgID) + organization, err := handler.orgGetter.Get(ctx, orgID) if err != nil { render.Error(rw, err) return @@ -70,7 +71,7 @@ func (handler *handler) Update(rw http.ResponseWriter, r *http.Request) { } req.ID = orgID - err = handler.module.Update(ctx, req) + err = handler.orgSetter.Update(ctx, req) if err != nil { render.Error(rw, err) return diff --git a/pkg/modules/organization/implorganization/module.go b/pkg/modules/organization/implorganization/module.go deleted file mode 100644 index 27fe2f1e6f..0000000000 --- a/pkg/modules/organization/implorganization/module.go +++ /dev/null @@ -1,33 +0,0 @@ -package implorganization - -import ( - "context" - - "github.com/SigNoz/signoz/pkg/modules/organization" - "github.com/SigNoz/signoz/pkg/types" - "github.com/SigNoz/signoz/pkg/valuer" -) - -type module struct { - store types.OrganizationStore -} - -func NewModule(organizationStore types.OrganizationStore) organization.Module { - return &module{store: organizationStore} -} - -func (module *module) Create(ctx context.Context, organization *types.Organization) error { - return module.store.Create(ctx, organization) -} - -func (module *module) Get(ctx context.Context, id valuer.UUID) (*types.Organization, error) { - return module.store.Get(ctx, id) -} - -func (module *module) GetAll(ctx context.Context) ([]*types.Organization, error) { - return module.store.GetAll(ctx) -} - -func (module *module) Update(ctx context.Context, updatedOrganization *types.Organization) error { - return module.store.Update(ctx, updatedOrganization) -} diff --git a/pkg/modules/organization/implorganization/setter.go b/pkg/modules/organization/implorganization/setter.go new file mode 100644 index 0000000000..693d4a3aab --- /dev/null +++ b/pkg/modules/organization/implorganization/setter.go @@ -0,0 +1,40 @@ +package implorganization + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/modules/organization" + "github.com/SigNoz/signoz/pkg/modules/quickfilter" + "github.com/SigNoz/signoz/pkg/types" +) + +type setter struct { + store types.OrganizationStore + alertmanager alertmanager.Alertmanager + quickfilter quickfilter.Module +} + +func NewSetter(store types.OrganizationStore, alertmanager alertmanager.Alertmanager, quickfilter quickfilter.Module) organization.Setter { + return &setter{store: store, alertmanager: alertmanager, quickfilter: quickfilter} +} + +func (module *setter) Create(ctx context.Context, organization *types.Organization) error { + if err := module.store.Create(ctx, organization); err != nil { + return err + } + + if err := module.alertmanager.SetDefaultConfig(ctx, organization.ID.StringValue()); err != nil { + return err + } + + if err := module.quickfilter.SetDefaultConfig(ctx, organization.ID); err != nil { + return err + } + + return nil +} + +func (module *setter) Update(ctx context.Context, updatedOrganization *types.Organization) error { + return module.store.Update(ctx, updatedOrganization) +} diff --git a/pkg/modules/organization/implorganization/store.go b/pkg/modules/organization/implorganization/store.go index ec33cd04bd..6acd247dc5 100644 --- a/pkg/modules/organization/implorganization/store.go +++ b/pkg/modules/organization/implorganization/store.go @@ -92,3 +92,20 @@ func (store *store) Delete(ctx context.Context, id valuer.UUID) error { return nil } + +func (store *store) ListByKeyRange(ctx context.Context, start, end uint32) ([]*types.Organization, error) { + organizations := make([]*types.Organization, 0) + err := store. + sqlstore. + BunDB(). + NewSelect(). + Model(&organizations). + Where("key >= ?", start). + Where("key <= ?", end). + Scan(ctx) + if err != nil { + return nil, err + } + + return organizations, nil +} diff --git a/pkg/modules/organization/organization.go b/pkg/modules/organization/organization.go index 00ba48f95d..a48ef682d6 100644 --- a/pkg/modules/organization/organization.go +++ b/pkg/modules/organization/organization.go @@ -8,17 +8,22 @@ import ( "github.com/SigNoz/signoz/pkg/valuer" ) -type Module interface { - // Create creates the given organization - Create(context.Context, *types.Organization) error - +type Getter interface { // Get gets the organization based on the given id Get(context.Context, valuer.UUID) (*types.Organization, error) - // GetAll gets all the organizations - GetAll(context.Context) ([]*types.Organization, error) + // Lists all the organizations + List(context.Context) ([]*types.Organization, error) - // Update updates the given organization based on id present + // ListByOwnedKeyRange gets all the organizations owned by the instance + ListByOwnedKeyRange(context.Context) ([]*types.Organization, error) +} + +type Setter interface { + // Create creates the given organization + Create(context.Context, *types.Organization) error + + // Update updates the given organization Update(context.Context, *types.Organization) error } diff --git a/pkg/modules/user/impluser/module.go b/pkg/modules/user/impluser/module.go index ed7a58c9b7..45eaa4ba66 100644 --- a/pkg/modules/user/impluser/module.go +++ b/pkg/modules/user/impluser/module.go @@ -11,8 +11,10 @@ import ( "github.com/SigNoz/signoz/pkg/emailing" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/modules/user" "github.com/SigNoz/signoz/pkg/query-service/constants" + "github.com/SigNoz/signoz/pkg/query-service/model" "github.com/SigNoz/signoz/pkg/query-service/telemetry" "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/authtypes" @@ -22,20 +24,22 @@ import ( ) type Module struct { - store types.UserStore - jwt *authtypes.JWT - emailing emailing.Emailing - settings factory.ScopedProviderSettings + store types.UserStore + jwt *authtypes.JWT + emailing emailing.Emailing + settings factory.ScopedProviderSettings + orgSetter organization.Setter } // This module is a WIP, don't take inspiration from this. -func NewModule(store types.UserStore, jwt *authtypes.JWT, emailing emailing.Emailing, providerSettings factory.ProviderSettings) user.Module { +func NewModule(store types.UserStore, jwt *authtypes.JWT, emailing emailing.Emailing, providerSettings factory.ProviderSettings, orgSetter organization.Setter) user.Module { settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/user/impluser") return &Module{ - store: store, - jwt: jwt, - emailing: emailing, - settings: settings, + store: store, + jwt: jwt, + emailing: emailing, + settings: settings, + orgSetter: orgSetter, } } @@ -538,3 +542,36 @@ func (m *Module) ListDomains(ctx context.Context, orgID valuer.UUID) ([]*types.G func (m *Module) UpdateDomain(ctx context.Context, domain *types.GettableOrgDomain) error { return m.store.UpdateDomain(ctx, domain) } + +func (m *Module) Register(ctx context.Context, req *types.PostableRegisterOrgAndAdmin) (*types.User, error) { + if req.Email == "" { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "email is required") + } + + if req.Password == "" { + return nil, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "password is required") + } + + organization := types.NewOrganization(req.OrgDisplayName) + err := m.orgSetter.Create(ctx, organization) + if err != nil { + return nil, model.InternalError(err) + } + + user, err := types.NewUser(req.Name, req.Email, types.RoleAdmin.String(), organization.ID.StringValue()) + if err != nil { + return nil, model.InternalError(err) + } + + password, err := types.NewFactorPassword(req.Password) + if err != nil { + return nil, model.InternalError(err) + } + + user, err = m.CreateUserWithPassword(ctx, user, password) + if err != nil { + return nil, model.InternalError(err) + } + + return user, nil +} diff --git a/pkg/modules/user/user.go b/pkg/modules/user/user.go index f2f4153ddc..cb991ab95b 100644 --- a/pkg/modules/user/user.go +++ b/pkg/modules/user/user.go @@ -62,6 +62,9 @@ type Module interface { ListAPIKeys(ctx context.Context, orgID valuer.UUID) ([]*types.StorableAPIKeyUser, error) RevokeAPIKey(ctx context.Context, id, removedByUserID valuer.UUID) error GetAPIKey(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*types.StorableAPIKeyUser, error) + + // Register + Register(ctx context.Context, req *types.PostableRegisterOrgAndAdmin) (*types.User, error) } type Handler interface { diff --git a/pkg/query-service/app/cloudintegrations/controller_test.go b/pkg/query-service/app/cloudintegrations/controller_test.go index ff86b868db..dec3849b1c 100644 --- a/pkg/query-service/app/cloudintegrations/controller_test.go +++ b/pkg/query-service/app/cloudintegrations/controller_test.go @@ -3,17 +3,23 @@ package cloudintegrations import ( "context" "testing" + "time" - "github.com/SigNoz/signoz/pkg/emailing" - "github.com/SigNoz/signoz/pkg/emailing/noopemailing" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" "github.com/SigNoz/signoz/pkg/modules/user" - "github.com/SigNoz/signoz/pkg/modules/user/impluser" "github.com/SigNoz/signoz/pkg/query-service/model" "github.com/SigNoz/signoz/pkg/query-service/utils" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" + "github.com/SigNoz/signoz/pkg/signoz" "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -24,11 +30,16 @@ func TestRegenerateConnectionUrlWithUpdatedConfig(t *testing.T) { controller, err := NewController(sqlStore) require.NoError(err) - organizationModule := implorganization.NewModule(implorganization.NewStore(sqlStore)) providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - userModule := impluser.NewModule(impluser.NewStore(sqlStore, providerSettings), nil, emailing, providerSettings) - user, apiErr := createTestUser(organizationModule, userModule) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlStore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Provider: "signoz", Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, sqlStore, orgGetter) + require.NoError(err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) // should be able to generate connection url for @@ -74,11 +85,17 @@ func TestAgentCheckIns(t *testing.T) { sqlStore := utils.NewQueryServiceDBForTests(t) controller, err := NewController(sqlStore) require.NoError(err) - organizationModule := implorganization.NewModule(implorganization.NewStore(sqlStore)) + providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - userModule := impluser.NewModule(impluser.NewStore(sqlStore, providerSettings), nil, emailing, providerSettings) - user, apiErr := createTestUser(organizationModule, userModule) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlStore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Provider: "signoz", Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, sqlStore, orgGetter) + require.NoError(err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) // An agent should be able to check in from a cloud account even @@ -164,11 +181,16 @@ func TestCantDisconnectNonExistentAccount(t *testing.T) { controller, err := NewController(sqlStore) require.NoError(err) - organizationModule := implorganization.NewModule(implorganization.NewStore(sqlStore)) providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - userModule := impluser.NewModule(impluser.NewStore(sqlStore, providerSettings), nil, emailing, providerSettings) - user, apiErr := createTestUser(organizationModule, userModule) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlStore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Provider: "signoz", Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, sqlStore, orgGetter) + require.NoError(err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) // Attempting to disconnect a non-existent account should return error @@ -186,11 +208,16 @@ func TestConfigureService(t *testing.T) { controller, err := NewController(sqlStore) require.NoError(err) - organizationModule := implorganization.NewModule(implorganization.NewStore(sqlStore)) providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - userModule := impluser.NewModule(impluser.NewStore(sqlStore, providerSettings), nil, emailing, providerSettings) - user, apiErr := createTestUser(organizationModule, userModule) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlStore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Provider: "signoz", Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, sqlStore, orgGetter) + require.NoError(err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) require.Nil(apiErr) // create a connected account @@ -305,7 +332,7 @@ func makeTestConnectedAccount(t *testing.T, orgId string, controller *Controller return acc } -func createTestUser(organizationModule organization.Module, userModule user.Module) (*types.User, *model.ApiError) { +func createTestUser(organizationModule organization.Setter, userModule user.Module) (*types.User, *model.ApiError) { // Create a test user for auth ctx := context.Background() organization := types.NewOrganization("test") diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 8c6a6366fb..3712fdfc1d 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -55,7 +55,6 @@ import ( "github.com/SigNoz/signoz/pkg/query-service/app/queryBuilder" tracesV3 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v3" tracesV4 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v4" - "github.com/SigNoz/signoz/pkg/query-service/auth" "github.com/SigNoz/signoz/pkg/query-service/contextlinks" v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" "github.com/SigNoz/signoz/pkg/query-service/postprocess" @@ -255,7 +254,7 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { aH.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts) // TODO(nitya): remote this in later for multitenancy. - orgs, err := opts.Signoz.Modules.Organization.GetAll(context.Background()) + orgs, err := opts.Signoz.Modules.OrgGetter.ListByOwnedKeyRange(context.Background()) if err != nil { zap.L().Warn("unexpected error while fetching orgs while initializing base api handler", zap.Error(err)) } @@ -2062,9 +2061,9 @@ func (aH *APIHandler) registerUser(w http.ResponseWriter, r *http.Request) { return } - _, apiErr := auth.Register(context.Background(), &req, aH.Signoz.Alertmanager, aH.Signoz.Modules.Organization, aH.Signoz.Modules.User, aH.Signoz.Modules.QuickFilter) - if apiErr != nil { - RespondError(w, apiErr, nil) + _, errv2 := aH.Signoz.Modules.User.Register(r.Context(), &req) + if errv2 != nil { + render.Error(w, errv2) return } diff --git a/pkg/query-service/app/integrations/manager_test.go b/pkg/query-service/app/integrations/manager_test.go index c78413c0b7..39c9d1ff88 100644 --- a/pkg/query-service/app/integrations/manager_test.go +++ b/pkg/query-service/app/integrations/manager_test.go @@ -3,12 +3,18 @@ package integrations import ( "context" "testing" + "time" - "github.com/SigNoz/signoz/pkg/emailing" - "github.com/SigNoz/signoz/pkg/emailing/noopemailing" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" - "github.com/SigNoz/signoz/pkg/modules/user/impluser" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" + "github.com/SigNoz/signoz/pkg/signoz" + "github.com/SigNoz/signoz/pkg/types/authtypes" _ "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/require" ) @@ -19,11 +25,14 @@ func TestIntegrationLifecycle(t *testing.T) { mgr, store := NewTestIntegrationsManager(t) ctx := context.Background() - organizationModule := implorganization.NewModule(implorganization.NewStore(store)) providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - userModule := impluser.NewModule(impluser.NewStore(store, providerSettings), nil, emailing, providerSettings) - user, apiErr := createTestUser(organizationModule, userModule) + sharder, _ := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + orgGetter := implorganization.NewGetter(implorganization.NewStore(store), sharder) + alertmanager, _ := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Provider: "signoz", Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, store, orgGetter) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(store, jwt, emailing, providerSettings, orgGetter, alertmanager) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) if apiErr != nil { t.Fatalf("could not create test user: %v", apiErr) } diff --git a/pkg/query-service/app/integrations/test_utils.go b/pkg/query-service/app/integrations/test_utils.go index 94a2954e87..9230a63322 100644 --- a/pkg/query-service/app/integrations/test_utils.go +++ b/pkg/query-service/app/integrations/test_utils.go @@ -30,7 +30,7 @@ func NewTestIntegrationsManager(t *testing.T) (*Manager, sqlstore.SQLStore) { }, testDB } -func createTestUser(organizationModule organization.Module, userModule user.Module) (*types.User, *model.ApiError) { +func createTestUser(organizationModule organization.Setter, userModule user.Module) (*types.User, *model.ApiError) { // Create a test user for auth ctx := context.Background() organization := types.NewOrganization("test") diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index b060e47000..3f25901a81 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -15,6 +15,7 @@ import ( "github.com/SigNoz/signoz/pkg/apis/fields" "github.com/SigNoz/signoz/pkg/http/middleware" "github.com/SigNoz/signoz/pkg/licensing/nooplicensing" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/query-service/agentConf" "github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader" @@ -101,6 +102,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.SigNoz.SQLStore, serverOptions.SigNoz.TelemetryStore, serverOptions.SigNoz.Prometheus, + serverOptions.SigNoz.Modules.OrgGetter, ) if err != nil { return nil, err @@ -194,7 +196,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { &opAmpModel.AllAgents, agentConfMgr, ) - orgs, err := apiHandler.Signoz.Modules.Organization.GetAll(context.Background()) + orgs, err := apiHandler.Signoz.Modules.OrgGetter.ListByOwnedKeyRange(context.Background()) if err != nil { return nil, err } @@ -212,14 +214,14 @@ func (s *Server) createPrivateServer(api *APIHandler) (*http.Server, error) { r := NewRouter() - r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) + r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, s.serverOptions.SigNoz.Sharder, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) r.Use(middleware.NewTimeout(s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.Config.APIServer.Timeout.ExcludedRoutes, s.serverOptions.Config.APIServer.Timeout.Default, s.serverOptions.Config.APIServer.Timeout.Max, ).Wrap) r.Use(middleware.NewAnalytics().Wrap) - r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) + r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.SigNoz.Sharder).Wrap) r.Use(middleware.NewLogging(s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.Config.APIServer.Logging.ExcludedRoutes).Wrap) api.RegisterPrivateRoutes(r) @@ -243,14 +245,14 @@ func (s *Server) createPrivateServer(api *APIHandler) (*http.Server, error) { func (s *Server) createPublicServer(api *APIHandler, web web.Web) (*http.Server, error) { r := NewRouter() - r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) + r.Use(middleware.NewAuth(s.serverOptions.Jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, s.serverOptions.SigNoz.Sharder, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) r.Use(middleware.NewTimeout(s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.Config.APIServer.Timeout.ExcludedRoutes, s.serverOptions.Config.APIServer.Timeout.Default, s.serverOptions.Config.APIServer.Timeout.Max, ).Wrap) r.Use(middleware.NewAnalytics().Wrap) - r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger()).Wrap) + r.Use(middleware.NewAPIKey(s.serverOptions.SigNoz.SQLStore, []string{"SIGNOZ-API-KEY"}, s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.SigNoz.Sharder).Wrap) r.Use(middleware.NewLogging(s.serverOptions.SigNoz.Instrumentation.Logger(), s.serverOptions.Config.APIServer.Logging.ExcludedRoutes).Wrap) am := middleware.NewAuthZ(s.serverOptions.SigNoz.Instrumentation.Logger()) @@ -416,6 +418,7 @@ func makeRulesManager( sqlstore sqlstore.SQLStore, telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, + orgGetter organization.Getter, ) (*rules.Manager, error) { // create manager opts managerOpts := &rules.ManagerOptions{ @@ -428,6 +431,7 @@ func makeRulesManager( Cache: cache, EvalDelay: constants.GetEvalDelay(), SQLStore: sqlstore, + OrgGetter: orgGetter, } // create Manager diff --git a/pkg/query-service/auth/auth.go b/pkg/query-service/auth/auth.go deleted file mode 100644 index d157083d4e..0000000000 --- a/pkg/query-service/auth/auth.go +++ /dev/null @@ -1,65 +0,0 @@ -package auth - -import ( - "context" - - "github.com/SigNoz/signoz/pkg/alertmanager" - "github.com/SigNoz/signoz/pkg/modules/organization" - "github.com/SigNoz/signoz/pkg/modules/quickfilter" - "github.com/SigNoz/signoz/pkg/modules/user" - "github.com/SigNoz/signoz/pkg/valuer" - - "github.com/SigNoz/signoz/pkg/query-service/model" - "github.com/SigNoz/signoz/pkg/types" -) - -func RegisterOrgAndFirstUser(ctx context.Context, req *types.PostableRegisterOrgAndAdmin, organizationModule organization.Module, userModule user.Module) (*types.User, *model.ApiError) { - if req.Email == "" { - return nil, model.BadRequest(model.ErrEmailRequired{}) - } - - if req.Password == "" { - return nil, model.BadRequest(model.ErrPasswordRequired{}) - } - - organization := types.NewOrganization(req.OrgDisplayName) - err := organizationModule.Create(ctx, organization) - if err != nil { - return nil, model.InternalError(err) - } - - user, err := types.NewUser(req.Name, req.Email, types.RoleAdmin.String(), organization.ID.StringValue()) - if err != nil { - return nil, model.InternalError(err) - } - - password, err := types.NewFactorPassword(req.Password) - if err != nil { - return nil, model.InternalError(err) - } - - user, err = userModule.CreateUserWithPassword(ctx, user, password) - if err != nil { - return nil, model.InternalError(err) - } - - return user, nil -} - -// First user registration -func Register(ctx context.Context, req *types.PostableRegisterOrgAndAdmin, alertmanager alertmanager.Alertmanager, organizationModule organization.Module, userModule user.Module, quickfiltermodule quickfilter.Module) (*types.User, *model.ApiError) { - user, err := RegisterOrgAndFirstUser(ctx, req, organizationModule, userModule) - if err != nil { - return nil, err - } - - if err := alertmanager.SetDefaultConfig(ctx, user.OrgID); err != nil { - return nil, model.InternalError(err) - } - - if err := quickfiltermodule.SetDefaultConfig(ctx, valuer.MustNewUUID(user.OrgID)); err != nil { - return nil, model.InternalError(err) - } - - return user, nil -} diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index 18565541ae..ddfb28002b 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -12,6 +12,7 @@ import ( "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/licensing" "github.com/SigNoz/signoz/pkg/licensing/nooplicensing" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/query-service/app" "github.com/SigNoz/signoz/pkg/query-service/constants" "github.com/SigNoz/signoz/pkg/signoz" @@ -121,7 +122,7 @@ func main() { zeus.Config{}, noopzeus.NewProviderFactory(), licensing.Config{}, - func(_ sqlstore.SQLStore, _ zeus.Zeus) factory.ProviderFactory[licensing.Licensing, licensing.Config] { + func(_ sqlstore.SQLStore, _ zeus.Zeus, _ organization.Getter) factory.ProviderFactory[licensing.Licensing, licensing.Config] { return nooplicensing.NewFactory() }, signoz.NewEmailingProviderFactories(), diff --git a/pkg/query-service/model/errors.go b/pkg/query-service/model/errors.go deleted file mode 100644 index d0b95d279c..0000000000 --- a/pkg/query-service/model/errors.go +++ /dev/null @@ -1,36 +0,0 @@ -package model - -import "fmt" - -// custom errors related to registration -type ErrFeatureUnavailable struct { - Key string -} - -func (errFeatureUnavailable ErrFeatureUnavailable) Error() string { - return fmt.Sprintf("feature unavailable: %s", errFeatureUnavailable.Key) -} - -type ErrEmailRequired struct{} - -func (errEmailRequired ErrEmailRequired) Error() string { - return "email is required" -} - -type ErrPasswordRequired struct{} - -func (errPasswordRequired ErrPasswordRequired) Error() string { - return "password is required" -} - -type ErrSignupFailed struct{} - -func (errSignupFailed ErrSignupFailed) Error() string { - return "failed to register user" -} - -type ErrNoOrgFound struct{} - -func (errNoOrgFound ErrNoOrgFound) Error() string { - return "no org found" -} diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index 1d13bfc53d..b7f3f20fb1 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -19,6 +19,7 @@ import ( "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/cache" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/query-service/interfaces" "github.com/SigNoz/signoz/pkg/query-service/model" @@ -95,6 +96,7 @@ type ManagerOptions struct { PrepareTestRuleFunc func(opts PrepareTestRuleOptions) (int, *model.ApiError) Alertmanager alertmanager.Alertmanager SQLStore sqlstore.SQLStore + OrgGetter organization.Getter } // The Manager manages recording and alerting rules. @@ -116,6 +118,7 @@ type Manager struct { alertmanager alertmanager.Alertmanager sqlstore sqlstore.SQLStore + orgGetter organization.Getter } func defaultOptions(o *ManagerOptions) *ManagerOptions { @@ -210,6 +213,7 @@ func NewManager(o *ManagerOptions) (*Manager, error) { prepareTestRuleFunc: o.PrepareTestRuleFunc, alertmanager: o.Alertmanager, sqlstore: o.SQLStore, + orgGetter: o.OrgGetter, } return m, nil @@ -239,14 +243,14 @@ func (m *Manager) Pause(b bool) { } func (m *Manager) initiate(ctx context.Context) error { - orgIDs, err := m.ruleStore.ListOrgs(ctx) + orgs, err := m.orgGetter.ListByOwnedKeyRange(ctx) if err != nil { return err } var loadErrors []error - for _, orgID := range orgIDs { - storedRules, err := m.ruleStore.GetStoredRules(ctx, orgID.StringValue()) + for _, org := range orgs { + storedRules, err := m.ruleStore.GetStoredRules(ctx, org.ID.StringValue()) if err != nil { return err } @@ -279,7 +283,7 @@ func (m *Manager) initiate(ctx context.Context) error { } } if !parsedRule.Disabled { - err := m.addTask(ctx, orgID, parsedRule, taskName) + err := m.addTask(ctx, org.ID, parsedRule, taskName) if err != nil { zap.L().Error("failed to load the rule definition", zap.String("name", taskName), zap.Error(err)) } diff --git a/pkg/query-service/tests/integration/filter_suggestions_test.go b/pkg/query-service/tests/integration/filter_suggestions_test.go index 21665bb3d2..22dda58f07 100644 --- a/pkg/query-service/tests/integration/filter_suggestions_test.go +++ b/pkg/query-service/tests/integration/filter_suggestions_test.go @@ -11,8 +11,12 @@ import ( "testing" "time" - "github.com/SigNoz/signoz/pkg/emailing" - "github.com/SigNoz/signoz/pkg/emailing/noopemailing" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/emailing/emailingtest" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/http/middleware" @@ -304,16 +308,22 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed { mockClickhouse.MatchExpectationsInOrder(false) providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(t, err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(testDB), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, testDB, orgGetter) + require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) - modules := signoz.NewModules(testDB, jwt, emailing, providerSettings) + emailing := emailingtest.New() + modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager) + handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ Reader: reader, JWT: jwt, Signoz: &signoz.SigNoz{ Modules: modules, - Handlers: signoz.NewHandlers(modules), + Handlers: handlers, }, }) if err != nil { @@ -322,13 +332,12 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed { router := app.NewRouter() //add the jwt middleware - router.Use(middleware.NewAuth(jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) + router.Use(middleware.NewAuth(jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, sharder, instrumentationtest.New().Logger()).Wrap) am := middleware.NewAuthZ(instrumentationtest.New().Logger()) apiHandler.RegisterRoutes(router, am) apiHandler.RegisterQueryRangeV3Routes(router, am) - organizationModule := implorganization.NewModule(implorganization.NewStore(testDB)) - user, apiErr := createTestUser(organizationModule, modules.User) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) if apiErr != nil { t.Fatalf("could not create a test user: %v", apiErr) } diff --git a/pkg/query-service/tests/integration/logparsingpipeline_test.go b/pkg/query-service/tests/integration/logparsingpipeline_test.go index 69e4102ac3..8413d5c358 100644 --- a/pkg/query-service/tests/integration/logparsingpipeline_test.go +++ b/pkg/query-service/tests/integration/logparsingpipeline_test.go @@ -11,8 +11,10 @@ import ( "testing" "time" - "github.com/SigNoz/signoz/pkg/emailing" - "github.com/SigNoz/signoz/pkg/emailing/noopemailing" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" "github.com/SigNoz/signoz/pkg/modules/user" @@ -26,6 +28,8 @@ import ( v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" "github.com/SigNoz/signoz/pkg/query-service/queryBuilderToExpr" "github.com/SigNoz/signoz/pkg/query-service/utils" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/signoz" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/types" @@ -480,9 +484,14 @@ func NewTestbedWithoutOpamp(t *testing.T, sqlStore sqlstore.SQLStore) *LogPipeli } providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - jwt := authtypes.NewJWT("", 10*time.Minute, 30*time.Minute) - modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(t, err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlStore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, sqlStore, orgGetter) + require.NoError(t, err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ @@ -497,8 +506,7 @@ func NewTestbedWithoutOpamp(t *testing.T, sqlStore sqlstore.SQLStore) *LogPipeli t.Fatalf("could not create a new ApiHandler: %v", err) } - organizationModule := implorganization.NewModule(implorganization.NewStore(sqlStore)) - user, apiErr := createTestUser(organizationModule, modules.User) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) if apiErr != nil { t.Fatalf("could not create a test user: %v", apiErr) } diff --git a/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go b/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go index 8ea9d7a974..31197263cd 100644 --- a/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_cloud_integrations_test.go @@ -9,8 +9,12 @@ import ( "testing" "time" - "github.com/SigNoz/signoz/pkg/emailing" - "github.com/SigNoz/signoz/pkg/emailing/noopemailing" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/emailing/emailingtest" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/http/middleware" @@ -365,9 +369,14 @@ func NewCloudIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *CloudI mockClickhouse.MatchExpectationsInOrder(false) providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - jwt := authtypes.NewJWT("", 10*time.Minute, 30*time.Minute) - modules := signoz.NewModules(testDB, jwt, emailing, providerSettings) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(t, err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(testDB), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, testDB, orgGetter) + require.NoError(t, err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ @@ -384,13 +393,12 @@ func NewCloudIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *CloudI } router := app.NewRouter() - router.Use(middleware.NewAuth(jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) + router.Use(middleware.NewAuth(jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, sharder, instrumentationtest.New().Logger()).Wrap) am := middleware.NewAuthZ(instrumentationtest.New().Logger()) apiHandler.RegisterRoutes(router, am) apiHandler.RegisterCloudIntegrationsRoutes(router, am) - organizationModule := implorganization.NewModule(implorganization.NewStore(testDB)) - user, apiErr := createTestUser(organizationModule, modules.User) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) if apiErr != nil { t.Fatalf("could not create a test user: %v", apiErr) } diff --git a/pkg/query-service/tests/integration/signoz_integrations_test.go b/pkg/query-service/tests/integration/signoz_integrations_test.go index 1b221267fb..a4aad441c4 100644 --- a/pkg/query-service/tests/integration/signoz_integrations_test.go +++ b/pkg/query-service/tests/integration/signoz_integrations_test.go @@ -9,9 +9,10 @@ import ( "testing" "time" - "github.com/SigNoz/signoz/pkg/emailing" - "github.com/SigNoz/signoz/pkg/emailing/noopemailing" - + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" + "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/http/middleware" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" @@ -22,6 +23,8 @@ import ( "github.com/SigNoz/signoz/pkg/query-service/model" v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" "github.com/SigNoz/signoz/pkg/query-service/utils" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/signoz" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/types" @@ -571,9 +574,14 @@ func NewIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *Integration } providerSettings := instrumentationtest.New().ToProviderSettings() - emailing, _ := noopemailing.New(context.Background(), providerSettings, emailing.Config{}) - jwt := authtypes.NewJWT("", 10*time.Minute, 30*time.Minute) - modules := signoz.NewModules(testDB, jwt, emailing, providerSettings) + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(t, err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(testDB), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{Signoz: alertmanager.Signoz{PollInterval: 10 * time.Second, Config: alertmanagerserver.NewConfig()}}, testDB, orgGetter) + require.NoError(t, err) + jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) + emailing := emailingtest.New() + modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager) handlers := signoz.NewHandlers(modules) apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{ @@ -592,13 +600,12 @@ func NewIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *Integration } router := app.NewRouter() - router.Use(middleware.NewAuth(jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}).Wrap) + router.Use(middleware.NewAuth(jwt, []string{"Authorization", "Sec-WebSocket-Protocol"}, sharder, instrumentationtest.New().Logger()).Wrap) am := middleware.NewAuthZ(instrumentationtest.New().Logger()) apiHandler.RegisterRoutes(router, am) apiHandler.RegisterIntegrationRoutes(router, am) - organizationModule := implorganization.NewModule(implorganization.NewStore(testDB)) - user, apiErr := createTestUser(organizationModule, modules.User) + user, apiErr := createTestUser(modules.OrgSetter, modules.User) if apiErr != nil { t.Fatalf("could not create a test user: %v", apiErr) } diff --git a/pkg/query-service/tests/integration/test_utils.go b/pkg/query-service/tests/integration/test_utils.go index 869d6eeda7..90fb293005 100644 --- a/pkg/query-service/tests/integration/test_utils.go +++ b/pkg/query-service/tests/integration/test_utils.go @@ -147,11 +147,11 @@ func makeTestSignozLog( return testLog } -func createTestUser(organizationModule organization.Module, userModule user.Module) (*types.User, *model.ApiError) { +func createTestUser(orgSetter organization.Setter, userModule user.Module) (*types.User, *model.ApiError) { // Create a test user for auth ctx := context.Background() organization := types.NewOrganization("test") - err := organizationModule.Create(ctx, organization) + err := orgSetter.Create(ctx, organization) if err != nil { return nil, model.InternalError(err) } diff --git a/pkg/query-service/utils/testutils.go b/pkg/query-service/utils/testutils.go index 5cd7960786..c429f44378 100644 --- a/pkg/query-service/utils/testutils.go +++ b/pkg/query-service/utils/testutils.go @@ -67,6 +67,7 @@ func NewTestSqliteDB(t *testing.T) (sqlStore sqlstore.SQLStore, testDBFilePath s sqlmigration.NewAuthRefactorFactory(sqlStore), sqlmigration.NewMigratePATToFactorAPIKey(sqlStore), sqlmigration.NewUpdateApiMonitoringFiltersFactory(sqlStore), + sqlmigration.NewAddKeyOrganizationFactory(sqlStore), ), ) if err != nil { diff --git a/pkg/ruler/rulestore/sqlrulestore/rule.go b/pkg/ruler/rulestore/sqlrulestore/rule.go index 735cce6b20..68093eb233 100644 --- a/pkg/ruler/rulestore/sqlrulestore/rule.go +++ b/pkg/ruler/rulestore/sqlrulestore/rule.go @@ -4,7 +4,6 @@ import ( "context" "github.com/SigNoz/signoz/pkg/sqlstore" - "github.com/SigNoz/signoz/pkg/types" ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes" "github.com/SigNoz/signoz/pkg/valuer" "github.com/jmoiron/sqlx" @@ -118,27 +117,3 @@ func (r *rule) GetRuleUUID(ctx context.Context, ruleID int) (*ruletypes.RuleHist } return ruleHistory, nil } - -func (r *rule) ListOrgs(ctx context.Context) ([]valuer.UUID, error) { - orgIDStrs := make([]string, 0) - err := r.sqlstore. - BunDB(). - NewSelect(). - Model(new(types.Organization)). - Column("id"). - Scan(ctx, &orgIDStrs) - if err != nil { - return nil, err - } - - orgIDs := make([]valuer.UUID, len(orgIDStrs)) - for idx, orgIDStr := range orgIDStrs { - orgID, err := valuer.NewUUID(orgIDStr) - if err != nil { - return nil, err - } - orgIDs[idx] = orgID - } - - return orgIDs, nil -} diff --git a/pkg/sharder/config.go b/pkg/sharder/config.go new file mode 100644 index 0000000000..785d5545c1 --- /dev/null +++ b/pkg/sharder/config.go @@ -0,0 +1,32 @@ +package sharder + +import ( + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/valuer" +) + +type Config struct { + Provider string `mapstructure:"provider"` + Single Single `mapstructure:"single"` +} + +type Single struct { + OrgID valuer.UUID `mapstructure:"org_id"` +} + +func NewConfigFactory() factory.ConfigFactory { + return factory.NewConfigFactory(factory.MustNewName("sharder"), newConfig) +} + +func newConfig() factory.Config { + return &Config{ + Provider: "noop", + Single: Single{ + OrgID: valuer.UUID{}, + }, + } +} + +func (c Config) Validate() error { + return nil +} diff --git a/pkg/sharder/noopsharder/provider.go b/pkg/sharder/noopsharder/provider.go new file mode 100644 index 0000000000..b7c3d02a7d --- /dev/null +++ b/pkg/sharder/noopsharder/provider.go @@ -0,0 +1,33 @@ +package noopsharder + +import ( + "context" + "math" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/sharder" +) + +type provider struct { + settings factory.ScopedProviderSettings +} + +func NewFactory() factory.ProviderFactory[sharder.Sharder, sharder.Config] { + return factory.NewProviderFactory(factory.MustNewName("noop"), New) +} + +func New(ctx context.Context, providerSettings factory.ProviderSettings, config sharder.Config) (sharder.Sharder, error) { + settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/sharder/noopsharder") + + return &provider{ + settings: settings, + }, nil +} + +func (provider *provider) GetMyOwnedKeyRange(ctx context.Context) (uint32, uint32, error) { + return 0, math.MaxUint32, nil +} + +func (provider *provider) IsMyOwnedKey(ctx context.Context, key uint32) error { + return nil +} diff --git a/pkg/sharder/sharder.go b/pkg/sharder/sharder.go new file mode 100644 index 0000000000..b627ef37a5 --- /dev/null +++ b/pkg/sharder/sharder.go @@ -0,0 +1,13 @@ +package sharder + +import ( + "context" +) + +type Sharder interface { + // Returns the keys owned by the current instance. + GetMyOwnedKeyRange(context.Context) (uint32, uint32, error) + + // Returns true if the key is owned by the current instance. + IsMyOwnedKey(context.Context, uint32) error +} diff --git a/pkg/sharder/singlesharder/provider.go b/pkg/sharder/singlesharder/provider.go new file mode 100644 index 0000000000..86b7d7197d --- /dev/null +++ b/pkg/sharder/singlesharder/provider.go @@ -0,0 +1,43 @@ +package singlesharder + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/types" + "github.com/SigNoz/signoz/pkg/valuer" +) + +type provider struct { + settings factory.ScopedProviderSettings + orgID valuer.UUID + orgIDKey uint32 +} + +func NewFactory() factory.ProviderFactory[sharder.Sharder, sharder.Config] { + return factory.NewProviderFactory(factory.MustNewName("single"), New) +} + +func New(ctx context.Context, providerSettings factory.ProviderSettings, config sharder.Config) (sharder.Sharder, error) { + settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/sharder/singlesharder") + + return &provider{ + settings: settings, + orgID: config.Single.OrgID, + orgIDKey: types.NewOrganizationKey(config.Single.OrgID), + }, nil +} + +func (provider *provider) GetMyOwnedKeyRange(ctx context.Context) (uint32, uint32, error) { + return provider.orgIDKey, provider.orgIDKey, nil +} + +func (provider *provider) IsMyOwnedKey(ctx context.Context, key uint32) error { + if key == provider.orgIDKey { + return nil + } + + return errors.Newf(errors.TypeForbidden, errors.CodeForbidden, "key %d for org %s is not owned by my current instance", key, provider.orgID) +} diff --git a/pkg/signoz/config.go b/pkg/signoz/config.go index 4fb3bbf142..2554fb3279 100644 --- a/pkg/signoz/config.go +++ b/pkg/signoz/config.go @@ -17,6 +17,7 @@ import ( "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/instrumentation" "github.com/SigNoz/signoz/pkg/prometheus" + "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sqlmigration" "github.com/SigNoz/signoz/pkg/sqlmigrator" "github.com/SigNoz/signoz/pkg/sqlstore" @@ -62,6 +63,9 @@ type Config struct { // Emailing config Emailing emailing.Config `mapstructure:"emailing" yaml:"emailing"` + + // Sharder config + Sharder sharder.Config `mapstructure:"sharder" yaml:"sharder"` } // DeprecatedFlags are the flags that are deprecated and scheduled for removal. @@ -86,6 +90,7 @@ func NewConfig(ctx context.Context, resolverConfig config.ResolverConfig, deprec prometheus.NewConfigFactory(), alertmanager.NewConfigFactory(), emailing.NewConfigFactory(), + sharder.NewConfigFactory(), } conf, err := config.New(ctx, resolverConfig, configFactories) diff --git a/pkg/signoz/handler.go b/pkg/signoz/handler.go index 2b1512b63a..f0d1528ec1 100644 --- a/pkg/signoz/handler.go +++ b/pkg/signoz/handler.go @@ -29,7 +29,7 @@ type Handlers struct { func NewHandlers(modules Modules) Handlers { return Handlers{ - Organization: implorganization.NewHandler(modules.Organization), + Organization: implorganization.NewHandler(modules.OrgGetter, modules.OrgSetter), Preference: implpreference.NewHandler(modules.Preference), User: impluser.NewHandler(modules.User), SavedView: implsavedview.NewHandler(modules.SavedView), diff --git a/pkg/signoz/handler_test.go b/pkg/signoz/handler_test.go index b5ebd97d3e..0844ebf243 100644 --- a/pkg/signoz/handler_test.go +++ b/pkg/signoz/handler_test.go @@ -1,28 +1,40 @@ package signoz import ( + "context" "reflect" "testing" "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/factory/factorytest" + "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // This is a test to ensure that all fields of the handlers are initialized. // It also helps us catch these errors at compile time instead of runtime. func TestNewHandlers(t *testing.T) { sqlstore := sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual) + providerSettings := factorytest.NewSettings() + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(t, err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlstore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{}, sqlstore, orgGetter) + require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - providerSettings := factorytest.NewSettings() + modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager) - modules := NewModules(sqlstore, jwt, emailing, providerSettings) handlers := NewHandlers(modules) reflectVal := reflect.ValueOf(handlers) diff --git a/pkg/signoz/module.go b/pkg/signoz/module.go index 0abdff2f85..c4f45e8492 100644 --- a/pkg/signoz/module.go +++ b/pkg/signoz/module.go @@ -1,6 +1,7 @@ package signoz import ( + "github.com/SigNoz/signoz/pkg/alertmanager" "github.com/SigNoz/signoz/pkg/emailing" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/modules/apdex" @@ -23,23 +24,35 @@ import ( ) type Modules struct { - Organization organization.Module - Preference preference.Module - User user.Module - SavedView savedview.Module - Apdex apdex.Module - Dashboard dashboard.Module - QuickFilter quickfilter.Module + OrgGetter organization.Getter + OrgSetter organization.Setter + Preference preference.Module + User user.Module + SavedView savedview.Module + Apdex apdex.Module + Dashboard dashboard.Module + QuickFilter quickfilter.Module } -func NewModules(sqlstore sqlstore.SQLStore, jwt *authtypes.JWT, emailing emailing.Emailing, providerSettings factory.ProviderSettings) Modules { +func NewModules( + sqlstore sqlstore.SQLStore, + jwt *authtypes.JWT, + emailing emailing.Emailing, + providerSettings factory.ProviderSettings, + orgGetter organization.Getter, + alertmanager alertmanager.Alertmanager, +) Modules { + quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore)) + orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter) + user := impluser.NewModule(impluser.NewStore(sqlstore, providerSettings), jwt, emailing, providerSettings, orgSetter) return Modules{ - Organization: implorganization.NewModule(implorganization.NewStore(sqlstore)), - Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewDefaultPreferenceMap()), - SavedView: implsavedview.NewModule(sqlstore), - Apdex: implapdex.NewModule(sqlstore), - Dashboard: impldashboard.NewModule(sqlstore), - User: impluser.NewModule(impluser.NewStore(sqlstore, providerSettings), jwt, emailing, providerSettings), - QuickFilter: implquickfilter.NewModule(implquickfilter.NewStore(sqlstore)), + OrgGetter: orgGetter, + OrgSetter: orgSetter, + Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewDefaultPreferenceMap()), + SavedView: implsavedview.NewModule(sqlstore), + Apdex: implapdex.NewModule(sqlstore), + Dashboard: impldashboard.NewModule(sqlstore), + User: user, + QuickFilter: quickfilter, } } diff --git a/pkg/signoz/module_test.go b/pkg/signoz/module_test.go index 67f6aa23b6..f18596b059 100644 --- a/pkg/signoz/module_test.go +++ b/pkg/signoz/module_test.go @@ -1,27 +1,39 @@ package signoz import ( + "context" "reflect" "testing" "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/SigNoz/signoz/pkg/alertmanager" + "github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager" "github.com/SigNoz/signoz/pkg/emailing/emailingtest" "github.com/SigNoz/signoz/pkg/factory/factorytest" + "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest" "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // This is a test to ensure that all fields of the modules are initialized. // It also helps us catch these errors at compile time instead of runtime. func TestNewModules(t *testing.T) { sqlstore := sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual) + providerSettings := factorytest.NewSettings() + sharder, err := noopsharder.New(context.TODO(), providerSettings, sharder.Config{}) + require.NoError(t, err) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlstore), sharder) + alertmanager, err := signozalertmanager.New(context.TODO(), providerSettings, alertmanager.Config{}, sqlstore, orgGetter) + require.NoError(t, err) jwt := authtypes.NewJWT("", 1*time.Hour, 1*time.Hour) emailing := emailingtest.New() - providerSettings := factorytest.NewSettings() - modules := NewModules(sqlstore, jwt, emailing, providerSettings) + modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager) reflectVal := reflect.ValueOf(modules) for i := 0; i < reflectVal.NumField(); i++ { diff --git a/pkg/signoz/provider.go b/pkg/signoz/provider.go index 831d2e2a62..bfbcdeafe1 100644 --- a/pkg/signoz/provider.go +++ b/pkg/signoz/provider.go @@ -11,8 +11,12 @@ import ( "github.com/SigNoz/signoz/pkg/emailing/noopemailing" "github.com/SigNoz/signoz/pkg/emailing/smtpemailing" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus" + "github.com/SigNoz/signoz/pkg/sharder" + "github.com/SigNoz/signoz/pkg/sharder/noopsharder" + "github.com/SigNoz/signoz/pkg/sharder/singlesharder" "github.com/SigNoz/signoz/pkg/sqlmigration" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore" @@ -83,6 +87,7 @@ func NewSQLMigrationProviderFactories(sqlstore sqlstore.SQLStore) factory.NamedM sqlmigration.NewUpdateLicenseFactory(sqlstore), sqlmigration.NewMigratePATToFactorAPIKey(sqlstore), sqlmigration.NewUpdateApiMonitoringFiltersFactory(sqlstore), + sqlmigration.NewAddKeyOrganizationFactory(sqlstore), ) } @@ -98,10 +103,10 @@ func NewPrometheusProviderFactories(telemetryStore telemetrystore.TelemetryStore ) } -func NewAlertmanagerProviderFactories(sqlstore sqlstore.SQLStore) factory.NamedMap[factory.ProviderFactory[alertmanager.Alertmanager, alertmanager.Config]] { +func NewAlertmanagerProviderFactories(sqlstore sqlstore.SQLStore, orgGetter organization.Getter) factory.NamedMap[factory.ProviderFactory[alertmanager.Alertmanager, alertmanager.Config]] { return factory.MustNewNamedMap( - legacyalertmanager.NewFactory(sqlstore), - signozalertmanager.NewFactory(sqlstore), + legacyalertmanager.NewFactory(sqlstore, orgGetter), + signozalertmanager.NewFactory(sqlstore, orgGetter), ) } @@ -111,3 +116,10 @@ func NewEmailingProviderFactories() factory.NamedMap[factory.ProviderFactory[ema smtpemailing.NewFactory(), ) } + +func NewSharderProviderFactories() factory.NamedMap[factory.ProviderFactory[sharder.Sharder, sharder.Config]] { + return factory.MustNewNamedMap( + singlesharder.NewFactory(), + noopsharder.NewFactory(), + ) +} diff --git a/pkg/signoz/provider_test.go b/pkg/signoz/provider_test.go index 7245d50f66..752ff19db8 100644 --- a/pkg/signoz/provider_test.go +++ b/pkg/signoz/provider_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest" "github.com/SigNoz/signoz/pkg/telemetrystore" @@ -40,10 +41,15 @@ func TestNewProviderFactories(t *testing.T) { }) assert.NotPanics(t, func() { - NewAlertmanagerProviderFactories(sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual)) + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual)), nil) + NewAlertmanagerProviderFactories(sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherEqual), orgGetter) }) assert.NotPanics(t, func() { NewEmailingProviderFactories() }) + + assert.NotPanics(t, func() { + NewSharderProviderFactories() + }) } diff --git a/pkg/signoz/signoz.go b/pkg/signoz/signoz.go index 748b4af9b4..919520b5b3 100644 --- a/pkg/signoz/signoz.go +++ b/pkg/signoz/signoz.go @@ -9,7 +9,10 @@ import ( "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/instrumentation" "github.com/SigNoz/signoz/pkg/licensing" + "github.com/SigNoz/signoz/pkg/modules/organization" + "github.com/SigNoz/signoz/pkg/modules/organization/implorganization" "github.com/SigNoz/signoz/pkg/prometheus" + "github.com/SigNoz/signoz/pkg/sharder" "github.com/SigNoz/signoz/pkg/sqlmigration" "github.com/SigNoz/signoz/pkg/sqlmigrator" "github.com/SigNoz/signoz/pkg/sqlstore" @@ -33,6 +36,7 @@ type SigNoz struct { Zeus zeus.Zeus Licensing licensing.Licensing Emailing emailing.Emailing + Sharder sharder.Sharder Modules Modules Handlers Handlers } @@ -44,7 +48,7 @@ func New( zeusConfig zeus.Config, zeusProviderFactory factory.ProviderFactory[zeus.Zeus, zeus.Config], licenseConfig licensing.Config, - licenseProviderFactoryCb func(sqlstore.SQLStore, zeus.Zeus) factory.ProviderFactory[licensing.Licensing, licensing.Config], + licenseProviderFactoryCb func(sqlstore.SQLStore, zeus.Zeus, organization.Getter) factory.ProviderFactory[licensing.Licensing, licensing.Config], emailingProviderFactories factory.NamedMap[factory.ProviderFactory[emailing.Emailing, emailing.Config]], cacheProviderFactories factory.NamedMap[factory.ProviderFactory[cache.Cache, cache.Config]], webProviderFactories factory.NamedMap[factory.ProviderFactory[web.Web, web.Config]], @@ -162,19 +166,34 @@ func New( return nil, err } + // Initialize sharder from the available sharder provider factories + sharder, err := factory.NewProviderFromNamedMap( + ctx, + providerSettings, + config.Sharder, + NewSharderProviderFactories(), + config.Sharder.Provider, + ) + if err != nil { + return nil, err + } + + // Initialize organization getter + orgGetter := implorganization.NewGetter(implorganization.NewStore(sqlstore), sharder) + // Initialize alertmanager from the available alertmanager provider factories alertmanager, err := factory.NewProviderFromNamedMap( ctx, providerSettings, config.Alertmanager, - NewAlertmanagerProviderFactories(sqlstore), + NewAlertmanagerProviderFactories(sqlstore, orgGetter), config.Alertmanager.Provider, ) if err != nil { return nil, err } - licensingProviderFactory := licenseProviderFactoryCb(sqlstore, zeus) + licensingProviderFactory := licenseProviderFactoryCb(sqlstore, zeus, orgGetter) licensing, err := licensingProviderFactory.New( ctx, providerSettings, @@ -185,7 +204,7 @@ func New( } // Initialize all modules - modules := NewModules(sqlstore, jwt, emailing, providerSettings) + modules := NewModules(sqlstore, jwt, emailing, providerSettings, orgGetter, alertmanager) // Initialize all handlers for the modules handlers := NewHandlers(modules) @@ -212,6 +231,7 @@ func New( Zeus: zeus, Licensing: licensing, Emailing: emailing, + Sharder: sharder, Modules: modules, Handlers: handlers, }, nil diff --git a/pkg/sqlmigration/036_add_key_organization.go b/pkg/sqlmigration/036_add_key_organization.go new file mode 100644 index 0000000000..753736a0ed --- /dev/null +++ b/pkg/sqlmigration/036_add_key_organization.go @@ -0,0 +1,112 @@ +package sqlmigration + +import ( + "context" + "hash/fnv" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/uptrace/bun" + "github.com/uptrace/bun/migrate" +) + +type addKeyOrganization struct { + sqlstore sqlstore.SQLStore +} + +func NewAddKeyOrganizationFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] { + return factory.NewProviderFactory(factory.MustNewName("add_key_organization"), func(ctx context.Context, providerSettings factory.ProviderSettings, config Config) (SQLMigration, error) { + return newAddKeyOrganization(ctx, providerSettings, config, sqlstore) + }) +} + +func newAddKeyOrganization(_ context.Context, _ factory.ProviderSettings, _ Config, sqlstore sqlstore.SQLStore) (SQLMigration, error) { + return &addKeyOrganization{ + sqlstore: sqlstore, + }, nil +} + +func (migration *addKeyOrganization) Register(migrations *migrate.Migrations) error { + if err := migrations.Register(migration.Up, migration.Down); err != nil { + return err + } + + return nil +} + +func (migration *addKeyOrganization) Up(ctx context.Context, db *bun.DB) error { + ok, err := migration.sqlstore.Dialect().ColumnExists(ctx, db, "organizations", "key") + if err != nil { + return err + } + + if ok { + return nil + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + + defer func() { + _ = tx.Rollback() + }() + + if _, err := tx. + NewAddColumn(). + Table("organizations"). + ColumnExpr("key BIGINT"). + Exec(ctx); err != nil { + return err + } + + var existingOrgIDs []string + if err := tx.NewSelect(). + Table("organizations"). + Column("id"). + Scan(ctx, &existingOrgIDs); err != nil { + return err + } + + for _, orgID := range existingOrgIDs { + key := migration.getHash(ctx, orgID) + if _, err := tx. + NewUpdate(). + Table("organizations"). + Set("key = ?", key). + Where("id = ?", orgID). + Exec(ctx); err != nil { + return err + } + } + + if _, err := tx. + NewCreateIndex(). + Unique(). + IfNotExists(). + Index("idx_unique_key"). + Table("organizations"). + Column("key"). + Exec(ctx); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +func (migration *addKeyOrganization) Down(ctx context.Context, db *bun.DB) error { + return nil +} + +func (migration *addKeyOrganization) getHash(_ context.Context, orgID string) uint32 { + hasher := fnv.New32a() + + // Hasher never returns err. + _, _ = hasher.Write([]byte(orgID)) + return hasher.Sum32() +} diff --git a/pkg/types/alertmanagertypes/config.go b/pkg/types/alertmanagertypes/config.go index 21657fa1da..2cc7adf1bd 100644 --- a/pkg/types/alertmanagertypes/config.go +++ b/pkg/types/alertmanagertypes/config.go @@ -369,9 +369,6 @@ type ConfigStore interface { // Get returns the config for the given orgID Get(context.Context, string) (*Config, error) - // ListOrgs returns the list of orgs - ListOrgs(context.Context) ([]string, error) - // CreateChannel creates a new channel. CreateChannel(context.Context, *Channel, ...StoreOption) error diff --git a/pkg/types/apdextypes/settings.go b/pkg/types/apdextypes/settings.go new file mode 100644 index 0000000000..d08a1a7ee0 --- /dev/null +++ b/pkg/types/apdextypes/settings.go @@ -0,0 +1,15 @@ +package apdextypes + +import ( + "github.com/SigNoz/signoz/pkg/types" + "github.com/uptrace/bun" +) + +type Settings struct { + bun.BaseModel `bun:"table:apdex_setting"` + types.Identifiable + OrgID string `bun:"org_id,type:text" json:"orgId"` + ServiceName string `bun:"service_name,type:text" json:"serviceName"` + Threshold float64 `bun:"threshold,type:float,notnull" json:"threshold"` + ExcludeStatusCodes string `bun:"exclude_status_codes,type:text,notnull" json:"excludeStatusCodes"` +} diff --git a/pkg/types/licensetypes/license.go b/pkg/types/licensetypes/license.go index 55e4204238..e0a1a1a49e 100644 --- a/pkg/types/licensetypes/license.go +++ b/pkg/types/licensetypes/license.go @@ -383,7 +383,4 @@ type Store interface { GetFeature(context.Context, string) (*featuretypes.StorableFeature, error) GetAllFeatures(context.Context) ([]*featuretypes.StorableFeature, error) UpdateFeature(context.Context, *featuretypes.StorableFeature) error - - // ListOrganizations returns the list of orgs - ListOrganizations(context.Context) ([]valuer.UUID, error) } diff --git a/pkg/types/organization.go b/pkg/types/organization.go index d18e149890..2c177b2cbf 100644 --- a/pkg/types/organization.go +++ b/pkg/types/organization.go @@ -2,6 +2,7 @@ package types import ( "context" + "hash/fnv" "time" "github.com/SigNoz/signoz/pkg/errors" @@ -20,13 +21,15 @@ type Organization struct { Identifiable Name string `bun:"name,type:text,nullzero" json:"name"` Alias string `bun:"alias,type:text,nullzero" json:"alias"` + Key uint32 `bun:"key,type:bigint,notnull" json:"key"` DisplayName string `bun:"display_name,type:text,notnull" json:"displayName"` } func NewOrganization(displayName string) *Organization { + id := valuer.GenerateUUID() return &Organization{ Identifiable: Identifiable{ - ID: valuer.GenerateUUID(), + ID: id, }, TimeAuditable: TimeAuditable{ CreatedAt: time.Now(), @@ -34,22 +37,23 @@ func NewOrganization(displayName string) *Organization { }, // Name: "default/main", TODO: take the call and uncomment this later DisplayName: displayName, + Key: NewOrganizationKey(id), } } -type ApdexSettings struct { - bun.BaseModel `bun:"table:apdex_setting"` - Identifiable - OrgID string `bun:"org_id,type:text" json:"orgId"` - ServiceName string `bun:"service_name,type:text" json:"serviceName"` - Threshold float64 `bun:"threshold,type:float,notnull" json:"threshold"` - ExcludeStatusCodes string `bun:"exclude_status_codes,type:text,notnull" json:"excludeStatusCodes"` +func NewOrganizationKey(orgID valuer.UUID) uint32 { + hasher := fnv.New32a() + + // Hasher never returns err. + _, _ = hasher.Write([]byte(orgID.String())) + return hasher.Sum32() } type OrganizationStore interface { Create(context.Context, *Organization) error Get(context.Context, valuer.UUID) (*Organization, error) GetAll(context.Context) ([]*Organization, error) + ListByKeyRange(context.Context, uint32, uint32) ([]*Organization, error) Update(context.Context, *Organization) error Delete(context.Context, valuer.UUID) error } diff --git a/pkg/types/ruletypes/rule.go b/pkg/types/ruletypes/rule.go index 55fcc801e6..ffe5774f27 100644 --- a/pkg/types/ruletypes/rule.go +++ b/pkg/types/ruletypes/rule.go @@ -31,5 +31,4 @@ type RuleStore interface { GetStoredRules(context.Context, string) ([]*Rule, error) GetStoredRule(context.Context, valuer.UUID) (*Rule, error) GetRuleUUID(context.Context, int) (*RuleHistory, error) - ListOrgs(context.Context) ([]valuer.UUID, error) } diff --git a/pkg/valuer/string.go b/pkg/valuer/string.go index 18259ed0c9..ce2b5f7ea4 100644 --- a/pkg/valuer/string.go +++ b/pkg/valuer/string.go @@ -67,3 +67,8 @@ func (enum *String) Scan(val interface{}) error { *enum = NewString(str) return nil } + +func (enum *String) UnmarshalText(text []byte) error { + *enum = NewString(string(text)) + return nil +} diff --git a/pkg/valuer/uuid.go b/pkg/valuer/uuid.go index 35b49cda8a..a12979e0ce 100644 --- a/pkg/valuer/uuid.go +++ b/pkg/valuer/uuid.go @@ -122,3 +122,13 @@ func (enum *UUID) Scan(val interface{}) error { *enum = enumVal return nil } + +func (enum *UUID) UnmarshalText(text []byte) error { + uuid, err := NewUUID(string(text)) + if err != nil { + return err + } + + *enum = uuid + return nil +} diff --git a/pkg/valuer/valuer.go b/pkg/valuer/valuer.go index f069da04d8..7e65ac2857 100644 --- a/pkg/valuer/valuer.go +++ b/pkg/valuer/valuer.go @@ -3,6 +3,7 @@ package valuer import ( "database/sql" "database/sql/driver" + "encoding" "encoding/json" "fmt" ) @@ -28,4 +29,7 @@ type Valuer interface { // Implement fmt.Stringer to allow the value to be printed as a string fmt.Stringer + + // Implement encoding.TextUnmarshaler to allow the value to be unmarshalled from a string + encoding.TextUnmarshaler }