From c574adc6341889c38b00e8ede709403858782484 Mon Sep 17 00:00:00 2001 From: Vibhu Pandey Date: Fri, 17 Jan 2025 15:54:48 +0530 Subject: [PATCH] feat(sqlstore): add sqlstore package (#6835) ### Summary Add `sqlstore` package --- go.mod | 9 +++- go.sum | 18 +++++++- pkg/sqlstore/config.go | 45 ++++++++++++++++++ pkg/sqlstore/sqlitesqlstore/provider.go | 55 ++++++++++++++++++++++ pkg/sqlstore/sqlstore.go | 18 ++++++++ pkg/sqlstore/sqlstoretest/provider.go | 61 +++++++++++++++++++++++++ 6 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 pkg/sqlstore/config.go create mode 100644 pkg/sqlstore/sqlitesqlstore/provider.go create mode 100644 pkg/sqlstore/sqlstore.go create mode 100644 pkg/sqlstore/sqlstoretest/provider.go diff --git a/go.mod b/go.mod index 0f0ea1d907..fd4ae47279 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,8 @@ require ( github.com/soheilhy/cmux v0.1.5 github.com/srikanthccv/ClickHouse-go-mock v0.9.0 github.com/stretchr/testify v1.9.0 + github.com/uptrace/bun v1.2.8 + github.com/uptrace/bun/dialect/sqlitedialect v1.2.8 go.opentelemetry.io/collector/confmap v1.17.0 go.opentelemetry.io/collector/pdata v1.17.0 go.opentelemetry.io/collector/processor v0.111.0 @@ -120,6 +122,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -151,6 +154,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/backo-go v1.0.1 // indirect @@ -162,8 +166,11 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.13 // indirect github.com/tklauser/numcpus v0.7.0 // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/vjeantet/grok v1.0.1 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect @@ -212,7 +219,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/atomic v1.11.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/time v0.6.0 // indirect gonum.org/v1/gonum v0.15.1 // indirect google.golang.org/api v0.199.0 // indirect diff --git a/go.sum b/go.sum index 06eb093e9c..31f0509fc0 100644 --- a/go.sum +++ b/go.sum @@ -436,6 +436,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ionos-cloud/sdk-go/v6 v6.2.1 h1:mxxN+frNVmbFrmmFfXnBC3g2USYJrl6mc1LW2iNYbFY= github.com/ionos-cloud/sdk-go/v6 v6.2.1/go.mod h1:SXrO9OGyWjd2rZhAhEpdYN6VUAODzzqRdqA9BCviQtI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -661,6 +663,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4= +github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= @@ -740,12 +744,22 @@ github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08 github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/uptrace/bun v1.2.8 h1:HEiLvy9wc7ehU5S02+O6NdV5BLz48lL4REPhTkMX3Dg= +github.com/uptrace/bun v1.2.8/go.mod h1:JBq0uBKsKqNT0Ccce1IAFZY337Wkf08c6F6qlmfOHE8= +github.com/uptrace/bun/dialect/sqlitedialect v1.2.8 h1:Huqw7YhLFTbocbSv8NETYYXqKtwLa6XsciCWtjzWSWU= +github.com/uptrace/bun/dialect/sqlitedialect v1.2.8/go.mod h1:ni7h2uwIc5zPhxgmCMTEbefONc4XsVr/ATfz1Q7d3CE= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4= github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -1086,8 +1100,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= diff --git a/pkg/sqlstore/config.go b/pkg/sqlstore/config.go new file mode 100644 index 0000000000..f3f45d9553 --- /dev/null +++ b/pkg/sqlstore/config.go @@ -0,0 +1,45 @@ +package sqlstore + +import ( + "go.signoz.io/signoz/pkg/factory" +) + +type Config struct { + // Provider is the provider to use. + Provider string `mapstructure:"provider"` + // Connection is the connection configuration. + Connection ConnectionConfig `mapstructure:",squash"` + // Sqlite is the sqlite configuration. + Sqlite SqliteConfig `mapstructure:"sqlite"` +} + +type SqliteConfig struct { + // Path is the path to the sqlite database. + Path string `mapstructure:"path"` +} + +type ConnectionConfig struct { + // MaxOpenConns is the maximum number of open connections to the database. + MaxOpenConns int `mapstructure:"max_open_conns"` +} + +func NewConfigFactory() factory.ConfigFactory { + return factory.NewConfigFactory(factory.MustNewName("sqlstore"), newConfig) +} + +func newConfig() factory.Config { + return Config{ + Provider: "sqlite", + Connection: ConnectionConfig{ + MaxOpenConns: 100, + }, + Sqlite: SqliteConfig{ + Path: "/var/lib/signoz/signoz.db", + }, + } + +} + +func (c Config) Validate() error { + return nil +} diff --git a/pkg/sqlstore/sqlitesqlstore/provider.go b/pkg/sqlstore/sqlitesqlstore/provider.go new file mode 100644 index 0000000000..da3bc97dd3 --- /dev/null +++ b/pkg/sqlstore/sqlitesqlstore/provider.go @@ -0,0 +1,55 @@ +package sqlitesqlstore + +import ( + "context" + "database/sql" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + "go.signoz.io/signoz/pkg/factory" + "go.signoz.io/signoz/pkg/sqlstore" + "go.uber.org/zap" +) + +type provider struct { + settings factory.ScopedProviderSettings + sqldb *sql.DB + bundb *bun.DB + sqlxdb *sqlx.DB +} + +func NewFactory() factory.ProviderFactory[sqlstore.SQLStore, sqlstore.Config] { + return factory.NewProviderFactory(factory.MustNewName("sqlite"), New) +} + +func New(ctx context.Context, providerSettings factory.ProviderSettings, config sqlstore.Config) (sqlstore.SQLStore, error) { + settings := factory.NewScopedProviderSettings(providerSettings, "go.signoz.io/signoz/pkg/sqlitesqlstore") + + sqldb, err := sql.Open("sqlite3", "file:"+config.Sqlite.Path+"?_foreign_keys=true") + if err != nil { + return nil, err + } + settings.ZapLogger().Info("connected to sqlite", zap.String("path", config.Sqlite.Path)) + sqldb.SetMaxOpenConns(config.Connection.MaxOpenConns) + + return &provider{ + settings: settings, + sqldb: sqldb, + bundb: bun.NewDB(sqldb, sqlitedialect.New()), + sqlxdb: sqlx.NewDb(sqldb, "sqlite3"), + }, nil +} + +func (provider *provider) BunDB() *bun.DB { + return provider.bundb +} + +func (provider *provider) SQLDB() *sql.DB { + return provider.sqldb +} + +func (provider *provider) SQLxDB() *sqlx.DB { + return provider.sqlxdb +} diff --git a/pkg/sqlstore/sqlstore.go b/pkg/sqlstore/sqlstore.go new file mode 100644 index 0000000000..7249fc3849 --- /dev/null +++ b/pkg/sqlstore/sqlstore.go @@ -0,0 +1,18 @@ +package sqlstore + +import ( + "database/sql" + + "github.com/jmoiron/sqlx" + "github.com/uptrace/bun" +) + +// SQLStore is the interface for the SQLStore. +type SQLStore interface { + // SQLDB returns the underlying sql.DB. + SQLDB() *sql.DB + // BunDB returns an instance of bun.DB. This is the recommended way to interact with the database. + BunDB() *bun.DB + // SQLxDB returns an instance of sqlx.DB. + SQLxDB() *sqlx.DB +} diff --git a/pkg/sqlstore/sqlstoretest/provider.go b/pkg/sqlstore/sqlstoretest/provider.go new file mode 100644 index 0000000000..c47bf8bc12 --- /dev/null +++ b/pkg/sqlstore/sqlstoretest/provider.go @@ -0,0 +1,61 @@ +package sqlstoretest + +import ( + "database/sql" + "fmt" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/jmoiron/sqlx" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + "go.signoz.io/signoz/pkg/sqlstore" +) + +var _ sqlstore.SQLStore = (*Provider)(nil) + +type Provider struct { + db *sql.DB + mock sqlmock.Sqlmock + bunDB *bun.DB + sqlxDB *sqlx.DB +} + +func New(config sqlstore.Config, matcher sqlmock.QueryMatcher) *Provider { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(matcher)) + if err != nil { + panic(err) + } + + var bunDB *bun.DB + var sqlxDB *sqlx.DB + + if config.Provider == "sqlite" { + bunDB = bun.NewDB(db, sqlitedialect.New()) + sqlxDB = sqlx.NewDb(db, "sqlite3") + } else { + panic(fmt.Errorf("provider %q is not supported by mockSQLStore", config.Provider)) + } + + return &Provider{ + db: db, + mock: mock, + bunDB: bunDB, + sqlxDB: sqlxDB, + } +} + +func (provider *Provider) BunDB() *bun.DB { + return provider.bunDB +} + +func (provider *Provider) SQLDB() *sql.DB { + return provider.db +} + +func (provider *Provider) SQLxDB() *sqlx.DB { + return provider.sqlxDB +} + +func (provider *Provider) Mock() sqlmock.Sqlmock { + return provider.mock +}