From 4e335054fbfbbaead6939d20fbf0e8cf6a35099f Mon Sep 17 00:00:00 2001 From: Ahsan Barkati Date: Tue, 22 Mar 2022 00:03:20 +0530 Subject: [PATCH] chore(tests): Add end-to-end testing system for query service (#867) * Initial work on s3 * some more work * Add policy api * Cleanup * Add multi-tier TTL and remove storagePolicy API * Cleanup * Typo fix * Revert constants * Cleanup * Add API to get disks * Add more validations * Initial work on e2e tests * Basic ttl test * Add test which checks for objects in Minio * Address comments Co-authored-by: Ankit Nayan --- .gitignore | 4 + .../app/clickhouseReader/reader.go | 4 +- pkg/query-service/app/dashboards/model.go | 6 +- pkg/query-service/go.mod | 4 +- pkg/query-service/go.sum | 19 + pkg/query-service/tests/cold_storage_test.go | 112 ++++ pkg/query-service/tests/docker.go | 118 ++++ .../tests/test-deploy/alertmanager.yml | 35 ++ .../tests/test-deploy/alerts.yml | 11 + .../tests/test-deploy/clickhouse-config.xml | 544 ++++++++++++++++++ .../tests/test-deploy/docker-compose.arm.yaml | 98 ++++ .../tests/test-deploy/docker-compose.yaml | 93 +++ .../test-deploy/otel-collector-config.yaml | 67 +++ .../otel-collector-metrics-config.yaml | 44 ++ .../tests/test-deploy/prometheus.yml | 26 + 15 files changed, 1180 insertions(+), 5 deletions(-) create mode 100644 pkg/query-service/tests/cold_storage_test.go create mode 100644 pkg/query-service/tests/docker.go create mode 100644 pkg/query-service/tests/test-deploy/alertmanager.yml create mode 100644 pkg/query-service/tests/test-deploy/alerts.yml create mode 100644 pkg/query-service/tests/test-deploy/clickhouse-config.xml create mode 100644 pkg/query-service/tests/test-deploy/docker-compose.arm.yaml create mode 100644 pkg/query-service/tests/test-deploy/docker-compose.yaml create mode 100644 pkg/query-service/tests/test-deploy/otel-collector-config.yaml create mode 100644 pkg/query-service/tests/test-deploy/otel-collector-metrics-config.yaml create mode 100644 pkg/query-service/tests/test-deploy/prometheus.yml diff --git a/.gitignore b/.gitignore index cb49cc98ef..01a9526908 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,10 @@ frontend/cypress.env.json frontend/*.env pkg/query-service/signoz.db +pkg/query-service/tframe/test-deploy/data/ + + # local data + /deploy/docker/clickhouse-setup/data/ /deploy/docker-swarm/clickhouse-setup/data/ diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index d766b390b2..b211bcaf94 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -2644,7 +2644,9 @@ func (r *ClickHouseReader) GetDisks(ctx context.Context) (*[]model.DiskItem, *mo fmt.Errorf("error while getting disks. Err=%v", err)} } - zap.S().Debugf("GetDisks response: %+v\n", diskItems) + + zap.S().Infof("Got response: %+v\n", diskItems) + return &diskItems, nil } diff --git a/pkg/query-service/app/dashboards/model.go b/pkg/query-service/app/dashboards/model.go index f5c3106334..480fadf69d 100644 --- a/pkg/query-service/app/dashboards/model.go +++ b/pkg/query-service/app/dashboards/model.go @@ -35,7 +35,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating dashboard table: ", err.Error()) + return nil, fmt.Errorf("Error in creating dashboard table: %v", err) } table_schema = `CREATE TABLE IF NOT EXISTS rules ( @@ -47,7 +47,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating rules table: ", err.Error()) + return nil, fmt.Errorf("Error in creating rules table: %v", err) } table_schema = `CREATE TABLE IF NOT EXISTS notification_channels ( @@ -62,7 +62,7 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { _, err = db.Exec(table_schema) if err != nil { - return nil, fmt.Errorf("Error in creating notification_channles table: ", err.Error()) + return nil, fmt.Errorf("Error in creating notification_channles table: %v", err) } return db, nil diff --git a/pkg/query-service/go.mod b/pkg/query-service/go.mod index 78b1fcf908..50d11e9d91 100644 --- a/pkg/query-service/go.mod +++ b/pkg/query-service/go.mod @@ -35,7 +35,8 @@ require ( github.com/lib/pq v1.10.0 // indirect github.com/mattn/go-sqlite3 v1.14.8 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/minio/minio-go v6.0.14+incompatible + github.com/minio/minio-go/v6 v6.0.57 github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/oklog/oklog v0.3.2 @@ -57,6 +58,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 github.com/soheilhy/cmux v0.1.4 github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.7.0 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.uber.org/zap v1.16.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect diff --git a/pkg/query-service/go.sum b/pkg/query-service/go.sum index 81361bb527..07ffe02d8c 100644 --- a/pkg/query-service/go.sum +++ b/pkg/query-service/go.sum @@ -139,6 +139,7 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -162,6 +163,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.21.1 h1:+QXUYsI7Tfxc64oD6R5BxU/Aq+UwGkyjH4W/hMNG7bg= github.com/go-ini/ini v1.21.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c h1:lGtNy7NU/+ytYPPneoErOaNrYkF5DOVCYViUK/7t7XA= github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -370,6 +372,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -383,6 +386,8 @@ github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALr github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -418,6 +423,14 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.4 h1:Ec3LTJwwzqT1++63P12fhtdEbQhtPE7TBdD6rlhqrMM= github.com/miekg/dns v1.0.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= +github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -515,12 +528,14 @@ github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJ github.com/shurcooL/vfsgen v0.0.0-20180711163814-62bca832be04/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/snowflakedb/glog v0.0.0-20180824191149-f5055e6f21ce/go.mod h1:EB/w24pR5VKI60ecFnKqXzxX3dOorz1rnVicQTQrGM0= @@ -584,6 +599,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -641,6 +657,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1007,6 +1024,8 @@ gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/segmentio/analytics-go.v3 v3.1.0 h1:UzxH1uaGZRpMKDhJyBz0pexz6yUoBU3x8bJsRk/HV6U= gopkg.in/segmentio/analytics-go.v3 v3.1.0/go.mod h1:4QqqlTlSSpVlWA9/9nDcPw+FkM2yv1NQoYjUbL9/JAw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/pkg/query-service/tests/cold_storage_test.go b/pkg/query-service/tests/cold_storage_test.go new file mode 100644 index 0000000000..f748db30dc --- /dev/null +++ b/pkg/query-service/tests/cold_storage_test.go @@ -0,0 +1,112 @@ +package tests + +import ( + "fmt" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const ( + endpoint = "http://localhost:8180" +) + +var ( + client http.Client +) + +func setTTL(table, coldStorage, toColdTTL, deleteTTL string) ([]byte, error) { + params := fmt.Sprintf("type=%s&duration=%s", table, deleteTTL) + if len(toColdTTL) > 0 { + params += fmt.Sprintf("&coldStorage=%s&toColdDuration=%s", coldStorage, toColdTTL) + } + resp, err := client.Post(endpoint+"/api/v1/settings/ttl?"+params, "", nil) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return b, err + } + + return b, nil +} + +func TestListDisks(t *testing.T) { + resp, err := client.Get(endpoint + "/api/v1/disks") + require.NoError(t, err) + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.JSONEq(t, `[{"name":"default","type":"local"}, {"name":"s3","type":"s3"}]`, string(b)) +} + +func TestSetTTL(t *testing.T) { + + testCases := []struct { + caseNo int + coldStorage string + table string + coldTTL string + deleteTTL string + expected string + }{ + { + 1, "s3", "traces", "100s", "60s", + "Delete TTL should be greater than cold storage move TTL.", + }, + { + 2, "s3", "traces", "100", "60s", + "Not a valid toCold TTL duration 100", + }, + { + 3, "s3", "traces", "100s", "100", + "Not a valid TTL duration 100", + }, + { + 4, "s3", "traces", "", "60s", + "move ttl has been successfully set up", + }, + { + 5, "s3", "traces", "10s", "600s", + "move ttl has been successfully set up", + }, + { + 6, "s4", "traces", "10s", "600s", + "No such volume `s4` in storage policy `tiered`", + }, + } + + for _, tc := range testCases { + r, err := setTTL(tc.table, tc.coldStorage, tc.coldTTL, tc.deleteTTL) + require.NoErrorf(t, err, "Failed case: %d", tc.caseNo) + require.Containsf(t, string(r), tc.expected, "Failed case: %d", tc.caseNo) + } + + time.Sleep(20 * time.Second) + doneCh := make(chan struct{}) + defer close(doneCh) + + count := 0 + for range minioClient.ListObjects(bucketName, "", false, doneCh) { + count++ + } + + require.True(t, count > 0, "No objects are present in Minio") + fmt.Printf("=== Found %d objects in Minio\n", count) +} + +func TestMain(m *testing.M) { + if err := startCluster(); err != nil { + fmt.Println(err) + } + defer stopCluster() + + m.Run() +} diff --git a/pkg/query-service/tests/docker.go b/pkg/query-service/tests/docker.go new file mode 100644 index 0000000000..da2c07f775 --- /dev/null +++ b/pkg/query-service/tests/docker.go @@ -0,0 +1,118 @@ +package tests + +import ( + "context" + "fmt" + "net/http" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "log" + + minio "github.com/minio/minio-go/v6" + "go.uber.org/zap" +) + +const ( + prefix = "signoz_test" + minioEndpoint = "localhost:9100" + accessKey = "ash" + secretKey = "password" + bucketName = "test" +) + +var ( + minioClient *minio.Client + composeFile string +) + +func init() { + goArch := runtime.GOARCH + if goArch == "arm64" { + composeFile = "./test-deploy/docker-compose.arm.yaml" + } else if goArch == "amd64" { + composeFile = "./test-deploy/docker-compose.yaml" + } else { + zap.S().Fatalf("Unsupported architecture: %s", goArch) + } +} + +func getCmd(args ...string) *exec.Cmd { + cmd := exec.CommandContext(context.Background(), args[0], args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + return cmd +} + +func startMinio() error { + log.Printf("Starting minio") + cmd := getCmd("docker", "run", "-d", "-p", "9100:9000", "-p", "9101:9001", + "--name", "signoz-minio-test", "-e", "MINIO_ROOT_USER=ash", + "-e", "MINIO_ROOT_PASSWORD=password", + "quay.io/minio/minio", "server", "/data", "--console-address", ":9001") + + if err := cmd.Run(); err != nil { + return err + } + + var err error + minioClient, err = minio.New(minioEndpoint, accessKey, secretKey, false) + if err != nil { + return err + } + if err = minioClient.MakeBucket(bucketName, ""); err != nil { + return err + } + return nil +} + +func startCluster() error { + if err := os.MkdirAll("./test-deploy/data/minio/test", 0777); err != nil { + return err + } + + if err := startMinio(); err != nil { + return err + } + + cmd := getCmd("docker-compose", "-f", composeFile, "-p", prefix, + "up", "--force-recreate", "--build", "--remove-orphans", "--detach") + + log.Printf("Starting signoz cluster...\n") + if err := cmd.Run(); err != nil { + log.Printf("While running command: %q Error: %v\n", strings.Join(cmd.Args, " "), err) + return err + } + + client := http.Client{} + for i := 0; i < 10; i++ { + if _, err := client.Get("http://localhost:8180/api/v1/version"); err != nil { + time.Sleep(2 * time.Second) + } else { + log.Printf("CLUSTER UP\n") + return nil + } + } + return fmt.Errorf("query-service is not healthy") +} + +func stopCluster() { + cmd := getCmd("docker-compose", "-f", composeFile, "-p", prefix, "down", "-v") + if err := cmd.Run(); err != nil { + log.Printf("Error while stopping the cluster. Error: %v\n", err) + } + if err := os.RemoveAll("./test-deploy/data"); err != nil { + log.Printf("Error while cleaning temporary dir. Error: %v\n", err) + } + + cmd = getCmd("docker", "container", "rm", "-f", "signoz-minio-test") + if err := cmd.Run(); err != nil { + log.Printf("While running command: %q Error: %v\n", strings.Join(cmd.Args, " "), err) + } + + log.Printf("CLUSTER DOWN: %s\n", prefix) +} diff --git a/pkg/query-service/tests/test-deploy/alertmanager.yml b/pkg/query-service/tests/test-deploy/alertmanager.yml new file mode 100644 index 0000000000..d69357f9dd --- /dev/null +++ b/pkg/query-service/tests/test-deploy/alertmanager.yml @@ -0,0 +1,35 @@ +global: + resolve_timeout: 1m + slack_api_url: 'https://hooks.slack.com/services/xxx' + +route: + receiver: 'slack-notifications' + +receivers: +- name: 'slack-notifications' + slack_configs: + - channel: '#alerts' + send_resolved: true + icon_url: https://avatars3.githubusercontent.com/u/3380462 + title: |- + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }} + {{- if gt (len .CommonLabels) (len .GroupLabels) -}} + {{" "}}( + {{- with .CommonLabels.Remove .GroupLabels.Names }} + {{- range $index, $label := .SortedPairs -}} + {{ if $index }}, {{ end }} + {{- $label.Name }}="{{ $label.Value -}}" + {{- end }} + {{- end -}} + ) + {{- end }} + text: >- + {{ range .Alerts -}} + *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} + + *Description:* {{ .Annotations.description }} + + *Details:* + {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` + {{ end }} + {{ end }} \ No newline at end of file diff --git a/pkg/query-service/tests/test-deploy/alerts.yml b/pkg/query-service/tests/test-deploy/alerts.yml new file mode 100644 index 0000000000..810a20750c --- /dev/null +++ b/pkg/query-service/tests/test-deploy/alerts.yml @@ -0,0 +1,11 @@ +groups: +- name: ExampleCPULoadGroup + rules: + - alert: HighCpuLoad + expr: system_cpu_load_average_1m > 0.1 + for: 0m + labels: + severity: warning + annotations: + summary: High CPU load + description: "CPU load is > 0.1\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" diff --git a/pkg/query-service/tests/test-deploy/clickhouse-config.xml b/pkg/query-service/tests/test-deploy/clickhouse-config.xml new file mode 100644 index 0000000000..a955a6a3fa --- /dev/null +++ b/pkg/query-service/tests/test-deploy/clickhouse-config.xml @@ -0,0 +1,544 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + + + s3 + http://172.17.0.1:9100/test// + ash + password + + + + + + + default + + + s3 + + + + + + + + + + + +9009 + + + + + +:: + + + + + + +4096 +3 + + +100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + users.xml + + + default + + + default + + + + + + + + + + + + + + localhost + 9000 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + ^carbon\. + any + + 0 + 60 + + + 7776000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^collectd\. + any + + 0 + 10 + + + 43200 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^high\. + any + + 0 + 10 + + + 172800 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^medium\. + any + + 0 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + ^low\. + any + + 0 + 600 + + + 15552000 + 1800 + + + 31536000 + 3600 + + + 63072000 + 21600 + + + 126144000 + 43200 + + + 252288000 + 86400 + + + 315360000 + 604800 + + + + + any + + 0 + 60 + + + 864000 + 900 + + + 1728000 + 1800 + + + 3456000 + 3600 + + + 10368000 + 21600 + + + 34560000 + 43200 + + + 63072000 + 86400 + + + 94608000 + 604800 + + + + + + /var/lib/clickhouse/format_schemas/ + diff --git a/pkg/query-service/tests/test-deploy/docker-compose.arm.yaml b/pkg/query-service/tests/test-deploy/docker-compose.arm.yaml new file mode 100644 index 0000000000..defed773e8 --- /dev/null +++ b/pkg/query-service/tests/test-deploy/docker-compose.arm.yaml @@ -0,0 +1,98 @@ +version: "2.4" + +services: + clickhouse: + image: altinity/clickhouse-server:21.12.3.32.altinitydev.arm + volumes: + - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml + - ./data/clickhouse/:/var/lib/clickhouse/ + healthcheck: + # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" + test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + interval: 30s + timeout: 5s + retries: 3 + + alertmanager: + image: signoz/alertmanager:0.5.0 + volumes: + - ./alertmanager.yml:/prometheus/alertmanager.yml + - ./data/alertmanager:/data + command: + - '--config.file=/prometheus/alertmanager.yml' + - '--storage.path=/data' + + query-service: + image: signoz/query-service:0.7.1 + container_name: query-service + command: ["-config=/root/config/prometheus.yml"] + volumes: + - ./prometheus.yml:/root/config/prometheus.yml + - ../dashboards:/root/config/dashboards + - ./data/signoz/:/var/lib/signoz/ + environment: + - ClickHouseUrl=tcp://clickhouse:9000 + - STORAGE=clickhouse + - GODEBUG=netdns=go + - TELEMETRY_ENABLED=true + depends_on: + clickhouse: + condition: service_healthy + + frontend: + image: signoz/frontend:0.7.1 + container_name: frontend + depends_on: + - query-service + ports: + - "3301:3301" + volumes: + - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf + + otel-collector: + image: signoz/otelcontribcol:0.43.0 + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317:4317" # OTLP GRPC receiver + mem_limit: 2000m + restart: always + depends_on: + clickhouse: + condition: service_healthy + + otel-collector-metrics: + image: signoz/otelcontribcol:0.43.0 + command: ["--config=/etc/otel-collector-metrics-config.yaml"] + volumes: + - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml + depends_on: + clickhouse: + condition: service_healthy + + hotrod: + image: jaegertracing/example-hotrod:1.30 + container_name: hotrod + logging: + options: + max-size: 50m + max-file: "3" + command: ["all"] + environment: + - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces + + load-hotrod: + image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" + container_name: load-hotrod + hostname: load-hotrod + environment: + ATTACKED_HOST: http://hotrod:8080 + LOCUST_MODE: standalone + NO_PROXY: standalone + TASK_DELAY_FROM: 5 + TASK_DELAY_TO: 30 + QUIET_MODE: "${QUIET_MODE:-false}" + LOCUST_OPTS: "--headless -u 10 -r 1" + volumes: + - ../common/locust-scripts:/locust diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml new file mode 100644 index 0000000000..2dbafff539 --- /dev/null +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -0,0 +1,93 @@ +version: "2.4" + +services: + clickhouse: + image: yandex/clickhouse-server:21.12.3.32 + volumes: + - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml + healthcheck: + # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" + test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + interval: 30s + timeout: 5s + retries: 3 + + alertmanager: + image: signoz/alertmanager:0.5.0 + volumes: + - ./alertmanager.yml:/prometheus/alertmanager.yml + command: + - '--config.file=/prometheus/alertmanager.yml' + - '--storage.path=/data' + +# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` + + + query-service: + image: signoz/query-service:latest + command: ["-config=/root/config/prometheus.yml"] + volumes: + - ./prometheus.yml:/root/config/prometheus.yml + - ../dashboards:/root/config/dashboards + environment: + - ClickHouseUrl=tcp://clickhouse:9000 + - STORAGE=clickhouse + - GODEBUG=netdns=go + - TELEMETRY_ENABLED=true + depends_on: + clickhouse: + condition: service_healthy + ports: + - "8180:8080" + volumes: + - type: bind + source: ./data + target: /var/lib/signoz + read_only: false + + otel-collector: + image: signoz/otelcontribcol:0.43.0 + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4317:4317" # OTLP GRPC receiver + mem_limit: 2000m + restart: always + depends_on: + clickhouse: + condition: service_healthy + + otel-collector-metrics: + image: signoz/otelcontribcol:0.43.0 + command: ["--config=/etc/otel-collector-metrics-config.yaml"] + volumes: + - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml + depends_on: + clickhouse: + condition: service_healthy + hotrod: + image: jaegertracing/example-hotrod:1.30 + container_name: hotrod + logging: + options: + max-size: 50m + max-file: "3" + command: ["all"] + environment: + - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces + + load-hotrod: + image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" + container_name: load-hotrod + hostname: load-hotrod + environment: + ATTACKED_HOST: http://hotrod:8080 + LOCUST_MODE: standalone + NO_PROXY: standalone + TASK_DELAY_FROM: 5 + TASK_DELAY_TO: 30 + QUIET_MODE: "${QUIET_MODE:-false}" + LOCUST_OPTS: "--headless -u 10 -r 1" + volumes: + - ../../../../deploy/docker/common/locust-scripts:/locust diff --git a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml new file mode 100644 index 0000000000..a4a2641daa --- /dev/null +++ b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml @@ -0,0 +1,67 @@ +receivers: + otlp/spanmetrics: + protocols: + grpc: + endpoint: "localhost:12345" + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: + hostmetrics: + collection_interval: 30s + scrapers: + cpu: + load: + memory: + disk: + filesystem: + network: +processors: + batch: + send_batch_size: 1000 + timeout: 10s + signozspanmetrics/prometheus: + metrics_exporter: prometheus + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + # memory_limiter: + # # Same as --mem-ballast-size-mib CLI argument + # ballast_size_mib: 683 + # # 80% of maximum memory up to 2G + # limit_mib: 1500 + # # 25% of limit up to 2G + # spike_limit_mib: 512 + # check_interval: 5s + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + health_check: {} + zpages: {} +exporters: + clickhouse: + datasource: tcp://clickhouse:9000 + clickhousemetricswrite: + endpoint: tcp://clickhouse:9000/?database=signoz_metrics + resource_to_telemetry_conversion: + enabled: true + prometheus: + endpoint: "0.0.0.0:8889" +service: + extensions: [health_check, zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhouse] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] + metrics/spanmetrics: + receivers: [otlp/spanmetrics] + exporters: [prometheus] \ No newline at end of file diff --git a/pkg/query-service/tests/test-deploy/otel-collector-metrics-config.yaml b/pkg/query-service/tests/test-deploy/otel-collector-metrics-config.yaml new file mode 100644 index 0000000000..3af039268c --- /dev/null +++ b/pkg/query-service/tests/test-deploy/otel-collector-metrics-config.yaml @@ -0,0 +1,44 @@ +receivers: + otlp: + protocols: + grpc: + http: + + # Data sources: metrics + prometheus: + config: + scrape_configs: + - job_name: "otel-collector" + scrape_interval: 30s + static_configs: + - targets: ["otel-collector:8889"] +processors: + batch: + send_batch_size: 1000 + timeout: 10s + # memory_limiter: + # # Same as --mem-ballast-size-mib CLI argument + # ballast_size_mib: 683 + # # 80% of maximum memory up to 2G + # limit_mib: 1500 + # # 25% of limit up to 2G + # spike_limit_mib: 512 + # check_interval: 5s + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + health_check: {} + zpages: {} +exporters: + clickhousemetricswrite: + endpoint: tcp://clickhouse:9000/?database=signoz_metrics + +service: + extensions: [health_check, zpages] + pipelines: + metrics: + receivers: [otlp, prometheus] + processors: [batch] + exporters: [clickhousemetricswrite] \ No newline at end of file diff --git a/pkg/query-service/tests/test-deploy/prometheus.yml b/pkg/query-service/tests/test-deploy/prometheus.yml new file mode 100644 index 0000000000..16e65ff18c --- /dev/null +++ b/pkg/query-service/tests/test-deploy/prometheus.yml @@ -0,0 +1,26 @@ +# my global config +global: + scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + - 'alerts.yml' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + +remote_read: + - url: tcp://clickhouse:9000/?database=signoz_metrics