diff --git a/deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml b/deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml index 7a5f40d299..3bb26a3a36 100644 --- a/deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml +++ b/deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml @@ -1,137 +1,567 @@ - + + - information - 1 + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + 8123 + + 9000 - - + + 9004 - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - + + 9005 - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - + + - - + + + + - - + + 9009 - -9009 + + + If not specified, then it is determined analogous to 'hostname -f' command. + This setting could be used to switch replication to another network interface + (the server may be connected to multiple networks via multiple addresses) + --> + + example.clickhouse.com + --> - -:: - - + + - - + + -4096 -3 - -100 + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + + + + 0.9 + + + 4194304 + + + 0 + correct maximum value. --> - 8589934592 + + Note: uncompressed cache can be pointless for lz4, because memory bandwidth + is slower than multi-core decompression on some server configurations. + Enabling it can sometimes paradoxically make queries slower. + --> + 8589934592 - 5368709120 + --> + 5368709120 - - /var/lib/clickhouse/ + + 1000 - - /var/lib/clickhouse/tmp/ + + 134217728 - - users.xml + + 10000 - - default + + /var/lib/clickhouse/ - - default + + /var/lib/clickhouse/tmp/ + + + + ` + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default - + Example: Zulu is an alias for UTC. + --> + - + --> + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + - + https://clickhouse.com/docs/en/operations/table_engines/distributed/ + --> + + + + + + + + + + + localhost + 9000 + + + + + + + + false + + 127.0.0.1 + 9000 + + + 127.0.0.2 + 9000 + + + 127.0.0.3 + 9000 + + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + localhost 9000 - + + + localhost + 1 + + + + + + + + + + Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file. + --> - + See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ + --> + + - + See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables + --> + - - 3600 + + 3600 - - 3600 + + 3600 - - 60 + + 60 - + + --> + + + - + true + true + true + true + + --> + + + + system + query_log
+ + toYYYYMM(event_date) + + + - system - query_log
- - 7500 - + + 7500 + + + + system + trace_log
- + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 + + + system part_log
- + toYYYYMM(event_date) 7500 ---> + + + + + + system + metric_log
+ 7500 + 1000 + + + + + system + asynchronous_metric_log
+ + 7000 + + + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + - + + + system + crash_log
- - + + 1000 + - - + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + + + + + + + - *_dictionary.xml + https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts + --> + *_dictionary.xml + + + *_function.xml - + --> - + + + + + + + + + + + + + + + + + + + + + + + + + Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. --> /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + @@ -304,239 +1149,156 @@ 5 ---> + --> - + --> + + - - - + + - ^carbon\. + click_cost any 0 + 3600 + + + 86400 60 - - 7776000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - ^collectd\. - any - - 0 - 10 - - - 43200 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^high\. - any - - 0 - 10 - - - 172800 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^medium\. - any - - 0 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^low\. - any - - 0 - 600 - - - 15552000 - 1800 - - - 31536000 - 3600 - - - 63072000 - 21600 - - - 126144000 - 43200 - - - 252288000 - 86400 - - - 315360000 - 604800 - - - - any + max 0 60 - 864000 - 900 + 3600 + 300 - 1728000 - 1800 - - - 3456000 + 86400 3600 - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - + - /var/lib/clickhouse/format_schemas/ - + --> + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + + + + + + + + 268435456 + true + + diff --git a/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml b/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml new file mode 100644 index 0000000000..aab0c15da7 --- /dev/null +++ b/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml @@ -0,0 +1,28 @@ + + + + + + 10485760 + + + s3 + https://BUCKET-NAME.s3.amazonaws.com/data/ + ACCESS-KEY-ID + SECRET-ACCESS-KEY + + + + + + + default + + + s3 + + + + + + diff --git a/deploy/docker-swarm/clickhouse-setup/clickhouse-users.xml b/deploy/docker-swarm/clickhouse-setup/clickhouse-users.xml new file mode 100644 index 0000000000..f18562071d --- /dev/null +++ b/deploy/docker-swarm/clickhouse-setup/clickhouse-users.xml @@ -0,0 +1,123 @@ + + + + + + + + + + 10000000000 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index bc58e956b5..45b4137d11 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -2,12 +2,14 @@ version: "3.9" services: clickhouse: - image: yandex/clickhouse-server:21.12.3.32 + image: clickhouse/clickhouse-server:22.4.5-alpine # ports: # - "9000:9000" # - "8123:8123" volumes: - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml + - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml + # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml - ./data/clickhouse/:/var/lib/clickhouse/ deploy: restart_policy: diff --git a/deploy/docker/clickhouse-setup/clickhouse-config.xml b/deploy/docker/clickhouse-setup/clickhouse-config.xml index 7a5f40d299..3bb26a3a36 100644 --- a/deploy/docker/clickhouse-setup/clickhouse-config.xml +++ b/deploy/docker/clickhouse-setup/clickhouse-config.xml @@ -1,137 +1,567 @@ - + + - information - 1 + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + 8123 + + 9000 - - + + 9004 - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - + + 9005 - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - + + - - + + + + - - + + 9009 - -9009 + + + If not specified, then it is determined analogous to 'hostname -f' command. + This setting could be used to switch replication to another network interface + (the server may be connected to multiple networks via multiple addresses) + --> + + example.clickhouse.com + --> - -:: - - + + - - + + -4096 -3 - -100 + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + + + + 0.9 + + + 4194304 + + + 0 + correct maximum value. --> - 8589934592 + + Note: uncompressed cache can be pointless for lz4, because memory bandwidth + is slower than multi-core decompression on some server configurations. + Enabling it can sometimes paradoxically make queries slower. + --> + 8589934592 - 5368709120 + --> + 5368709120 - - /var/lib/clickhouse/ + + 1000 - - /var/lib/clickhouse/tmp/ + + 134217728 - - users.xml + + 10000 - - default + + /var/lib/clickhouse/ - - default + + /var/lib/clickhouse/tmp/ + + + + ` + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default - + Example: Zulu is an alias for UTC. + --> + - + --> + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + - + https://clickhouse.com/docs/en/operations/table_engines/distributed/ + --> + + + + + + + + + + + localhost + 9000 + + + + + + + + false + + 127.0.0.1 + 9000 + + + 127.0.0.2 + 9000 + + + 127.0.0.3 + 9000 + + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + localhost 9000 - + + + localhost + 1 + + + + + + + + + + Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file. + --> - + See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ + --> + + - + See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables + --> + - - 3600 + + 3600 - - 3600 + + 3600 - - 60 + + 60 - + + --> + + + - + true + true + true + true + + --> + + + + system + query_log
+ + toYYYYMM(event_date) + + + - system - query_log
- - 7500 - + + 7500 + + + + system + trace_log
- + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 + + + system part_log
- + toYYYYMM(event_date) 7500 ---> + + + + + + system + metric_log
+ 7500 + 1000 + + + + + system + asynchronous_metric_log
+ + 7000 + + + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + - + + + system + crash_log
- - + + 1000 + - - + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + + + + + + + - *_dictionary.xml + https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts + --> + *_dictionary.xml + + + *_function.xml - + --> - + + + + + + + + + + + + + + + + + + + + + + + + + Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. --> /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + @@ -304,239 +1149,156 @@ 5 ---> + --> - + --> + + - - - + + - ^carbon\. + click_cost any 0 + 3600 + + + 86400 60 - - 7776000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - ^collectd\. - any - - 0 - 10 - - - 43200 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^high\. - any - - 0 - 10 - - - 172800 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^medium\. - any - - 0 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^low\. - any - - 0 - 600 - - - 15552000 - 1800 - - - 31536000 - 3600 - - - 63072000 - 21600 - - - 126144000 - 43200 - - - 252288000 - 86400 - - - 315360000 - 604800 - - - - any + max 0 60 - 864000 - 900 + 3600 + 300 - 1728000 - 1800 - - - 3456000 + 86400 3600 - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - + - /var/lib/clickhouse/format_schemas/ - + --> + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + + + + + + + + 268435456 + true + + diff --git a/deploy/docker/clickhouse-setup/clickhouse-storage.xml b/deploy/docker/clickhouse-setup/clickhouse-storage.xml new file mode 100644 index 0000000000..aab0c15da7 --- /dev/null +++ b/deploy/docker/clickhouse-setup/clickhouse-storage.xml @@ -0,0 +1,28 @@ + + + + + + 10485760 + + + s3 + https://BUCKET-NAME.s3.amazonaws.com/data/ + ACCESS-KEY-ID + SECRET-ACCESS-KEY + + + + + + + default + + + s3 + + + + + + diff --git a/deploy/docker/clickhouse-setup/clickhouse-users.xml b/deploy/docker/clickhouse-setup/clickhouse-users.xml new file mode 100644 index 0000000000..f18562071d --- /dev/null +++ b/deploy/docker/clickhouse-setup/clickhouse-users.xml @@ -0,0 +1,123 @@ + + + + + + + + + + 10000000000 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/deploy/docker/clickhouse-setup/config.xml b/deploy/docker/clickhouse-setup/config.xml new file mode 100644 index 0000000000..3bb26a3a36 --- /dev/null +++ b/deploy/docker/clickhouse-setup/config.xml @@ -0,0 +1,1304 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + + + + 8123 + + + 9000 + + + 9004 + + + 9005 + + + + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + + + + 0.9 + + + 4194304 + + + 0 + + + + + + 8589934592 + + + 5368709120 + + + + 1000 + + + 134217728 + + + 10000 + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + + ` + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default + + + + + + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + + + + + + + + + + + + + + + + localhost + 9000 + + + + + + + + false + + 127.0.0.1 + 9000 + + + 127.0.0.2 + 9000 + + + 127.0.0.3 + 9000 + + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + + + + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 + + + + + system + part_log
+ toYYYYMM(event_date) + 7500 + + + + + + + system + metric_log
+ 7500 + 1000 + + + + + system + asynchronous_metric_log
+ + 7000 + + + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + + + + + + system + crash_log
+ + + 1000 + + + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + + + + + + + + + + *_dictionary.xml + + + *_function.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + + + + + + + + 268435456 + true + + diff --git a/deploy/docker/clickhouse-setup/docker-compose.arm.yaml b/deploy/docker/clickhouse-setup/docker-compose.arm.yaml deleted file mode 100644 index c30bf82064..0000000000 --- a/deploy/docker/clickhouse-setup/docker-compose.arm.yaml +++ /dev/null @@ -1,133 +0,0 @@ -version: "2.4" - -services: - clickhouse: - image: altinity/clickhouse-server:21.12.3.32.altinitydev.arm - # ports: - # - "9000:9000" - # - "8123:8123" - volumes: - - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml - - ./data/clickhouse/:/var/lib/clickhouse/ - restart: on-failure - logging: - options: - max-size: 50m - max-file: "3" - healthcheck: - # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] - interval: 30s - timeout: 5s - retries: 3 - - alertmanager: - image: signoz/alertmanager:0.23.0-0.1 - volumes: - - ./data/alertmanager:/data - depends_on: - query-service: - condition: service_healthy - restart: on-failure - command: - - --queryService.url=http://query-service:8085 - - --storage.path=/data - -# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` - - - query-service: - image: signoz/query-service:0.8.2 - container_name: query-service - command: ["-config=/root/config/prometheus.yml"] - # ports: - # - "6060:6060" # pprof port - # - "8080:8080" # query-service port - volumes: - - ./prometheus.yml:/root/config/prometheus.yml - - ../dashboards:/root/config/dashboards - - ./data/signoz/:/var/lib/signoz/ - environment: - - ClickHouseUrl=tcp://clickhouse:9000/?database=signoz_traces - - STORAGE=clickhouse - - GODEBUG=netdns=go - - TELEMETRY_ENABLED=true - - DEPLOYMENT_TYPE=docker-standalone-arm - restart: on-failure - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/version"] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - clickhouse: - condition: service_healthy - - frontend: - image: signoz/frontend:0.8.2 - container_name: frontend - restart: on-failure - depends_on: - - alertmanager - - query-service - ports: - - "3301:3301" - volumes: - - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf - - otel-collector: - image: signoz/otelcontribcol:0.45.1-0.3 - command: ["--config=/etc/otel-collector-config.yaml"] - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ports: - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP receiver - # - "8889:8889" # Prometheus metrics exposed by the agent - # - "13133:13133" # health_check - # - "14268:14268" # Jaeger receiver - # - "55678:55678" # OpenCensus receiver - # - "55679:55679" # zpages extension - # - "55680:55680" # OTLP gRPC legacy receiver - # - "55681:55681" # OTLP HTTP legacy receiver - mem_limit: 2000m - restart: on-failure - depends_on: - clickhouse: - condition: service_healthy - - otel-collector-metrics: - image: signoz/otelcontribcol:0.45.1-0.3 - command: ["--config=/etc/otel-collector-metrics-config.yaml"] - volumes: - - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml - restart: on-failure - depends_on: - clickhouse: - condition: service_healthy - - hotrod: - image: jaegertracing/example-hotrod:1.30 - container_name: hotrod - logging: - options: - max-size: 50m - max-file: "3" - command: ["all"] - environment: - - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - - load-hotrod: - image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" - container_name: load-hotrod - hostname: load-hotrod - environment: - ATTACKED_HOST: http://hotrod:8080 - LOCUST_MODE: standalone - NO_PROXY: standalone - TASK_DELAY_FROM: 5 - TASK_DELAY_TO: 30 - QUIET_MODE: "${QUIET_MODE:-false}" - LOCUST_OPTS: "--headless -u 10 -r 1" - volumes: - - ../common/locust-scripts:/locust diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 44a0e25554..166a2096e3 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -2,12 +2,14 @@ version: "2.4" services: clickhouse: - image: yandex/clickhouse-server:21.12.3.32 + image: clickhouse/clickhouse-server:22.4.5-alpine # ports: # - "9000:9000" # - "8123:8123" volumes: - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml + - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml + # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml - ./data/clickhouse/:/var/lib/clickhouse/ restart: on-failure logging: diff --git a/deploy/docker/clickhouse-setup/users.xml b/deploy/docker/clickhouse-setup/users.xml new file mode 100644 index 0000000000..f18562071d --- /dev/null +++ b/deploy/docker/clickhouse-setup/users.xml @@ -0,0 +1,123 @@ + + + + + + + + + + 10000000000 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/frontend/src/container/GridGraphLayout/index.tsx b/frontend/src/container/GridGraphLayout/index.tsx index 88250f3b7c..8f7ea56c15 100644 --- a/frontend/src/container/GridGraphLayout/index.tsx +++ b/frontend/src/container/GridGraphLayout/index.tsx @@ -218,7 +218,7 @@ function GridGraph(props: Props): JSX.Element { const onLayoutChangeHandler = async (layout: Layout[]): Promise => { setLayoutFunction(layout); - await onLayoutSaveHandler(layout); + // await onLayoutSaveHandler(layout); }; const onAddPanelHandler = useCallback(() => { diff --git a/frontend/src/container/ListOfDashboard/SearchFilter/__tests__/utils.test.ts b/frontend/src/container/ListOfDashboard/SearchFilter/__tests__/utils.test.ts new file mode 100644 index 0000000000..db9825b677 --- /dev/null +++ b/frontend/src/container/ListOfDashboard/SearchFilter/__tests__/utils.test.ts @@ -0,0 +1,58 @@ +import { Dashboard } from 'types/api/dashboard/getAll'; +import { v4 as uuid } from 'uuid'; + +import { TOperator } from '../types'; +import { executeSearchQueries } from '../utils'; + +describe('executeSearchQueries', () => { + const firstDashboard: Dashboard = { + id: 11111, + uuid: uuid(), + created_at: '', + updated_at: '', + data: { + title: 'first dashboard', + }, + }; + const secondDashboard: Dashboard = { + id: 22222, + uuid: uuid(), + created_at: '', + updated_at: '', + data: { + title: 'second dashboard', + }, + }; + const thirdDashboard: Dashboard = { + id: 333333, + uuid: uuid(), + created_at: '', + updated_at: '', + data: { + title: 'third dashboard (with special characters +?\\)', + }, + }; + const dashboards = [firstDashboard, secondDashboard, thirdDashboard]; + + it('should filter dashboards based on title', () => { + const query = { + category: 'title', + id: 'someid', + operator: '=' as TOperator, + value: 'first dashboard', + }; + + expect(executeSearchQueries([query], dashboards)).toEqual([firstDashboard]); + }); + + it('should filter dashboards with special characters', () => { + const query = { + category: 'title', + id: 'someid', + operator: '=' as TOperator, + value: 'third dashboard (with special characters +?\\)', + }; + + expect(executeSearchQueries([query], dashboards)).toEqual([thirdDashboard]); + }); +}); diff --git a/frontend/src/container/ListOfDashboard/SearchFilter/utils.ts b/frontend/src/container/ListOfDashboard/SearchFilter/utils.ts index 5f9b37cc3e..6487ccd789 100644 --- a/frontend/src/container/ListOfDashboard/SearchFilter/utils.ts +++ b/frontend/src/container/ListOfDashboard/SearchFilter/utils.ts @@ -42,6 +42,8 @@ export const executeSearchQueries = ( if (!searchData.length || !queries.length) { return searchData; } + const escapeRegExp = (regExp: string): string => + regExp.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); queries.forEach((query: IQueryStructure) => { const { operator } = query; @@ -61,7 +63,7 @@ export const executeSearchQueries = ( for (const searchSpaceItem of searchSpace) { if (searchSpaceItem) for (const queryValue of value) { - if (searchSpaceItem.match(queryValue)) { + if (searchSpaceItem.match(escapeRegExp(queryValue))) { return resolveOperator(true, operator); } } diff --git a/pkg/query-service/tests/test-deploy/clickhouse-config.xml b/pkg/query-service/tests/test-deploy/clickhouse-config.xml index 06ddb2b723..3bb26a3a36 100644 --- a/pkg/query-service/tests/test-deploy/clickhouse-config.xml +++ b/pkg/query-service/tests/test-deploy/clickhouse-config.xml @@ -1,136 +1,567 @@ - + + - information - 1 + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + 8123 + + 9000 - - + + 9004 - - - - - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key - - /etc/clickhouse-server/dhparam.pem - none - true - true - sslv2,sslv3 - true - + + 9005 - - true - true - sslv2,sslv3 - true - - - - RejectCertificateHandler - - - + + - - - - - - - s3 - http://172.17.0.1:9100/test// - ash - password - - - - - - - default - - - s3 - - - - - + + + + - - + + 9009 - -9009 + + + If not specified, then it is determined analogous to 'hostname -f' command. + This setting could be used to switch replication to another network interface + (the server may be connected to multiple networks via multiple addresses) + --> + + example.clickhouse.com + --> - -:: - - + + - - + + -4096 -3 - -100 + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + + + + 0.9 + + + 4194304 + + + 0 + correct maximum value. --> - 8589934592 + + Note: uncompressed cache can be pointless for lz4, because memory bandwidth + is slower than multi-core decompression on some server configurations. + Enabling it can sometimes paradoxically make queries slower. + --> + 8589934592 - 5368709120 + --> + 5368709120 - - /var/lib/clickhouse/ + + 1000 - - /var/lib/clickhouse/tmp/ + + 134217728 - - users.xml + + 10000 - - default + + /var/lib/clickhouse/ - - default + + /var/lib/clickhouse/tmp/ + + + + ` + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default - + Example: Zulu is an alias for UTC. + --> + - + --> + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + - + https://clickhouse.com/docs/en/operations/table_engines/distributed/ + --> + + + + + + + + + + + localhost + 9000 + + + + + + + + false + + 127.0.0.1 + 9000 + + + 127.0.0.2 + 9000 + + + 127.0.0.3 + 9000 + + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + localhost 9000 - + + + localhost + 1 + + + + + + + + + + Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file. + --> - + See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ + --> + + - + See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables + --> + - - 3600 + + 3600 - - 3600 + + 3600 - - 60 + + 60 - + + --> + + + - + true + true + true + true + + --> + + + + system + query_log
+ + toYYYYMM(event_date) + + + - system - query_log
- - 7500 - + + 7500 + + + + system + trace_log
- + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 + + + system part_log
- + toYYYYMM(event_date) 7500 ---> + + + + + + system + metric_log
+ 7500 + 1000 + + + + + system + asynchronous_metric_log
+ + 7000 + + + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 + - + + + system + crash_log
- - + + 1000 + - - + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 + + + + + + + - *_dictionary.xml + https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts + --> + *_dictionary.xml + + + *_function.xml - + --> - + + + + + + + + + + + + + + + + + + + + + + + + + Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. --> /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + @@ -303,239 +1149,156 @@ 5 ---> + --> - + --> + + - - - + + - ^carbon\. + click_cost any 0 + 3600 + + + 86400 60 - - 7776000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - ^collectd\. - any - - 0 - 10 - - - 43200 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^high\. - any - - 0 - 10 - - - 172800 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^medium\. - any - - 0 - 60 - - - 864000 - 900 - - - 1728000 - 1800 - - - 3456000 - 3600 - - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - - - - ^low\. - any - - 0 - 600 - - - 15552000 - 1800 - - - 31536000 - 3600 - - - 63072000 - 21600 - - - 126144000 - 43200 - - - 252288000 - 86400 - - - 315360000 - 604800 - - - - any + max 0 60 - 864000 - 900 + 3600 + 300 - 1728000 - 1800 - - - 3456000 + 86400 3600 - - 10368000 - 21600 - - - 34560000 - 43200 - - - 63072000 - 86400 - - - 94608000 - 604800 - - + - /var/lib/clickhouse/format_schemas/ - + --> + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + + + + + + + + 268435456 + true + + diff --git a/pkg/query-service/tests/test-deploy/clickhouse-storage.xml b/pkg/query-service/tests/test-deploy/clickhouse-storage.xml new file mode 100644 index 0000000000..eaf1e7e99d --- /dev/null +++ b/pkg/query-service/tests/test-deploy/clickhouse-storage.xml @@ -0,0 +1,28 @@ + + + + + + 10485760 + + + s3 + http://172.17.0.1:9100/test// + ash + password + + + + + + + default + + + s3 + + + + + + diff --git a/pkg/query-service/tests/test-deploy/clickhouse-users.xml b/pkg/query-service/tests/test-deploy/clickhouse-users.xml new file mode 100644 index 0000000000..f18562071d --- /dev/null +++ b/pkg/query-service/tests/test-deploy/clickhouse-users.xml @@ -0,0 +1,123 @@ + + + + + + + + + + 10000000000 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/pkg/query-service/tests/test-deploy/docker-compose.arm.yaml b/pkg/query-service/tests/test-deploy/docker-compose.arm.yaml deleted file mode 100644 index 48a1449fe4..0000000000 --- a/pkg/query-service/tests/test-deploy/docker-compose.arm.yaml +++ /dev/null @@ -1,99 +0,0 @@ -version: "2.4" - -services: - clickhouse: - image: altinity/clickhouse-server:21.12.3.32.altinitydev.arm - volumes: - - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml - restart: on-failure - logging: - options: - max-size: 50m - max-file: "3" - healthcheck: - # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'" - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] - interval: 30s - timeout: 5s - retries: 3 - - alertmanager: - image: signoz/alertmanager:0.23.0-0.1 - depends_on: - - query-service - restart: on-failure - command: - - --queryService.url=http://query-service:8085 - - --storage.path=/data - - query-service: - image: signoz/query-service:latest - container_name: query-service - command: ["-config=/root/config/prometheus.yml"] - volumes: - - ./prometheus.yml:/root/config/prometheus.yml - - ../dashboards:/root/config/dashboards - - ./data:/var/lib/signoz - ports: - - "8180:8080" - environment: - - ClickHouseUrl=tcp://clickhouse:9000/?database=signoz_traces - - STORAGE=clickhouse - - GODEBUG=netdns=go - - TELEMETRY_ENABLED=true - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/version"] - interval: 30s - timeout: 5s - retries: 3 - depends_on: - clickhouse: - condition: service_healthy - - otel-collector: - image: signoz/otelcontribcol:0.45.1-0.3 - command: ["--config=/etc/otel-collector-config.yaml"] - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ports: - - "4317:4317" # OTLP GRPC receiver - mem_limit: 2000m - restart: always - depends_on: - clickhouse: - condition: service_healthy - - otel-collector-metrics: - image: signoz/otelcontribcol:0.45.1-0.3 - command: ["--config=/etc/otel-collector-metrics-config.yaml"] - volumes: - - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml - depends_on: - clickhouse: - condition: service_healthy - - hotrod: - image: jaegertracing/example-hotrod:1.30 - container_name: hotrod - logging: - options: - max-size: 50m - max-file: "3" - command: ["all"] - environment: - - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces - - load-hotrod: - image: "grubykarol/locust:1.2.3-python3.9-alpine3.12" - container_name: load-hotrod - hostname: load-hotrod - environment: - ATTACKED_HOST: http://hotrod:8080 - LOCUST_MODE: standalone - NO_PROXY: standalone - TASK_DELAY_FROM: 5 - TASK_DELAY_TO: 30 - QUIET_MODE: "${QUIET_MODE:-false}" - LOCUST_OPTS: "--headless -u 10 -r 1" - volumes: - - ../../../../deploy/docker/common/locust-scripts:/locust diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index 4a3ad41870..b1c6a39d7e 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -2,9 +2,11 @@ version: "2.4" services: clickhouse: - image: yandex/clickhouse-server:21.12.3.32 + image: clickhouse/clickhouse-server:22.4.5-alpine volumes: - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml + - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml + - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml restart: on-failure logging: options: @@ -17,8 +19,8 @@ services: timeout: 5s retries: 3 ports: - - "9000:9000" - - "8123:8123" + - "9000:9000" + - "8123:8123" alertmanager: image: signoz/alertmanager:0.23.0-0.1