GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-13 02:29:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: get and update log fields updated to support new materialized c… (#3275)

Browse Source 
* feat: get and update log fields updated to support new materialized columns and index

* feat: get attribute keys logic updated

* feat: logs qb changes to support new materialized columns

* feat: don't allow updating static top level fields

* feat: old logs parser updated

* feat: upgrade clickhouse

* fix: columnname format updated for materialized columns

* feat: get fields updated

* feat: minor changes

* fix: default values for distributed columns

* feat: common function for for clickhouse formatted column name

* chore: remove interesting selected fields

* fix: update clickhouse version in other place

---------

Co-authored-by: Prashant Shahi <prashant@signoz.io>
This commit is contained in:
Nityananda Gohain 2023-08-23 15:03:24 +05:30 committed by GitHub
parent 5d6bb18679
commit ee6b290a0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 248 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
View File

@ -1,8 +1,7 @@
version: "3.9"
x-clickhouse-defaults:
&clickhouse-defaults
image: clickhouse/clickhouse-server:22.8.8-alpine
x-clickhouse-defaults: &clickhouse-defaults
image: clickhouse/clickhouse-server:23.7.3-alpine
tty: true
deploy:
restart_policy:
@ -34,8 +33,7 @@ x-clickhouse-defaults:
soft: 262144
hard: 262144
x-clickhouse-depend:
&clickhouse-depend
x-clickhouse-depend: &clickhouse-depend
depends_on:
- clickhouse
# - clickhouse-2

2
deploy/docker/clickhouse-setup/docker-compose-core.yaml
View File

@ -2,7 +2,7 @@ version: "2.4"
services:
clickhouse:
image: clickhouse/clickhouse-server:22.8.8-alpine
image: clickhouse/clickhouse-server:23.7.3-alpine
container_name: signoz-clickhouse
# ports:
# - "9000:9000"

9
deploy/docker/clickhouse-setup/docker-compose.yaml
View File

@ -1,9 +1,9 @@
version: "2.4"
x-clickhouse-defaults:
&clickhouse-defaults
x-clickhouse-defaults: &clickhouse-defaults
restart: on-failure
image: clickhouse/clickhouse-server:22.8.8-alpine
# addding non LTS version due to this fix https://github.com/ClickHouse/ClickHouse/commit/32caf8716352f45c1b617274c7508c86b7d1afab
image: clickhouse/clickhouse-server:23.7.3-alpine
tty: true
depends_on:
- zookeeper-1
@ -32,8 +32,7 @@ x-clickhouse-defaults:
soft: 262144
hard: 262144
x-clickhouse-depend:
&clickhouse-depend
x-clickhouse-depend: &clickhouse-depend
depends_on:
clickhouse:
condition: service_healthy

133
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -3440,7 +3440,6 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe
extractSelectedAndInterestingFields(statements[0].Statement, constants.Attributes, &attributes, &response)
extractSelectedAndInterestingFields(statements[0].Statement, constants.Resources, &resources, &response)
extractSelectedAndInterestingFields(statements[0].Statement, constants.Static, &constants.StaticInterestingLogFields, &response)
return &response, nil
}
@ -3448,7 +3447,8 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe
func extractSelectedAndInterestingFields(tableStatement string, fieldType string, fields *[]model.LogField, response *model.GetFieldsResponse) {
for _, field := range *fields {
field.Type = fieldType
if isSelectedField(tableStatement, field.Name) {
// all static fields are assumed to be selected as we don't allow changing them
if isSelectedField(tableStatement, field) {
response.Selected = append(response.Selected, field)
} else {
response.Interesting = append(response.Interesting, field)
@ -3456,28 +3456,72 @@ func extractSelectedAndInterestingFields(tableStatement string, fieldType string
}
}
func isSelectedField(tableStatement, field string) bool {
return strings.Contains(tableStatement, fmt.Sprintf("INDEX %s_idx", field))
func isSelectedField(tableStatement string, field model.LogField) bool {
// in case of attributes and resources, if there is a materialized column present then it is selected
// TODO: handle partial change complete eg:- index is removed but materialized column is still present
name := utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name)
return strings.Contains(tableStatement, fmt.Sprintf("`%s`", name))
}
func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.UpdateField) *model.ApiError {
// don't allow updating static fields
if field.Type == constants.Static {
err := errors.New("cannot update static fields")
return &model.ApiError{Err: err, Typ: model.ErrorBadData}
}
colname := utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name)
// if a field is selected it means that the field needs to be indexed
if field.Selected {
// if the type is attribute or resource, create the materialized column first
if field.Type == constants.Attributes || field.Type == constants.Resources {
// create materialized
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED %s_%s_value[indexOf(%s_%s_key, '%s')] CODEC(LZ4)", r.logsDB, r.logsLocalTable, cluster, field.Name, field.DataType, field.Type, strings.ToLower(field.DataType), field.Type, strings.ToLower(field.DataType), field.Name)
keyColName := fmt.Sprintf("%s_%s_key", field.Type, strings.ToLower(field.DataType))
valueColName := fmt.Sprintf("%s_%s_value", field.Type, strings.ToLower(field.DataType))
err := r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
// create materialized column
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED %s[indexOf(%s, '%s')] CODEC(ZSTD(1))",
r.logsDB, r.logsLocalTable,
cluster,
colname, field.DataType,
valueColName,
keyColName,
field.Name,
)
err := r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED -1", r.logsDB, r.logsTable, cluster, field.Name, field.DataType)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s %s MATERIALIZED -1",
r.logsDB, r.logsTable,
cluster,
colname, field.DataType,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
// create exists column
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists bool MATERIALIZED if(indexOf(%s, '%s') != 0, true, false) CODEC(ZSTD(1))",
r.logsDB, r.logsLocalTable,
cluster,
colname,
keyColName,
field.Name,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS %s_exists bool MATERIALIZED false",
r.logsDB, r.logsTable,
cluster,
colname,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
// create the index
@ -3487,20 +3531,52 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda
if field.IndexGranularity == 0 {
field.IndexGranularity = constants.DefaultLogSkipIndexGranularity
}
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d", r.logsDB, r.logsLocalTable, cluster, field.Name, field.Name, field.IndexType, field.IndexGranularity)
err := r.db.Exec(ctx, query)
query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS %s_idx (%s) TYPE %s GRANULARITY %d",
r.logsDB, r.logsLocalTable,
cluster,
colname,
colname,
field.IndexType,
field.IndexGranularity,
)
err = r.db.Exec(ctx, query)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
} else {
// remove index
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s DROP INDEX IF EXISTS %s_idx", r.logsDB, r.logsLocalTable, cluster, field.Name)
// Delete the index first
query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s DROP INDEX IF EXISTS %s_idx", r.logsDB, r.logsLocalTable, cluster, colname)
err := r.db.Exec(ctx, query)
// we are ignoring errors with code 341 as it is an error with updating old part https://github.com/SigNoz/engineering-pod/issues/919#issuecomment-1366344346
if err != nil && !strings.HasPrefix(err.Error(), "code: 341") {
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
for _, table := range []string{r.logsLocalTable, r.logsTable} {
// drop materialized column from logs table
query := "ALTER TABLE %s.%s ON CLUSTER %s DROP COLUMN IF EXISTS %s "
err := r.db.Exec(ctx, fmt.Sprintf(query,
r.logsDB, table,
cluster,
colname,
),
)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
// drop exists column on logs table
query = "ALTER TABLE %s.%s ON CLUSTER %s DROP COLUMN IF EXISTS %s_exists "
err = r.db.Exec(ctx, fmt.Sprintf(query,
r.logsDB, table,
cluster,
colname,
),
)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorInternal}
}
}
}
return nil
}
@ -3897,8 +3973,11 @@ func (r *ClickHouseReader) GetLatencyMetricMetadata(ctx context.Context, metricN
}, nil
}
func isColumn(tableStatement, field string) bool {
return strings.Contains(tableStatement, fmt.Sprintf("`%s` ", field))
func isColumn(tableStatement, attrType, field, datType string) bool {
// value of attrType will be `resource` or `tag`, if `tag` change it to `attribute`
name := utils.GetClickhouseColumnName(attrType, datType, field)
return strings.Contains(tableStatement, fmt.Sprintf("`%s` ", name))
}
func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) {
@ -3970,7 +4049,7 @@ func (r *ClickHouseReader) GetLogAggregateAttributes(ctx context.Context, req *v
Key: tagKey,
DataType: v3.AttributeKeyDataType(dataType),
Type: v3.AttributeKeyType(attType),
IsColumn: isColumn(statements[0].Statement, tagKey),
IsColumn: isColumn(statements[0].Statement, attType, tagKey, dataType),
}
response.AttributeKeys = append(response.AttributeKeys, key)
}
@ -4025,7 +4104,7 @@ func (r *ClickHouseReader) GetLogAttributeKeys(ctx context.Context, req *v3.Filt
Key: attributeKey,
DataType: v3.AttributeKeyDataType(attributeDataType),
Type: v3.AttributeKeyType(tagType),
IsColumn: isColumn(statements[0].Statement, attributeKey),
IsColumn: isColumn(statements[0].Statement, tagType, attributeKey, attributeDataType),
}
response.AttributeKeys = append(response.AttributeKeys, key)

6
pkg/query-service/app/logs/parser.go
View File

@ -9,6 +9,7 @@ import (
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/utils"
)
var operatorMapping = map[string]string{
@ -304,6 +305,11 @@ func replaceFieldInToken(queryToken string, selectedFieldsLookup map[string]mode
} else if strings.Compare(strings.ToLower(*col), "fulltext") != 0 && field.Type != constants.Static {
return "", fmt.Errorf("field not found for filtering")
}
} else {
field := selectedFieldsLookup[sqlColName]
if field.Type != constants.Static {
sqlColName = utils.GetClickhouseColumnName(field.Type, field.DataType, field.Name)
}
}
}
return strings.Replace(queryToken, *col, sqlColName, 1), nil

24
pkg/query-service/app/logs/parser_test.go
View File

@ -252,7 +252,7 @@ func TestReplaceInterestingFields(t *testing.T) {
},
}
expectedTokens := []string{"attributes_int64_value[indexOf(attributes_int64_key, 'id.userid')] IN (100) ", "and id_key >= 50 ", `AND body ILIKE '%searchstring%'`}
expectedTokens := []string{"attributes_int64_value[indexOf(attributes_int64_key, 'id.userid')] IN (100) ", "and attribute_int64_id_key >= 50 ", `AND body ILIKE '%searchstring%'`}
Convey("testInterestingFields", t, func() {
tokens, err := replaceInterestingFields(&allFields, queryTokens)
So(err, ShouldBeNil)
@ -340,6 +340,11 @@ var generateSQLQueryFields = model.GetFieldsResponse{
DataType: "string",
Type: "attributes",
},
{
Name: "severity_number",
DataType: "string",
Type: "static",
},
},
Interesting: []model.LogField{
{
@ -369,7 +374,7 @@ var generateSQLQueryTestCases = []struct {
IdGt: "2BsKLKv8cZrLCn6rkOcRGkdjBdM",
IdLT: "2BsKG6tRpFWjYMcWsAGKfSxoQdU",
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' ) and ( field1 < 100 and field1 > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' ) and ( attribute_int64_field1 < 100 and attribute_int64_field1 > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
},
{
Name: "second query with only timestamp range",
@ -378,7 +383,7 @@ var generateSQLQueryTestCases = []struct {
TimestampStart: uint64(1657689292000),
TimestampEnd: uint64(1657689294000),
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( field1 < 100 and field1 > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( attribute_int64_field1 < 100 and attribute_int64_field1 > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
},
{
Name: "generate case sensitive query",
@ -387,7 +392,7 @@ var generateSQLQueryTestCases = []struct {
TimestampStart: uint64(1657689292000),
TimestampEnd: uint64(1657689294000),
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( field1 < 100 and attributes_int64_value[indexOf(attributes_int64_key, 'FielD1')] > 50 and Field2 > 10 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( attribute_int64_field1 < 100 and attributes_int64_value[indexOf(attributes_int64_key, 'FielD1')] > 50 and attribute_double64_Field2 > 10 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ",
},
{
Name: "Check exists and not exists",
@ -396,7 +401,16 @@ var generateSQLQueryTestCases = []struct {
TimestampStart: uint64(1657689292000),
TimestampEnd: uint64(1657689294000),
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( has(attributes_int64_key, 'field1') and NOT has(attributes_double64_key, 'Field2') and Field2 > 10 ) ",
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( has(attributes_int64_key, 'field1') and NOT has(attributes_double64_key, 'Field2') and attribute_double64_Field2 > 10 ) ",
},
{
Name: "Check top level key filter",
Filter: model.LogsFilterParams{
Query: "severity_number in (1)",
TimestampStart: uint64(1657689292000),
TimestampEnd: uint64(1657689294000),
},
SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( severity_number IN (1) ) ",
},
{
Name: "Check exists and not exists on top level keys",

14
pkg/query-service/app/logs/v3/query_builder.go
View File

@ -83,7 +83,17 @@ func getClickhouseColumnName(key v3.AttributeKey) string {
columnType := getClickhouseLogsColumnType(key.Type)
columnDataType := getClickhouseLogsColumnDataType(key.DataType)
clickhouseColumn = fmt.Sprintf("%s_%s_value[indexOf(%s_%s_key, '%s')]", columnType, columnDataType, columnType, columnDataType, key.Key)
return clickhouseColumn
}
// check if it is a static field
if key.Type == v3.AttributeKeyTypeUnspecified {
// name is the column name
return clickhouseColumn
}
// materialized column created from query
clickhouseColumn = utils.GetClickhouseColumnName(string(key.Type), string(key.DataType), key.Key)
return clickhouseColumn
}
@ -123,6 +133,7 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey)
if err != nil {
return "", fmt.Errorf("failed to validate and cast value for %s: %v", item.Key.Key, err)
}
if logsOp, ok := logOperators[op]; ok {
switch op {
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
@ -153,6 +164,9 @@ func buildLogsTimeSeriesFilterQuery(fs *v3.FilterSet, groupBy []v3.AttributeKey)
columnType := getClickhouseLogsColumnType(attr.Type)
columnDataType := getClickhouseLogsColumnDataType(attr.DataType)
conditions = append(conditions, fmt.Sprintf("indexOf(%s_%s_key, '%s') > 0", columnType, columnDataType, attr.Key))
} else if attr.Type != v3.AttributeKeyTypeUnspecified {
// for materialzied columns
conditions = append(conditions, fmt.Sprintf("%s_exists=true", getClickhouseColumnName(attr)))
}
}

20
pkg/query-service/app/logs/v3/query_builder_test.go
View File

@ -26,7 +26,17 @@ var testGetClickhouseColumnNameData = []struct {
{
Name: "selected field",
AttributeKey: v3.AttributeKey{Key: "servicename", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
ExpectedColumnName: "servicename",
ExpectedColumnName: "attribute_string_servicename",
},
{
Name: "selected field resource",
AttributeKey: v3.AttributeKey{Key: "sdk_version", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeResource, IsColumn: true},
ExpectedColumnName: "resource_int64_sdk_version",
},
{
Name: "selected field float",
AttributeKey: v3.AttributeKey{Key: "sdk_version", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: true},
ExpectedColumnName: "attribute_float64_sdk_version",
},
{
Name: "same name as top level column",
@ -35,7 +45,7 @@ var testGetClickhouseColumnNameData = []struct {
},
{
Name: "top level column",
AttributeKey: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true},
AttributeKey: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true},
ExpectedColumnName: "trace_id",
},
}
@ -121,7 +131,7 @@ var timeSeriesFilterQueryData = []struct {
{Key: v3.AttributeKey{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "john", Operator: "="},
{Key: v3.AttributeKey{Key: "k8s_namespace", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "my_service", Operator: "!="},
}},
ExpectedFilter: " AND user_name = 'john' AND resources_string_value[indexOf(resources_string_key, 'k8s_namespace')] != 'my_service'",
ExpectedFilter: " AND attribute_string_user_name = 'john' AND resources_string_value[indexOf(resources_string_key, 'k8s_namespace')] != 'my_service'",
},
{
Name: "Test like",
@ -184,7 +194,7 @@ var timeSeriesFilterQueryData = []struct {
FilterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "host: \"(?P<host>\\S+)\"", Operator: "regex"},
}},
ExpectedFilter: " AND match(host, 'host: \"(?P<host>\\\\S+)\"')",
ExpectedFilter: " AND match(attribute_string_host, 'host: \"(?P<host>\\\\S+)\"')",
},
{
Name: "Test not regex",
@ -207,7 +217,7 @@ var timeSeriesFilterQueryData = []struct {
{Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: "ncontains"},
}},
GroupBy: []v3.AttributeKey{{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}},
ExpectedFilter: " AND attributes_string_value[indexOf(attributes_string_key, 'host')] NOT ILIKE '%102.%'",
ExpectedFilter: " AND attributes_string_value[indexOf(attributes_string_key, 'host')] NOT ILIKE '%102.%' AND attribute_string_host_exists=true",
},
{
Name: "Wrong data",

31
pkg/query-service/constants/constants.go
View File

@ -221,22 +221,17 @@ const (
UINT8 = "Uint8"
)
var StaticInterestingLogFields = []model.LogField{
var StaticSelectedLogFields = []model.LogField{
{
Name: "trace_id",
DataType: STRING,
Type: Static,
},
{
Name: "span_id",
DataType: STRING,
Type: Static,
},
{
Name: "trace_flags",
Name: "timestamp",
DataType: UINT32,
Type: Static,
},
{
Name: "id",
DataType: STRING,
Type: Static,
},
{
Name: "severity_text",
DataType: LOWCARDINALITY_STRING,
@ -247,16 +242,18 @@ var StaticInterestingLogFields = []model.LogField{
DataType: UINT8,
Type: Static,
},
}
var StaticSelectedLogFields = []model.LogField{
{
Name: "timestamp",
Name: "trace_flags",
DataType: UINT32,
Type: Static,
},
{
Name: "id",
Name: "trace_id",
DataType: STRING,
Type: Static,
},
{
Name: "span_id",
DataType: STRING,
Type: Static,
},

8
pkg/query-service/tests/test-deploy/docker-compose.yaml
View File

@ -1,9 +1,8 @@
version: "2.4"
x-clickhouse-defaults:
&clickhouse-defaults
x-clickhouse-defaults: &clickhouse-defaults
restart: on-failure
image: clickhouse/clickhouse-server:22.8.8-alpine
image: clickhouse/clickhouse-server:23.7.3-alpine
tty: true
depends_on:
- zookeeper-1
@ -32,8 +31,7 @@ x-clickhouse-defaults:
soft: 262144
hard: 262144
x-clickhouse-depends:
&clickhouse-depends
x-clickhouse-depends: &clickhouse-depends
depends_on:
clickhouse:
condition: service_healthy

14
pkg/query-service/utils/format.go
View File

@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
"go.uber.org/zap"
)
@ -230,6 +231,19 @@ func getPointerValue(v interface{}) interface{} {
}
}
func GetClickhouseColumnName(typeName string, dataType, field string) string {
if typeName == string(v3.AttributeKeyTypeTag) {
typeName = constants.Attributes
}
if typeName != string(v3.AttributeKeyTypeResource) {
typeName = typeName[:len(typeName)-1]
}
colName := fmt.Sprintf("%s_%s_%s", strings.ToLower(typeName), strings.ToLower(dataType), field)
return colName
}
// GetEpochNanoSecs takes epoch and returns it in ns
func GetEpochNanoSecs(epoch int64) int64 {
temp := epoch

49
pkg/query-service/utils/format_test.go
View File

@ -4,6 +4,7 @@ import (
"reflect"
"testing"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
@ -388,6 +389,54 @@ func TestClickHouseFormattedValue(t *testing.T) {
}
}
var testGetClickhouseColumnName = []struct {
name string
typeName string
dataType string
field string
want string
}{
{
name: "tag",
typeName: string(v3.AttributeKeyTypeTag),
dataType: string(v3.AttributeKeyDataTypeInt64),
field: "tag1",
want: "attribute_int64_tag1",
},
{
name: "resource",
typeName: string(v3.AttributeKeyTypeResource),
dataType: string(v3.AttributeKeyDataTypeInt64),
field: "tag1",
want: "resource_int64_tag1",
},
{
name: "attribute old parser",
typeName: constants.Attributes,
dataType: string(v3.AttributeKeyDataTypeInt64),
field: "tag1",
want: "attribute_int64_tag1",
},
{
name: "resource old parser",
typeName: constants.Resources,
dataType: string(v3.AttributeKeyDataTypeInt64),
field: "tag1",
want: "resource_int64_tag1",
},
}
func TestGetClickhouseColumnName(t *testing.T) {
for _, tt := range testGetClickhouseColumnName {
t.Run(tt.name, func(t *testing.T) {
got := GetClickhouseColumnName(tt.typeName, tt.dataType, tt.field)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ClickHouseFormattedValue() = %v, want %v", got, tt.want)
}
})
}
}
var testGetEpochNanoSecsData = []struct {
Name string
Epoch int64