From e2df2e7c41e03d1c2920262a51b081bb4e5c0107 Mon Sep 17 00:00:00 2001
From: Vishal Sharma <makeavish786@gmail.com>
Date: Mon, 24 Apr 2023 20:38:08 +0530
Subject: [PATCH 1/2] chore: validateAndCastValueData (#2610)

---
 pkg/query-service/utils/format.go      | 132 +++++++++++
 pkg/query-service/utils/format_test.go | 293 +++++++++++++++++++++++++
 2 files changed, 425 insertions(+)
 create mode 100644 pkg/query-service/utils/format_test.go

diff --git a/pkg/query-service/utils/format.go b/pkg/query-service/utils/format.go
index af722a70d3..3316205966 100644
--- a/pkg/query-service/utils/format.go
+++ b/pkg/query-service/utils/format.go
@@ -3,11 +3,143 @@ package utils
 import (
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
 
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.uber.org/zap"
 )
 
+func ValidateAndCastValue(v interface{}, dataType v3.AttributeKeyDataType) (interface{}, error) {
+	switch dataType {
+	case v3.AttributeKeyDataTypeString:
+		switch x := v.(type) {
+		case string, int, int64, float32, float64, bool:
+			return fmt.Sprintf("%v", x), nil
+		case []interface{}:
+			for i, val := range x {
+				// if val is not string and it is int, int64, float, bool, convert it to string
+				if _, ok := val.(string); ok {
+					continue
+				} else if _, ok := val.(int); ok {
+					x[i] = fmt.Sprintf("%v", val)
+				} else if _, ok := val.(int64); ok {
+					x[i] = fmt.Sprintf("%v", val)
+				} else if _, ok := val.(float32); ok {
+					x[i] = fmt.Sprintf("%v", val)
+				} else if _, ok := val.(float64); ok {
+					x[i] = fmt.Sprintf("%v", val)
+				} else if _, ok := val.(bool); ok {
+					x[i] = fmt.Sprintf("%v", val)
+				} else {
+					return nil, fmt.Errorf("invalid data type, expected string, got %v", reflect.TypeOf(val))
+				}
+			}
+			return x, nil
+		default:
+			return nil, fmt.Errorf("invalid data type, expected string, got %v", reflect.TypeOf(v))
+		}
+	case v3.AttributeKeyDataTypeBool:
+		switch x := v.(type) {
+		case []interface{}:
+			for i, val := range x {
+				if _, ok := val.(string); ok {
+					boolean, err := strconv.ParseBool(val.(string))
+					if err != nil {
+						return nil, fmt.Errorf("invalid data type, expected bool, got %v", reflect.TypeOf(val))
+					}
+					x[i] = boolean
+				} else if _, ok := val.(bool); !ok {
+					return nil, fmt.Errorf("invalid data type, expected bool, got %v", reflect.TypeOf(val))
+				} else {
+					x[i] = val.(bool)
+				}
+			}
+			return x, nil
+		case bool:
+			return x, nil
+		case string:
+			boolean, err := strconv.ParseBool(x)
+			if err != nil {
+				return nil, fmt.Errorf("invalid data type, expected bool, got %v", reflect.TypeOf(v))
+			}
+			return boolean, nil
+		default:
+			return nil, fmt.Errorf("invalid data type, expected bool, got %v", reflect.TypeOf(v))
+		}
+	case v3.AttributeKeyDataTypeInt64:
+		switch x := v.(type) {
+		case []interface{}:
+			for i, val := range x {
+				if _, ok := val.(string); ok {
+					int64val, err := strconv.ParseInt(val.(string), 10, 64)
+					if err != nil {
+						return nil, fmt.Errorf("invalid data type, expected int, got %v", reflect.TypeOf(val))
+					}
+					x[i] = int64val
+				} else if _, ok := val.(int); ok {
+					x[i] = int64(val.(int))
+				} else if _, ok := val.(int64); !ok {
+					return nil, fmt.Errorf("invalid data type, expected int, got %v", reflect.TypeOf(val))
+				} else {
+					x[i] = val.(int64)
+				}
+			}
+			return x, nil
+		case int, int64:
+			return x, nil
+		case string:
+			int64val, err := strconv.ParseInt(x, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid data type, expected int, got %v", reflect.TypeOf(v))
+			}
+			return int64val, nil
+		default:
+			return nil, fmt.Errorf("invalid data type, expected int, got %v", reflect.TypeOf(v))
+		}
+	case v3.AttributeKeyDataTypeFloat64:
+		switch x := v.(type) {
+		case []interface{}:
+			for i, val := range x {
+				if _, ok := val.(string); ok {
+					float64val, err := strconv.ParseFloat(val.(string), 64)
+					if err != nil {
+						return nil, fmt.Errorf("invalid data type, expected float, got %v", reflect.TypeOf(val))
+					}
+					x[i] = float64val
+				} else if _, ok := val.(float32); ok {
+					x[i] = float64(val.(float32))
+				} else if _, ok := val.(int); ok {
+					x[i] = float64(val.(int))
+				} else if _, ok := val.(int64); ok {
+					x[i] = float64(val.(int64))
+				} else if _, ok := val.(float64); !ok {
+					return nil, fmt.Errorf("invalid data type, expected float, got %v", reflect.TypeOf(val))
+				} else {
+					x[i] = val.(float64)
+				}
+			}
+			return x, nil
+		case float32, float64:
+			return x, nil
+		case string:
+			float64val, err := strconv.ParseFloat(x, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid data type, expected float, got %v", reflect.TypeOf(v))
+			}
+			return float64val, nil
+		case int:
+			return float64(x), nil
+		case int64:
+			return float64(x), nil
+		default:
+			return nil, fmt.Errorf("invalid data type, expected float, got %v", reflect.TypeOf(v))
+		}
+	default:
+		return nil, fmt.Errorf("invalid data type, expected float, bool, int, string or []interface{} but got %v", dataType)
+	}
+}
+
 // ClickHouseFormattedValue formats the value to be used in clickhouse query
 func ClickHouseFormattedValue(v interface{}) string {
 	switch x := v.(type) {
diff --git a/pkg/query-service/utils/format_test.go b/pkg/query-service/utils/format_test.go
new file mode 100644
index 0000000000..85482381ff
--- /dev/null
+++ b/pkg/query-service/utils/format_test.go
@@ -0,0 +1,293 @@
+package utils
+
+import (
+	"reflect"
+	"testing"
+
+	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
+)
+
+type args struct {
+	v        interface{}
+	dataType v3.AttributeKeyDataType
+}
+
+var testValidateAndCastValueData = []struct {
+	name    string
+	args    args
+	want    interface{}
+	wantErr bool
+}{
+	// Test cases for v3.AttributeKeyDataTypeString
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid string",
+		args: args{
+			v:        "test",
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    "test",
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid int",
+		args: args{
+			v:        1,
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    "1",
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid float32",
+		args: args{
+			v:        float32(1.1),
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    "1.1",
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid float64",
+		args: args{
+			v:        float64(1.1),
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    "1.1",
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid bool",
+		args: args{
+			v:        true,
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    "true",
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid []interface{}",
+		args: args{
+			v:        []interface{}{"test", "test2"},
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    []interface{}{"test", "test2"},
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Valid []interface{}",
+		args: args{
+			v:        []interface{}{"test", 1},
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    []interface{}{"test", "1"},
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Invalid []interface{}",
+		args: args{
+			v:        []interface{}{"test", [1]string{"string Array"}},
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeString: Invalid type",
+		args: args{
+			v:        map[string]interface{}{"test": "test"},
+			dataType: v3.AttributeKeyDataTypeString,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	// Test cases for v3.AttributeKeyDataTypeBool
+	{
+		name: "v3.AttributeKeyDataTypeBool: Valid bool",
+		args: args{
+			v:        true,
+			dataType: v3.AttributeKeyDataTypeBool,
+		},
+		want:    true,
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeBool: Valid string",
+		args: args{
+			v:        "true",
+			dataType: v3.AttributeKeyDataTypeBool,
+		},
+		want:    true,
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeBool: Valid []interface{}",
+		args: args{
+			v:        []interface{}{"true", false},
+			dataType: v3.AttributeKeyDataTypeBool,
+		},
+		want:    []interface{}{true, false},
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeBool: Invalid type",
+		args: args{
+			v:        1,
+			dataType: v3.AttributeKeyDataTypeBool,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeBool: Invalid []interface{}",
+		args: args{
+			v:        []interface{}{1, false},
+			dataType: v3.AttributeKeyDataTypeBool,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	// Test cases for v3.AttributeKeyDataTypeInt64
+	{
+		name: "v3.AttributeKeyDataTypeInt64: Valid int",
+		args: args{
+			v:        1,
+			dataType: v3.AttributeKeyDataTypeInt64,
+		},
+		want:    1,
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeInt64: Valid int64",
+		args: args{
+			v:        int64(1),
+			dataType: v3.AttributeKeyDataTypeInt64,
+		},
+		want:    int64(1),
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeInt64: Valid string",
+		args: args{
+			v:        "1",
+			dataType: v3.AttributeKeyDataTypeInt64,
+		},
+		want:    int64(1),
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeInt64: Valid []interface{}",
+		args: args{
+			v:        []interface{}{"1", 2},
+			dataType: v3.AttributeKeyDataTypeInt64,
+		},
+		want:    []interface{}{int64(1), int64(2)},
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeInt64: Invalid []interface{}",
+		args: args{
+			v:        []interface{}{"1", false},
+			dataType: v3.AttributeKeyDataTypeInt64,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeInt64: Invalid type",
+		args: args{
+			v:        true,
+			dataType: v3.AttributeKeyDataTypeInt64,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	// Test cases for v3.AttributeKeyDataTypeFloat64
+	{
+		name: "v3.AttributeKeyDataTypeFloat64: Valid float32",
+		args: args{
+			v:        float32(1.1),
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    float32(1.1),
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat64: Valid float64",
+		args: args{
+			v:        float64(1.1),
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    float64(1.1),
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat64: Valid int",
+		args: args{
+			v:        1,
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    float64(1),
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat64: Valid string",
+		args: args{
+			v:        "1.1",
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    float64(1.1),
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat: Valid []interface{}",
+		args: args{
+			v:        []interface{}{4, 3},
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    []interface{}{float64(4), float64(3)},
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat: Valid []interface{}",
+		args: args{
+			v:        []interface{}{4, "3"},
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    []interface{}{float64(4), float64(3)},
+		wantErr: false,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat: Invalid []interface{}",
+		args: args{
+			v:        []interface{}{4, "true"},
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+	{
+		name: "v3.AttributeKeyDataTypeFloat64: Invalid type",
+		args: args{
+			v:        true,
+			dataType: v3.AttributeKeyDataTypeFloat64,
+		},
+		want:    nil,
+		wantErr: true,
+	},
+}
+
+// Test cases for ValidateAndCastValue function in pkg/query-service/utils/format.go
+func TestValidateAndCastValue(t *testing.T) {
+	for _, tt := range testValidateAndCastValueData {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ValidateAndCastValue(tt.args.v, tt.args.dataType)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ValidateAndCastValue() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) && !reflect.DeepEqual(err, tt.wantErr) {
+				t.Errorf("ValidateAndCastValue() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

From 3e97d2ffa38e013a251d99ccc867f3be2ed6a77d Mon Sep 17 00:00:00 2001
From: Prashant Shahi <prashant@signoz.io>
Date: Mon, 24 Apr 2023 21:37:33 +0530
Subject: [PATCH 2/2] =?UTF-8?q?feat(clickhouse):=20=E2=9C=A8=20support=20f?=
 =?UTF-8?q?or=20Google=20Cloud=20Storage=20(GCS)=20in=20Docker/Swarm=20(#2?=
 =?UTF-8?q?605)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Prashant Shahi <prashant@signoz.io>
---
 .../clickhouse-setup/clickhouse-storage.xml        | 14 +++++++++++++-
 .../docker/clickhouse-setup/clickhouse-storage.xml | 14 +++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml b/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml
index 2b2f4010ac..54ec4976f5 100644
--- a/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml
+++ b/deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml
@@ -7,9 +7,21 @@
         </default>
         <s3>
             <type>s3</type>
-            <endpoint>https://BUCKET-NAME.s3.amazonaws.com/data/</endpoint>
+            <!-- For S3 cold storage,
+                    if region is us-east-1, endpoint can be https://<bucket-name>.s3.amazonaws.com
+                    if region is not us-east-1, endpoint should be https://<bucket-name>.s3-<region>.amazonaws.com
+                For GCS cold storage,
+                    endpoint should be https://storage.googleapis.com/<bucket-name>/data/
+                -->
+            <endpoint>https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/</endpoint>
             <access_key_id>ACCESS-KEY-ID</access_key_id>
             <secret_access_key>SECRET-ACCESS-KEY</secret_access_key>
+            <!-- In case of S3, uncomment the below configuration in case you want to read
+                AWS credentials from the Environment variables if they exist. -->
+            <!-- <use_environment_credentials>true</use_environment_credentials> -->
+            <!-- In case of GCS, uncomment the below configuration, since GCS does
+                not support batch deletion and result in error messages in logs. -->
+            <!-- <support_batch_delete>false</support_batch_delete> -->
         </s3>
    </disks>
    <policies>
diff --git a/deploy/docker/clickhouse-setup/clickhouse-storage.xml b/deploy/docker/clickhouse-setup/clickhouse-storage.xml
index 2b2f4010ac..54ec4976f5 100644
--- a/deploy/docker/clickhouse-setup/clickhouse-storage.xml
+++ b/deploy/docker/clickhouse-setup/clickhouse-storage.xml
@@ -7,9 +7,21 @@
         </default>
         <s3>
             <type>s3</type>
-            <endpoint>https://BUCKET-NAME.s3.amazonaws.com/data/</endpoint>
+            <!-- For S3 cold storage,
+                    if region is us-east-1, endpoint can be https://<bucket-name>.s3.amazonaws.com
+                    if region is not us-east-1, endpoint should be https://<bucket-name>.s3-<region>.amazonaws.com
+                For GCS cold storage,
+                    endpoint should be https://storage.googleapis.com/<bucket-name>/data/
+                -->
+            <endpoint>https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/</endpoint>
             <access_key_id>ACCESS-KEY-ID</access_key_id>
             <secret_access_key>SECRET-ACCESS-KEY</secret_access_key>
+            <!-- In case of S3, uncomment the below configuration in case you want to read
+                AWS credentials from the Environment variables if they exist. -->
+            <!-- <use_environment_credentials>true</use_environment_credentials> -->
+            <!-- In case of GCS, uncomment the below configuration, since GCS does
+                not support batch deletion and result in error messages in logs. -->
+            <!-- <support_batch_delete>false</support_batch_delete> -->
         </s3>
    </disks>
    <policies>