chore: add field type definitions for qb v5 (#7552)

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.0
 	github.com/gosimple/slug v1.10.0
+	github.com/huandu/go-sqlbuilder v1.35.0
 	github.com/jackc/pgx/v5 v5.7.2
 	github.com/jmoiron/sqlx v1.3.4
 	github.com/json-iterator/go v1.1.12
@@ -151,6 +152,7 @@ require (
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/memberlist v0.5.1 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect

go.sum

@@ -539,6 +539,12 @@ github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOF
 github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs=
 github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/go-assert v1.1.6 h1:oaAfYxq9KNDi9qswn/6aE0EydfxSa+tWZC1KabNitYs=
+github.com/huandu/go-assert v1.1.6/go.mod h1:JuIfbmYG9ykwvuxoJ3V8TB5QP+3+ajIA54Y44TmkMxs=
+github.com/huandu/go-sqlbuilder v1.35.0 h1:ESvxFHN8vxCTudY1Vq63zYpU5yJBESn19sf6k4v2T5Q=
+github.com/huandu/go-sqlbuilder v1.35.0/go.mod h1:mS0GAtrtW+XL6nM2/gXHRJax2RwSW1TraavWDFAc1JA=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=

pkg/types/querybuildertypes/querybuildertypesv5/qb.go

@@ -0,0 +1,54 @@
+package types
+
+import (
+	"context"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+// FilterOperator is the operator for the filter.
+type FilterOperator int
+
+const (
+	FilterOperatorUnknown FilterOperator = iota
+	FilterOperatorEqual
+	FilterOperatorNotEqual
+	FilterOperatorGreaterThan
+	FilterOperatorGreaterThanOrEq
+	FilterOperatorLessThan
+	FilterOperatorLessThanOrEq
+
+	FilterOperatorLike
+	FilterOperatorNotLike
+	FilterOperatorILike
+	FilterOperatorNotILike
+
+	FilterOperatorBetween
+	FilterOperatorNotBetween
+
+	FilterOperatorIn
+	FilterOperatorNotIn
+
+	FilterOperatorExists
+	FilterOperatorNotExists
+
+	FilterOperatorRegexp
+	FilterOperatorNotRegexp
+
+	FilterOperatorContains
+	FilterOperatorNotContains
+)
+
+// ConditionBuilder is the interface for building the condition part of the query.
+type ConditionBuilder interface {
+	// GetColumn returns the column for the given key.
+	GetColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error)
+
+	// GetTableFieldName returns the table field name for the given key.
+	GetTableFieldName(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error)
+
+	// GetCondition returns the condition for the given key, operator and value.
+	GetCondition(ctx context.Context, key *telemetrytypes.TelemetryFieldKey, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error)
+}

pkg/types/telemetrytypes/field.go

@@ -0,0 +1,125 @@
+package telemetrytypes
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+// FieldSelectorMatchType is the match type of the field key selector.
+type FieldSelectorMatchType struct {
+	valuer.String
+}
+
+var (
+	FieldSelectorMatchTypeExact = FieldSelectorMatchType{valuer.NewString("exact")}
+	FieldSelectorMatchTypeFuzzy = FieldSelectorMatchType{valuer.NewString("fuzzy")}
+)
+
+type TelemetryFieldKey struct {
+	Name          string        `json:"name"`
+	Description   string        `json:"description,omitempty"`
+	Unit          string        `json:"unit,omitempty"`
+	Signal        Signal        `json:"signal,omitempty"`
+	FieldContext  FieldContext  `json:"fieldContext,omitempty"`
+	FieldDataType FieldDataType `json:"fieldDataType,omitempty"`
+	Materialized  bool          `json:"materialized,omitempty"`
+}
+
+// GetFieldKeyFromKeyText returns a TelemetryFieldKey from a key text.
+// The key text is expected to be in the format of `fieldContext.fieldName:fieldDataType` in the search query.
+func GetFieldKeyFromKeyText(key string) TelemetryFieldKey {
+
+	keyTextParts := strings.Split(key, ".")
+
+	var explicitFieldContextProvided, explicitFieldDataTypeProvided bool
+	var explicitFieldContext FieldContext
+	var explicitFieldDataType FieldDataType
+	var ok bool
+
+	if len(keyTextParts) > 1 {
+		explicitFieldContext, ok = fieldContexts[keyTextParts[0]]
+		if ok && explicitFieldContext != FieldContextUnspecified {
+			explicitFieldContextProvided = true
+		}
+	}
+
+	if explicitFieldContextProvided {
+		keyTextParts = keyTextParts[1:]
+	}
+
+	// check if there is a field data type provided
+	if len(keyTextParts) >= 1 {
+		lastPart := keyTextParts[len(keyTextParts)-1]
+		lastPartParts := strings.Split(lastPart, ":")
+		if len(lastPartParts) > 1 {
+			explicitFieldDataType, ok = fieldDataTypes[lastPartParts[1]]
+			if ok && explicitFieldDataType != FieldDataTypeUnspecified {
+				explicitFieldDataTypeProvided = true
+			}
+		}
+
+		if explicitFieldDataTypeProvided {
+			keyTextParts[len(keyTextParts)-1] = lastPartParts[0]
+		}
+	}
+
+	realKey := strings.Join(keyTextParts, ".")
+
+	fieldKeySelector := TelemetryFieldKey{
+		Name: realKey,
+	}
+
+	if explicitFieldContextProvided {
+		fieldKeySelector.FieldContext = explicitFieldContext
+	} else {
+		fieldKeySelector.FieldContext = FieldContextUnspecified
+	}
+
+	if explicitFieldDataTypeProvided {
+		fieldKeySelector.FieldDataType = explicitFieldDataType
+	} else {
+		fieldKeySelector.FieldDataType = FieldDataTypeUnspecified
+	}
+
+	return fieldKeySelector
+}
+
+func FieldKeyToMaterializedColumnName(key TelemetryFieldKey) string {
+	return fmt.Sprintf("%s_%s_%s", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$"))
+}
+
+func FieldKeyToMaterializedColumnNameForExists(key TelemetryFieldKey) string {
+	return fmt.Sprintf("%s_%s_%s_exists", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$"))
+}
+
+type TelemetryFieldValues struct {
+	StringValues  []string  `json:"stringValues,omitempty"`
+	BoolValues    []bool    `json:"boolValues,omitempty"`
+	NumberValues  []float64 `json:"numberValues,omitempty"`
+	RelatedValues []string  `json:"relatedValues,omitempty"`
+}
+
+type MetricContext struct {
+	MetricName string `json:"metricName"`
+}
+
+type FieldKeySelector struct {
+	StartUnixMilli    int64                  `json:"startUnixMilli"`
+	EndUnixMilli      int64                  `json:"endUnixMilli"`
+	Signal            Signal                 `json:"signal"`
+	FieldContext      FieldContext           `json:"fieldContext"`
+	FieldDataType     FieldDataType          `json:"fieldDataType"`
+	Name              string                 `json:"name"`
+	SelectorMatchType FieldSelectorMatchType `json:"selectorMatchType"`
+	Limit             int                    `json:"limit"`
+	MetricContext     *MetricContext         `json:"metricContext,omitempty"`
+}
+
+type FieldValueSelector struct {
+	FieldKeySelector
+	ExistingQuery string `json:"existingQuery"`
+	Value         string `json:"value"`
+	Limit         int    `json:"limit"`
+}

pkg/types/telemetrytypes/field_context.go

@@ -0,0 +1,148 @@
+package telemetrytypes
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+// FieldContext is the context of the field being queried. It is expected to be used to disambiguate b/w
+// different contexts of the same field.
+//
+// - Use `resource.` prefix to the key to explicitly indicate and enforce resource context. Example
+//   - `resource.service.name`
+//   - `resource.k8s.namespace.name`
+//
+// - Use `scope.` prefix to explicitly indicate and enforce scope context. Example
+//   - `scope.name`
+//   - `scope.version`
+//   - `scope.my.custom.attribute` and `scope.attribute.my.custom.attribute` resolve to same attribute
+//
+// - Use `attribute.` to explicitly indicate and enforce attribute context. Example
+//   - `attribute.http.method`
+//   - `attribute.http.target`
+//
+// - Use `event.` to indicate and enforce event context and `event.attribute` to disambiguate b/w `event.name` and `event.attribute.name` . Examples
+//   - `event.name` will look for event name
+//   - `event.record_entry` will look for `record_entry` attribute in event
+//   - `event.attribute.name` will look for `name` attribute event
+//
+// - Use `span.` to indicate the span context.
+//   - `span.name` will resolve to the name of span
+//   - `span.kind` will resolve to the kind of span
+//   - `span.http.method` will resolve to `http.method` of attribute
+//
+// - Use `log.` for explicit log context
+//   - `log.severity_text` will always resolve to `severity_text` of log record
+type FieldContext struct {
+	valuer.String
+}
+
+var (
+	FieldContextMetric      = FieldContext{valuer.NewString("metric")}
+	FieldContextLog         = FieldContext{valuer.NewString("log")}
+	FieldContextSpan        = FieldContext{valuer.NewString("span")}
+	FieldContextTrace       = FieldContext{valuer.NewString("trace")}
+	FieldContextResource    = FieldContext{valuer.NewString("resource")}
+	FieldContextScope       = FieldContext{valuer.NewString("scope")}
+	FieldContextAttribute   = FieldContext{valuer.NewString("attribute")}
+	FieldContextEvent       = FieldContext{valuer.NewString("event")}
+	FieldContextUnspecified = FieldContext{valuer.NewString("")}
+
+	// Map string representations to FieldContext values
+	// We wouldn't need if not for the fact that we have historically used
+	// "tag" and "attribute" interchangeably.
+	// This means elsewhere in the system, we have used "tag" to refer to "attribute".
+	// There are DB entries that use "tag" and "attribute" interchangeably.
+	// This is a stop gap measure to ensure that we can still use the existing
+	// DB entries.
+	fieldContexts = map[string]FieldContext{
+		"resource":   FieldContextResource,
+		"scope":      FieldContextScope,
+		"tag":        FieldContextAttribute,
+		"attribute":  FieldContextAttribute,
+		"event":      FieldContextEvent,
+		"spanfield":  FieldContextSpan,
+		"span":       FieldContextSpan,
+		"logfield":   FieldContextLog,
+		"log":        FieldContextLog,
+		"metric":     FieldContextMetric,
+		"tracefield": FieldContextTrace,
+	}
+)
+
+// UnmarshalJSON implements the json.Unmarshaler interface
+func (f *FieldContext) UnmarshalJSON(data []byte) error {
+	var str string
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+
+	// Normalize the string
+	normalizedStr := strings.ToLower(strings.TrimSpace(str))
+
+	// Look up the context in our map
+	if ctx, exists := fieldContexts[normalizedStr]; exists {
+		*f = ctx
+		return nil
+	}
+
+	// Default to unspecified if not found
+	*f = FieldContextUnspecified
+	return nil
+}
+
+// Scan implements the sql.Scanner interface
+func (f *FieldContext) Scan(value interface{}) error {
+	if f == nil {
+		return fmt.Errorf("fieldcontext: nil receiver")
+	}
+
+	if value == nil {
+		*f = FieldContextUnspecified
+		return nil
+	}
+
+	str, ok := value.(string)
+	if !ok {
+		return fmt.Errorf("fieldcontext: expected string, got %T", value)
+	}
+
+	// Normalize the string
+	normalizedStr := strings.ToLower(strings.TrimSpace(str))
+
+	// Look up the context in our map
+	if ctx, exists := fieldContexts[normalizedStr]; exists {
+		*f = ctx
+		return nil
+	}
+
+	// Default to unspecified if not found
+	*f = FieldContextUnspecified
+	return nil
+}
+
+// TagType returns the tag type for the field context.
+func (f FieldContext) TagType() string {
+	switch f {
+	case FieldContextResource:
+		return "resource"
+	case FieldContextScope:
+		return "scope"
+	case FieldContextAttribute:
+		return "tag"
+	case FieldContextLog:
+		return "logfield"
+	case FieldContextSpan:
+		return "spanfield"
+	case FieldContextTrace:
+		return "tracefield"
+	case FieldContextMetric:
+		return "metricfield"
+	case FieldContextEvent:
+		return "eventfield"
+	}
+	return ""
+}

pkg/types/telemetrytypes/field_datatype.go

@@ -0,0 +1,121 @@
+package telemetrytypes
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+// FieldDataType is the data type of the field. It is expected to be used to disambiguate b/w
+// different data types of the same field.
+type FieldDataType struct {
+	valuer.String
+}
+
+var (
+	FieldDataTypeString  = FieldDataType{valuer.NewString("string")}
+	FieldDataTypeBool    = FieldDataType{valuer.NewString("bool")}
+	FieldDataTypeFloat64 = FieldDataType{valuer.NewString("float64")}
+	// int64 and number are synonyms for float64
+	FieldDataTypeInt64       = FieldDataType{valuer.NewString("int64")}
+	FieldDataTypeNumber      = FieldDataType{valuer.NewString("number")}
+	FieldDataTypeUnspecified = FieldDataType{valuer.NewString("")}
+
+	// Map string representations to FieldDataType values
+	// We want to handle all the possible string representations of the data types.
+	// Even if the user uses some non-standard representation, we want to be able to
+	// parse it correctly.
+	fieldDataTypes = map[string]FieldDataType{
+		// String types
+		"string": FieldDataTypeString,
+
+		// Boolean types
+		"bool": FieldDataTypeBool,
+
+		// Integer types
+		"int":    FieldDataTypeNumber,
+		"int8":   FieldDataTypeNumber,
+		"int16":  FieldDataTypeNumber,
+		"int32":  FieldDataTypeNumber,
+		"int64":  FieldDataTypeNumber,
+		"uint":   FieldDataTypeNumber,
+		"uint8":  FieldDataTypeNumber,
+		"uint16": FieldDataTypeNumber,
+		"uint32": FieldDataTypeNumber,
+		"uint64": FieldDataTypeNumber,
+
+		// Float types
+		"float":   FieldDataTypeNumber,
+		"float32": FieldDataTypeNumber,
+		"float64": FieldDataTypeNumber,
+		"double":  FieldDataTypeNumber,
+		"decimal": FieldDataTypeNumber,
+		"number":  FieldDataTypeNumber,
+	}
+)
+
+// UnmarshalJSON implements the json.Unmarshaler interface
+func (f *FieldDataType) UnmarshalJSON(data []byte) error {
+	var str string
+	if err := json.Unmarshal(data, &str); err != nil {
+		return err
+	}
+
+	// Normalize the string
+	normalizedStr := strings.ToLower(strings.TrimSpace(str))
+
+	// Look up the data type in our map
+	if dataType, exists := fieldDataTypes[normalizedStr]; exists {
+		*f = dataType
+		return nil
+	}
+
+	// Default to unspecified if not found
+	*f = FieldDataTypeUnspecified
+	return nil
+}
+
+// Scan implements the sql.Scanner interface
+func (f *FieldDataType) Scan(value interface{}) error {
+	if f == nil {
+		return fmt.Errorf("fielddatatype: nil receiver")
+	}
+
+	if value == nil {
+		*f = FieldDataTypeUnspecified
+		return nil
+	}
+
+	str, ok := value.(string)
+	if !ok {
+		return fmt.Errorf("fielddatatype: expected string, got %T", value)
+	}
+
+	// Normalize the string
+	normalizedStr := strings.ToLower(strings.TrimSpace(str))
+
+	// Look up the data type in our map
+	if dataType, exists := fieldDataTypes[normalizedStr]; exists {
+		*f = dataType
+		return nil
+	}
+
+	// Default to unspecified if not found
+	*f = FieldDataTypeUnspecified
+	return nil
+}
+
+func (f FieldDataType) TagDataType() string {
+	switch f {
+	case FieldDataTypeString:
+		return "string"
+	case FieldDataTypeBool:
+		return "bool"
+	case FieldDataTypeNumber, FieldDataTypeInt64, FieldDataTypeFloat64:
+		return "float64"
+	default:
+		return "string"
+	}
+}

pkg/types/telemetrytypes/field_test.go

@@ -0,0 +1,93 @@
+package telemetrytypes
+
+import (
+	"testing"
+)
+
+func TestGetFieldKeyFromKeyText(t *testing.T) {
+
+	testCases := []struct {
+		keyText  string
+		expected TelemetryFieldKey
+	}{
+		{
+			keyText: "resource.service.name:string",
+			expected: TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  FieldContextResource,
+				FieldDataType: FieldDataTypeString,
+			},
+		},
+		{
+			keyText: "scope.name",
+			expected: TelemetryFieldKey{
+				Name:          "name",
+				FieldContext:  FieldContextScope,
+				FieldDataType: FieldDataTypeUnspecified,
+			},
+		},
+		{
+			keyText: "scope.version",
+			expected: TelemetryFieldKey{
+				Name:          "version",
+				FieldContext:  FieldContextScope,
+				FieldDataType: FieldDataTypeUnspecified,
+			},
+		},
+		{
+			keyText: "attribute.http.method",
+			expected: TelemetryFieldKey{
+				Name:          "http.method",
+				FieldContext:  FieldContextAttribute,
+				FieldDataType: FieldDataTypeUnspecified,
+			},
+		},
+		{
+			keyText: "span.name",
+			expected: TelemetryFieldKey{
+				Name:          "name",
+				FieldContext:  FieldContextSpan,
+				FieldDataType: FieldDataTypeUnspecified,
+			},
+		},
+		{
+			keyText: "span.kind:string",
+			expected: TelemetryFieldKey{
+				Name:          "kind",
+				FieldContext:  FieldContextSpan,
+				FieldDataType: FieldDataTypeString,
+			},
+		},
+		{
+			keyText: "span.kind:int",
+			expected: TelemetryFieldKey{
+				Name:          "kind",
+				FieldContext:  FieldContextSpan,
+				FieldDataType: FieldDataTypeNumber,
+			},
+		},
+		{
+			keyText: "span.http.status_code:int",
+			expected: TelemetryFieldKey{
+				Name:          "http.status_code",
+				FieldContext:  FieldContextSpan,
+				FieldDataType: FieldDataTypeNumber,
+			},
+		},
+		{
+			keyText: "log.severity_text",
+			expected: TelemetryFieldKey{
+				Name:          "severity_text",
+				FieldContext:  FieldContextLog,
+				FieldDataType: FieldDataTypeUnspecified,
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		result := GetFieldKeyFromKeyText(testCase.keyText)
+		if result != testCase.expected {
+			t.Errorf("expected %v, got %v", testCase.expected, result)
+		}
+	}
+}

pkg/types/telemetrytypes/signal.go

@@ -0,0 +1,14 @@
+package telemetrytypes
+
+import "github.com/SigNoz/signoz/pkg/valuer"
+
+type Signal struct {
+	valuer.String
+}
+
+var (
+	SignalTraces      = Signal{valuer.NewString("traces")}
+	SignalLogs        = Signal{valuer.NewString("logs")}
+	SignalMetrics     = Signal{valuer.NewString("metrics")}
+	SignalUnspecified = Signal{valuer.NewString("")}
+)

pkg/types/telemetrytypes/store.go

@@ -0,0 +1,25 @@
+package telemetrytypes
+
+import (
+	"context"
+)
+
+// MetadataStore is the interface for the telemetry metadata store.
+type MetadataStore interface {
+	// GetKeys returns a map of field keys types.TelemetryFieldKey by name, there can be multiple keys with the same name
+	// if they have different types or data types.
+	GetKeys(ctx context.Context, fieldKeySelector *FieldKeySelector) (map[string][]*TelemetryFieldKey, error)
+
+	// GetKeys but with any number of fieldKeySelectors.
+	GetKeysMulti(ctx context.Context, fieldKeySelectors []*FieldKeySelector) (map[string][]*TelemetryFieldKey, error)
+
+	// GetKey returns a list of keys with the given name.
+	GetKey(ctx context.Context, fieldKeySelector *FieldKeySelector) ([]*TelemetryFieldKey, error)
+
+	// GetRelatedValues returns a list of related values for the given key name
+	// and the existing selection of keys.
+	GetRelatedValues(ctx context.Context, fieldValueSelector *FieldValueSelector) ([]string, error)
+
+	// GetAllValues returns a list of all values.
+	GetAllValues(ctx context.Context, fieldValueSelector *FieldValueSelector) (*TelemetryFieldValues, error)
+}

pkg/types/telemetrytypes/telemetrystmt.go

@@ -0,0 +1,5 @@
+package telemetrytypes
+
+type ShowCreateTableStatement struct {
+	Statement string `json:"statement" ch:"statement"`
+}