From 8ff05b2e8ff5af59d38e392679c163e83e94fa5f Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 8 Apr 2025 22:34:58 +0530 Subject: [PATCH] chore: add field type definitions for qb v5 (#7552) --- go.mod | 2 + go.sum | 6 + .../querybuildertypesv5/qb.go | 54 +++++++ pkg/types/telemetrytypes/field.go | 125 +++++++++++++++ pkg/types/telemetrytypes/field_context.go | 148 ++++++++++++++++++ pkg/types/telemetrytypes/field_datatype.go | 121 ++++++++++++++ pkg/types/telemetrytypes/field_test.go | 93 +++++++++++ pkg/types/telemetrytypes/signal.go | 14 ++ pkg/types/telemetrytypes/store.go | 25 +++ pkg/types/telemetrytypes/telemetrystmt.go | 5 + 10 files changed, 593 insertions(+) create mode 100644 pkg/types/querybuildertypes/querybuildertypesv5/qb.go create mode 100644 pkg/types/telemetrytypes/field.go create mode 100644 pkg/types/telemetrytypes/field_context.go create mode 100644 pkg/types/telemetrytypes/field_datatype.go create mode 100644 pkg/types/telemetrytypes/field_test.go create mode 100644 pkg/types/telemetrytypes/signal.go create mode 100644 pkg/types/telemetrytypes/store.go create mode 100644 pkg/types/telemetrytypes/telemetrystmt.go diff --git a/go.mod b/go.mod index 14617ed3c9..f56553f575 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.0 github.com/gosimple/slug v1.10.0 + github.com/huandu/go-sqlbuilder v1.35.0 github.com/jackc/pgx/v5 v5.7.2 github.com/jmoiron/sqlx v1.3.4 github.com/json-iterator/go v1.1.12 @@ -151,6 +152,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/memberlist v0.5.1 // indirect + github.com/huandu/xstrings v1.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/go.sum b/go.sum index 2070a6c41d..f64a050562 100644 --- a/go.sum +++ b/go.sum @@ -539,6 +539,12 @@ github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOF github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/go-assert v1.1.6 h1:oaAfYxq9KNDi9qswn/6aE0EydfxSa+tWZC1KabNitYs= +github.com/huandu/go-assert v1.1.6/go.mod h1:JuIfbmYG9ykwvuxoJ3V8TB5QP+3+ajIA54Y44TmkMxs= +github.com/huandu/go-sqlbuilder v1.35.0 h1:ESvxFHN8vxCTudY1Vq63zYpU5yJBESn19sf6k4v2T5Q= +github.com/huandu/go-sqlbuilder v1.35.0/go.mod h1:mS0GAtrtW+XL6nM2/gXHRJax2RwSW1TraavWDFAc1JA= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go new file mode 100644 index 0000000000..9c1118977f --- /dev/null +++ b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go @@ -0,0 +1,54 @@ +package types + +import ( + "context" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" +) + +// FilterOperator is the operator for the filter. +type FilterOperator int + +const ( + FilterOperatorUnknown FilterOperator = iota + FilterOperatorEqual + FilterOperatorNotEqual + FilterOperatorGreaterThan + FilterOperatorGreaterThanOrEq + FilterOperatorLessThan + FilterOperatorLessThanOrEq + + FilterOperatorLike + FilterOperatorNotLike + FilterOperatorILike + FilterOperatorNotILike + + FilterOperatorBetween + FilterOperatorNotBetween + + FilterOperatorIn + FilterOperatorNotIn + + FilterOperatorExists + FilterOperatorNotExists + + FilterOperatorRegexp + FilterOperatorNotRegexp + + FilterOperatorContains + FilterOperatorNotContains +) + +// ConditionBuilder is the interface for building the condition part of the query. +type ConditionBuilder interface { + // GetColumn returns the column for the given key. + GetColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) + + // GetTableFieldName returns the table field name for the given key. + GetTableFieldName(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) + + // GetCondition returns the condition for the given key, operator and value. + GetCondition(ctx context.Context, key *telemetrytypes.TelemetryFieldKey, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) +} diff --git a/pkg/types/telemetrytypes/field.go b/pkg/types/telemetrytypes/field.go new file mode 100644 index 0000000000..5f435b4f7d --- /dev/null +++ b/pkg/types/telemetrytypes/field.go @@ -0,0 +1,125 @@ +package telemetrytypes + +import ( + "fmt" + "strings" + + "github.com/SigNoz/signoz/pkg/valuer" +) + +// FieldSelectorMatchType is the match type of the field key selector. +type FieldSelectorMatchType struct { + valuer.String +} + +var ( + FieldSelectorMatchTypeExact = FieldSelectorMatchType{valuer.NewString("exact")} + FieldSelectorMatchTypeFuzzy = FieldSelectorMatchType{valuer.NewString("fuzzy")} +) + +type TelemetryFieldKey struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Unit string `json:"unit,omitempty"` + Signal Signal `json:"signal,omitempty"` + FieldContext FieldContext `json:"fieldContext,omitempty"` + FieldDataType FieldDataType `json:"fieldDataType,omitempty"` + Materialized bool `json:"materialized,omitempty"` +} + +// GetFieldKeyFromKeyText returns a TelemetryFieldKey from a key text. +// The key text is expected to be in the format of `fieldContext.fieldName:fieldDataType` in the search query. +func GetFieldKeyFromKeyText(key string) TelemetryFieldKey { + + keyTextParts := strings.Split(key, ".") + + var explicitFieldContextProvided, explicitFieldDataTypeProvided bool + var explicitFieldContext FieldContext + var explicitFieldDataType FieldDataType + var ok bool + + if len(keyTextParts) > 1 { + explicitFieldContext, ok = fieldContexts[keyTextParts[0]] + if ok && explicitFieldContext != FieldContextUnspecified { + explicitFieldContextProvided = true + } + } + + if explicitFieldContextProvided { + keyTextParts = keyTextParts[1:] + } + + // check if there is a field data type provided + if len(keyTextParts) >= 1 { + lastPart := keyTextParts[len(keyTextParts)-1] + lastPartParts := strings.Split(lastPart, ":") + if len(lastPartParts) > 1 { + explicitFieldDataType, ok = fieldDataTypes[lastPartParts[1]] + if ok && explicitFieldDataType != FieldDataTypeUnspecified { + explicitFieldDataTypeProvided = true + } + } + + if explicitFieldDataTypeProvided { + keyTextParts[len(keyTextParts)-1] = lastPartParts[0] + } + } + + realKey := strings.Join(keyTextParts, ".") + + fieldKeySelector := TelemetryFieldKey{ + Name: realKey, + } + + if explicitFieldContextProvided { + fieldKeySelector.FieldContext = explicitFieldContext + } else { + fieldKeySelector.FieldContext = FieldContextUnspecified + } + + if explicitFieldDataTypeProvided { + fieldKeySelector.FieldDataType = explicitFieldDataType + } else { + fieldKeySelector.FieldDataType = FieldDataTypeUnspecified + } + + return fieldKeySelector +} + +func FieldKeyToMaterializedColumnName(key TelemetryFieldKey) string { + return fmt.Sprintf("%s_%s_%s", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$")) +} + +func FieldKeyToMaterializedColumnNameForExists(key TelemetryFieldKey) string { + return fmt.Sprintf("%s_%s_%s_exists", key.FieldContext, key.FieldDataType.String, strings.ReplaceAll(key.Name, ".", "$$")) +} + +type TelemetryFieldValues struct { + StringValues []string `json:"stringValues,omitempty"` + BoolValues []bool `json:"boolValues,omitempty"` + NumberValues []float64 `json:"numberValues,omitempty"` + RelatedValues []string `json:"relatedValues,omitempty"` +} + +type MetricContext struct { + MetricName string `json:"metricName"` +} + +type FieldKeySelector struct { + StartUnixMilli int64 `json:"startUnixMilli"` + EndUnixMilli int64 `json:"endUnixMilli"` + Signal Signal `json:"signal"` + FieldContext FieldContext `json:"fieldContext"` + FieldDataType FieldDataType `json:"fieldDataType"` + Name string `json:"name"` + SelectorMatchType FieldSelectorMatchType `json:"selectorMatchType"` + Limit int `json:"limit"` + MetricContext *MetricContext `json:"metricContext,omitempty"` +} + +type FieldValueSelector struct { + FieldKeySelector + ExistingQuery string `json:"existingQuery"` + Value string `json:"value"` + Limit int `json:"limit"` +} diff --git a/pkg/types/telemetrytypes/field_context.go b/pkg/types/telemetrytypes/field_context.go new file mode 100644 index 0000000000..2d25f788cf --- /dev/null +++ b/pkg/types/telemetrytypes/field_context.go @@ -0,0 +1,148 @@ +package telemetrytypes + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/SigNoz/signoz/pkg/valuer" +) + +// FieldContext is the context of the field being queried. It is expected to be used to disambiguate b/w +// different contexts of the same field. +// +// - Use `resource.` prefix to the key to explicitly indicate and enforce resource context. Example +// - `resource.service.name` +// - `resource.k8s.namespace.name` +// +// - Use `scope.` prefix to explicitly indicate and enforce scope context. Example +// - `scope.name` +// - `scope.version` +// - `scope.my.custom.attribute` and `scope.attribute.my.custom.attribute` resolve to same attribute +// +// - Use `attribute.` to explicitly indicate and enforce attribute context. Example +// - `attribute.http.method` +// - `attribute.http.target` +// +// - Use `event.` to indicate and enforce event context and `event.attribute` to disambiguate b/w `event.name` and `event.attribute.name` . Examples +// - `event.name` will look for event name +// - `event.record_entry` will look for `record_entry` attribute in event +// - `event.attribute.name` will look for `name` attribute event +// +// - Use `span.` to indicate the span context. +// - `span.name` will resolve to the name of span +// - `span.kind` will resolve to the kind of span +// - `span.http.method` will resolve to `http.method` of attribute +// +// - Use `log.` for explicit log context +// - `log.severity_text` will always resolve to `severity_text` of log record +type FieldContext struct { + valuer.String +} + +var ( + FieldContextMetric = FieldContext{valuer.NewString("metric")} + FieldContextLog = FieldContext{valuer.NewString("log")} + FieldContextSpan = FieldContext{valuer.NewString("span")} + FieldContextTrace = FieldContext{valuer.NewString("trace")} + FieldContextResource = FieldContext{valuer.NewString("resource")} + FieldContextScope = FieldContext{valuer.NewString("scope")} + FieldContextAttribute = FieldContext{valuer.NewString("attribute")} + FieldContextEvent = FieldContext{valuer.NewString("event")} + FieldContextUnspecified = FieldContext{valuer.NewString("")} + + // Map string representations to FieldContext values + // We wouldn't need if not for the fact that we have historically used + // "tag" and "attribute" interchangeably. + // This means elsewhere in the system, we have used "tag" to refer to "attribute". + // There are DB entries that use "tag" and "attribute" interchangeably. + // This is a stop gap measure to ensure that we can still use the existing + // DB entries. + fieldContexts = map[string]FieldContext{ + "resource": FieldContextResource, + "scope": FieldContextScope, + "tag": FieldContextAttribute, + "attribute": FieldContextAttribute, + "event": FieldContextEvent, + "spanfield": FieldContextSpan, + "span": FieldContextSpan, + "logfield": FieldContextLog, + "log": FieldContextLog, + "metric": FieldContextMetric, + "tracefield": FieldContextTrace, + } +) + +// UnmarshalJSON implements the json.Unmarshaler interface +func (f *FieldContext) UnmarshalJSON(data []byte) error { + var str string + if err := json.Unmarshal(data, &str); err != nil { + return err + } + + // Normalize the string + normalizedStr := strings.ToLower(strings.TrimSpace(str)) + + // Look up the context in our map + if ctx, exists := fieldContexts[normalizedStr]; exists { + *f = ctx + return nil + } + + // Default to unspecified if not found + *f = FieldContextUnspecified + return nil +} + +// Scan implements the sql.Scanner interface +func (f *FieldContext) Scan(value interface{}) error { + if f == nil { + return fmt.Errorf("fieldcontext: nil receiver") + } + + if value == nil { + *f = FieldContextUnspecified + return nil + } + + str, ok := value.(string) + if !ok { + return fmt.Errorf("fieldcontext: expected string, got %T", value) + } + + // Normalize the string + normalizedStr := strings.ToLower(strings.TrimSpace(str)) + + // Look up the context in our map + if ctx, exists := fieldContexts[normalizedStr]; exists { + *f = ctx + return nil + } + + // Default to unspecified if not found + *f = FieldContextUnspecified + return nil +} + +// TagType returns the tag type for the field context. +func (f FieldContext) TagType() string { + switch f { + case FieldContextResource: + return "resource" + case FieldContextScope: + return "scope" + case FieldContextAttribute: + return "tag" + case FieldContextLog: + return "logfield" + case FieldContextSpan: + return "spanfield" + case FieldContextTrace: + return "tracefield" + case FieldContextMetric: + return "metricfield" + case FieldContextEvent: + return "eventfield" + } + return "" +} diff --git a/pkg/types/telemetrytypes/field_datatype.go b/pkg/types/telemetrytypes/field_datatype.go new file mode 100644 index 0000000000..ec6ef62f72 --- /dev/null +++ b/pkg/types/telemetrytypes/field_datatype.go @@ -0,0 +1,121 @@ +package telemetrytypes + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/SigNoz/signoz/pkg/valuer" +) + +// FieldDataType is the data type of the field. It is expected to be used to disambiguate b/w +// different data types of the same field. +type FieldDataType struct { + valuer.String +} + +var ( + FieldDataTypeString = FieldDataType{valuer.NewString("string")} + FieldDataTypeBool = FieldDataType{valuer.NewString("bool")} + FieldDataTypeFloat64 = FieldDataType{valuer.NewString("float64")} + // int64 and number are synonyms for float64 + FieldDataTypeInt64 = FieldDataType{valuer.NewString("int64")} + FieldDataTypeNumber = FieldDataType{valuer.NewString("number")} + FieldDataTypeUnspecified = FieldDataType{valuer.NewString("")} + + // Map string representations to FieldDataType values + // We want to handle all the possible string representations of the data types. + // Even if the user uses some non-standard representation, we want to be able to + // parse it correctly. + fieldDataTypes = map[string]FieldDataType{ + // String types + "string": FieldDataTypeString, + + // Boolean types + "bool": FieldDataTypeBool, + + // Integer types + "int": FieldDataTypeNumber, + "int8": FieldDataTypeNumber, + "int16": FieldDataTypeNumber, + "int32": FieldDataTypeNumber, + "int64": FieldDataTypeNumber, + "uint": FieldDataTypeNumber, + "uint8": FieldDataTypeNumber, + "uint16": FieldDataTypeNumber, + "uint32": FieldDataTypeNumber, + "uint64": FieldDataTypeNumber, + + // Float types + "float": FieldDataTypeNumber, + "float32": FieldDataTypeNumber, + "float64": FieldDataTypeNumber, + "double": FieldDataTypeNumber, + "decimal": FieldDataTypeNumber, + "number": FieldDataTypeNumber, + } +) + +// UnmarshalJSON implements the json.Unmarshaler interface +func (f *FieldDataType) UnmarshalJSON(data []byte) error { + var str string + if err := json.Unmarshal(data, &str); err != nil { + return err + } + + // Normalize the string + normalizedStr := strings.ToLower(strings.TrimSpace(str)) + + // Look up the data type in our map + if dataType, exists := fieldDataTypes[normalizedStr]; exists { + *f = dataType + return nil + } + + // Default to unspecified if not found + *f = FieldDataTypeUnspecified + return nil +} + +// Scan implements the sql.Scanner interface +func (f *FieldDataType) Scan(value interface{}) error { + if f == nil { + return fmt.Errorf("fielddatatype: nil receiver") + } + + if value == nil { + *f = FieldDataTypeUnspecified + return nil + } + + str, ok := value.(string) + if !ok { + return fmt.Errorf("fielddatatype: expected string, got %T", value) + } + + // Normalize the string + normalizedStr := strings.ToLower(strings.TrimSpace(str)) + + // Look up the data type in our map + if dataType, exists := fieldDataTypes[normalizedStr]; exists { + *f = dataType + return nil + } + + // Default to unspecified if not found + *f = FieldDataTypeUnspecified + return nil +} + +func (f FieldDataType) TagDataType() string { + switch f { + case FieldDataTypeString: + return "string" + case FieldDataTypeBool: + return "bool" + case FieldDataTypeNumber, FieldDataTypeInt64, FieldDataTypeFloat64: + return "float64" + default: + return "string" + } +} diff --git a/pkg/types/telemetrytypes/field_test.go b/pkg/types/telemetrytypes/field_test.go new file mode 100644 index 0000000000..179e6ee093 --- /dev/null +++ b/pkg/types/telemetrytypes/field_test.go @@ -0,0 +1,93 @@ +package telemetrytypes + +import ( + "testing" +) + +func TestGetFieldKeyFromKeyText(t *testing.T) { + + testCases := []struct { + keyText string + expected TelemetryFieldKey + }{ + { + keyText: "resource.service.name:string", + expected: TelemetryFieldKey{ + Name: "service.name", + FieldContext: FieldContextResource, + FieldDataType: FieldDataTypeString, + }, + }, + { + keyText: "scope.name", + expected: TelemetryFieldKey{ + Name: "name", + FieldContext: FieldContextScope, + FieldDataType: FieldDataTypeUnspecified, + }, + }, + { + keyText: "scope.version", + expected: TelemetryFieldKey{ + Name: "version", + FieldContext: FieldContextScope, + FieldDataType: FieldDataTypeUnspecified, + }, + }, + { + keyText: "attribute.http.method", + expected: TelemetryFieldKey{ + Name: "http.method", + FieldContext: FieldContextAttribute, + FieldDataType: FieldDataTypeUnspecified, + }, + }, + { + keyText: "span.name", + expected: TelemetryFieldKey{ + Name: "name", + FieldContext: FieldContextSpan, + FieldDataType: FieldDataTypeUnspecified, + }, + }, + { + keyText: "span.kind:string", + expected: TelemetryFieldKey{ + Name: "kind", + FieldContext: FieldContextSpan, + FieldDataType: FieldDataTypeString, + }, + }, + { + keyText: "span.kind:int", + expected: TelemetryFieldKey{ + Name: "kind", + FieldContext: FieldContextSpan, + FieldDataType: FieldDataTypeNumber, + }, + }, + { + keyText: "span.http.status_code:int", + expected: TelemetryFieldKey{ + Name: "http.status_code", + FieldContext: FieldContextSpan, + FieldDataType: FieldDataTypeNumber, + }, + }, + { + keyText: "log.severity_text", + expected: TelemetryFieldKey{ + Name: "severity_text", + FieldContext: FieldContextLog, + FieldDataType: FieldDataTypeUnspecified, + }, + }, + } + + for _, testCase := range testCases { + result := GetFieldKeyFromKeyText(testCase.keyText) + if result != testCase.expected { + t.Errorf("expected %v, got %v", testCase.expected, result) + } + } +} diff --git a/pkg/types/telemetrytypes/signal.go b/pkg/types/telemetrytypes/signal.go new file mode 100644 index 0000000000..28c90fa6fd --- /dev/null +++ b/pkg/types/telemetrytypes/signal.go @@ -0,0 +1,14 @@ +package telemetrytypes + +import "github.com/SigNoz/signoz/pkg/valuer" + +type Signal struct { + valuer.String +} + +var ( + SignalTraces = Signal{valuer.NewString("traces")} + SignalLogs = Signal{valuer.NewString("logs")} + SignalMetrics = Signal{valuer.NewString("metrics")} + SignalUnspecified = Signal{valuer.NewString("")} +) diff --git a/pkg/types/telemetrytypes/store.go b/pkg/types/telemetrytypes/store.go new file mode 100644 index 0000000000..6aeed6b811 --- /dev/null +++ b/pkg/types/telemetrytypes/store.go @@ -0,0 +1,25 @@ +package telemetrytypes + +import ( + "context" +) + +// MetadataStore is the interface for the telemetry metadata store. +type MetadataStore interface { + // GetKeys returns a map of field keys types.TelemetryFieldKey by name, there can be multiple keys with the same name + // if they have different types or data types. + GetKeys(ctx context.Context, fieldKeySelector *FieldKeySelector) (map[string][]*TelemetryFieldKey, error) + + // GetKeys but with any number of fieldKeySelectors. + GetKeysMulti(ctx context.Context, fieldKeySelectors []*FieldKeySelector) (map[string][]*TelemetryFieldKey, error) + + // GetKey returns a list of keys with the given name. + GetKey(ctx context.Context, fieldKeySelector *FieldKeySelector) ([]*TelemetryFieldKey, error) + + // GetRelatedValues returns a list of related values for the given key name + // and the existing selection of keys. + GetRelatedValues(ctx context.Context, fieldValueSelector *FieldValueSelector) ([]string, error) + + // GetAllValues returns a list of all values. + GetAllValues(ctx context.Context, fieldValueSelector *FieldValueSelector) (*TelemetryFieldValues, error) +} diff --git a/pkg/types/telemetrytypes/telemetrystmt.go b/pkg/types/telemetrytypes/telemetrystmt.go new file mode 100644 index 0000000000..783f454d4c --- /dev/null +++ b/pkg/types/telemetrytypes/telemetrystmt.go @@ -0,0 +1,5 @@ +package telemetrytypes + +type ShowCreateTableStatement struct { + Statement string `json:"statement" ch:"statement"` +}