chore: add API endpoints for fields keys and values (#7560)

ee/query-service/app/api/api.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/ee/query-service/license"
 	"github.com/SigNoz/signoz/ee/query-service/usage"
 	"github.com/SigNoz/signoz/pkg/alertmanager"
+	"github.com/SigNoz/signoz/pkg/apis/fields"
 	"github.com/SigNoz/signoz/pkg/modules/preference"
 	preferencecore "github.com/SigNoz/signoz/pkg/modules/preference/core"
 	baseapp "github.com/SigNoz/signoz/pkg/query-service/app"
@@ -74,6 +75,7 @@ func NewAPIHandler(opts APIHandlerOptions, signoz *signoz.SigNoz) (*APIHandler,
 		UseLogsNewSchema:              opts.UseLogsNewSchema,
 		UseTraceNewSchema:             opts.UseTraceNewSchema,
 		AlertmanagerAPI:               alertmanager.NewAPI(signoz.Alertmanager),
+		FieldsAPI:                     fields.NewAPI(signoz.TelemetryStore),
 		Signoz:                        signoz,
 		Preference:                    preference,
 	})

ee/query-service/app/server.go

@@ -367,6 +367,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler, web web.Web) (*h
 	apiHandler.RegisterLogsRoutes(r, am)
 	apiHandler.RegisterIntegrationRoutes(r, am)
 	apiHandler.RegisterCloudIntegrationsRoutes(r, am)
+	apiHandler.RegisterFieldsRoutes(r, am)
 	apiHandler.RegisterQueryRangeV3Routes(r, am)
 	apiHandler.RegisterInfraMetricsRoutes(r, am)
 	apiHandler.RegisterQueryRangeV4Routes(r, am)

pkg/apis/fields/api.go

@@ -0,0 +1,122 @@
+package fields
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+
+	"github.com/SigNoz/signoz/pkg/http/render"
+	"github.com/SigNoz/signoz/pkg/telemetrylogs"
+	"github.com/SigNoz/signoz/pkg/telemetrymetadata"
+	"github.com/SigNoz/signoz/pkg/telemetrymetrics"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/telemetrytraces"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"go.uber.org/zap"
+)
+
+type API struct {
+	telemetryStore         telemetrystore.TelemetryStore
+	telemetryMetadataStore telemetrytypes.MetadataStore
+}
+
+func NewAPI(telemetryStore telemetrystore.TelemetryStore) *API {
+
+	telemetryMetadataStore := telemetrymetadata.NewTelemetryMetaStore(
+		telemetryStore,
+		telemetrytraces.DBName,
+		telemetrytraces.TagAttributesV2TableName,
+		telemetrytraces.SpanIndexV3TableName,
+		telemetrymetrics.DBName,
+		telemetrymetrics.TimeseriesV41weekTableName,
+		telemetrymetrics.TimeseriesV41weekLocalTableName,
+		telemetrylogs.DBName,
+		telemetrylogs.LogsV2TableName,
+		telemetrylogs.TagAttributesV2TableName,
+		telemetrymetadata.DBName,
+		telemetrymetadata.AttributesMetadataLocalTableName,
+	)
+
+	return &API{
+		telemetryStore:         telemetryStore,
+		telemetryMetadataStore: telemetryMetadataStore,
+	}
+}
+
+func (api *API) GetFieldsKeys(w http.ResponseWriter, r *http.Request) {
+
+	type fieldKeysResponse struct {
+		Keys     map[string][]*telemetrytypes.TelemetryFieldKey `json:"keys"`
+		Complete bool                                           `json:"complete"`
+	}
+
+	bodyBytes, _ := io.ReadAll(r.Body)
+	r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
+	ctx := r.Context()
+
+	fieldKeySelector, err := parseFieldKeyRequest(r)
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	keys, err := api.telemetryMetadataStore.GetKeys(ctx, fieldKeySelector)
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	response := fieldKeysResponse{
+		Keys:     keys,
+		Complete: len(keys) < fieldKeySelector.Limit,
+	}
+
+	render.Success(w, http.StatusOK, response)
+}
+
+func (api *API) GetFieldsValues(w http.ResponseWriter, r *http.Request) {
+
+	type fieldValuesResponse struct {
+		Values   *telemetrytypes.TelemetryFieldValues `json:"values"`
+		Complete bool                                 `json:"complete"`
+	}
+
+	bodyBytes, _ := io.ReadAll(r.Body)
+	r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
+	ctx := r.Context()
+
+	fieldValueSelector, err := parseFieldValueRequest(r)
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	allValues, err := api.telemetryMetadataStore.GetAllValues(ctx, fieldValueSelector)
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	relatedValues, err := api.telemetryMetadataStore.GetRelatedValues(ctx, fieldValueSelector)
+	if err != nil {
+		// we don't want to return error if we fail to get related values for some reason
+		zap.L().Error("failed to get related values", zap.Error(err))
+		relatedValues = []string{}
+	}
+
+	values := &telemetrytypes.TelemetryFieldValues{
+		StringValues:  allValues.StringValues,
+		NumberValues:  allValues.NumberValues,
+		RelatedValues: relatedValues,
+	}
+
+	response := fieldValuesResponse{
+		Values: values,
+		Complete: len(values.StringValues) < fieldValueSelector.Limit &&
+			len(values.BoolValues) < fieldValueSelector.Limit &&
+			len(values.NumberValues) < fieldValueSelector.Limit &&
+			len(values.RelatedValues) < fieldValueSelector.Limit,
+	}
+
+	render.Success(w, http.StatusOK, response)
+}

pkg/apis/fields/parse.go

@@ -0,0 +1,112 @@
+package fields
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+func parseFieldKeyRequest(r *http.Request) (*telemetrytypes.FieldKeySelector, error) {
+	var req telemetrytypes.FieldKeySelector
+	var signal telemetrytypes.Signal
+	var err error
+
+	signalStr := r.URL.Query().Get("signal")
+	if signalStr != "" {
+		signal = telemetrytypes.Signal{String: valuer.NewString(signalStr)}
+	} else {
+		signal = telemetrytypes.SignalUnspecified
+	}
+
+	if r.URL.Query().Get("limit") != "" {
+		limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
+		if err != nil {
+			return nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "failed to parse limit")
+		}
+		req.Limit = limit
+	} else {
+		req.Limit = 1000
+	}
+
+	var startUnixMilli, endUnixMilli int64
+
+	if r.URL.Query().Get("startUnixMilli") != "" {
+		startUnixMilli, err := strconv.ParseInt(r.URL.Query().Get("startUnixMilli"), 10, 64)
+		if err != nil {
+			return nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "failed to parse startUnixMilli")
+		}
+		// Round down to the nearest 6 hours (21600000 milliseconds)
+		startUnixMilli -= startUnixMilli % 21600000
+	}
+	if r.URL.Query().Get("endUnixMilli") != "" {
+		endUnixMilli, err = strconv.ParseInt(r.URL.Query().Get("endUnixMilli"), 10, 64)
+		if err != nil {
+			return nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "failed to parse endUnixMilli")
+		}
+	}
+
+	// Parse fieldContext directly instead of using JSON unmarshalling.
+	var fieldContext telemetrytypes.FieldContext
+	fieldContextStr := r.URL.Query().Get("fieldContext")
+	if fieldContextStr != "" {
+		fieldContext = telemetrytypes.FieldContext{String: valuer.NewString(fieldContextStr)}
+	}
+
+	// Parse fieldDataType directly instead of using JSON unmarshalling.
+	var fieldDataType telemetrytypes.FieldDataType
+	fieldDataTypeStr := r.URL.Query().Get("fieldDataType")
+	if fieldDataTypeStr != "" {
+		fieldDataType = telemetrytypes.FieldDataType{String: valuer.NewString(fieldDataTypeStr)}
+	}
+
+	metricName := r.URL.Query().Get("metricName")
+	var metricContext *telemetrytypes.MetricContext
+	if metricName != "" {
+		metricContext = &telemetrytypes.MetricContext{
+			MetricName: metricName,
+		}
+	}
+
+	name := r.URL.Query().Get("name")
+
+	req = telemetrytypes.FieldKeySelector{
+		StartUnixMilli:    startUnixMilli,
+		EndUnixMilli:      endUnixMilli,
+		Signal:            signal,
+		Name:              name,
+		FieldContext:      fieldContext,
+		FieldDataType:     fieldDataType,
+		Limit:             req.Limit,
+		SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+		MetricContext:     metricContext,
+	}
+	return &req, nil
+}
+
+func parseFieldValueRequest(r *http.Request) (*telemetrytypes.FieldValueSelector, error) {
+	keySelector, err := parseFieldKeyRequest(r)
+	if err != nil {
+		return nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "failed to parse field key request")
+	}
+
+	existingQuery := r.URL.Query().Get("existingQuery")
+	value := r.URL.Query().Get("value")
+
+	// Parse limit for fieldValue request, fallback to default 50 if parsing fails.
+	limit, err := strconv.Atoi(r.URL.Query().Get("limit"))
+	if err != nil {
+		limit = 50
+	}
+
+	req := telemetrytypes.FieldValueSelector{
+		FieldKeySelector: keySelector,
+		ExistingQuery:    existingQuery,
+		Value:            value,
+		Limit:            limit,
+	}
+
+	return &req, nil
+}

pkg/query-service/app/http_handler.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager"
+	"github.com/SigNoz/signoz/pkg/apis/fields"
 	errorsV2 "github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/http/render"
 	"github.com/SigNoz/signoz/pkg/modules/preference"
@@ -142,6 +143,8 @@ type APIHandler struct {
 
 	AlertmanagerAPI *alertmanager.API
 
+	FieldsAPI *fields.API
+
 	Signoz *signoz.SigNoz
 
 	Preference preference.API
@@ -189,6 +192,8 @@ type APIHandlerOpts struct {
 
 	AlertmanagerAPI *alertmanager.API
 
+	FieldsAPI *fields.API
+
 	Signoz *signoz.SigNoz
 
 	Preference preference.API
@@ -261,6 +266,7 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) {
 		AlertmanagerAPI:               opts.AlertmanagerAPI,
 		Signoz:                        opts.Signoz,
 		Preference:                    opts.Preference,
+		FieldsAPI:                     opts.FieldsAPI,
 	}
 
 	logsQueryBuilder := logsv3.PrepareLogsQuery
@@ -417,6 +423,13 @@ func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMid
 	subRouter.HandleFunc("/logs/livetail", am.ViewAccess(aH.liveTailLogs)).Methods(http.MethodGet)
 }
 
+func (aH *APIHandler) RegisterFieldsRoutes(router *mux.Router, am *AuthMiddleware) {
+	subRouter := router.PathPrefix("/api/v1").Subrouter()
+
+	subRouter.HandleFunc("/fields/keys", am.ViewAccess(aH.FieldsAPI.GetFieldsKeys)).Methods(http.MethodGet)
+	subRouter.HandleFunc("/fields/values", am.ViewAccess(aH.FieldsAPI.GetFieldsValues)).Methods(http.MethodGet)
+}
+
 func (aH *APIHandler) RegisterInfraMetricsRoutes(router *mux.Router, am *AuthMiddleware) {
 	hostsSubRouter := router.PathPrefix("/api/v1/hosts").Subrouter()
 	hostsSubRouter.HandleFunc("/attribute_keys", am.ViewAccess(aH.getHostAttributeKeys)).Methods(http.MethodGet)

pkg/query-service/app/server.go

@@ -13,6 +13,7 @@ import (
 	"github.com/jmoiron/sqlx"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager"
+	"github.com/SigNoz/signoz/pkg/apis/fields"
 	"github.com/SigNoz/signoz/pkg/http/middleware"
 	"github.com/SigNoz/signoz/pkg/modules/preference"
 	preferencecore "github.com/SigNoz/signoz/pkg/modules/preference/core"
@@ -201,6 +202,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		UseTraceNewSchema:             serverOptions.UseTraceNewSchema,
 		JWT:                           serverOptions.Jwt,
 		AlertmanagerAPI:               alertmanager.NewAPI(serverOptions.SigNoz.Alertmanager),
+		FieldsAPI:                     fields.NewAPI(serverOptions.SigNoz.TelemetryStore),
 		Signoz:                        serverOptions.SigNoz,
 		Preference:                    preferenceModule,
 	})
@@ -321,6 +323,7 @@ func (s *Server) createPublicServer(api *APIHandler, web web.Web) (*http.Server,
 	api.RegisterLogsRoutes(r, am)
 	api.RegisterIntegrationRoutes(r, am)
 	api.RegisterCloudIntegrationsRoutes(r, am)
+	api.RegisterFieldsRoutes(r, am)
 	api.RegisterQueryRangeV3Routes(r, am)
 	api.RegisterInfraMetricsRoutes(r, am)
 	api.RegisterWebSocketPaths(r, am)

pkg/telemetrymetadata/metadata.go

@@ -51,7 +51,7 @@ func NewTelemetryMetaStore(
 	logsFieldsTblName string,
 	relatedMetadataDBName string,
 	relatedMetadataTblName string,
-) (telemetrytypes.MetadataStore, error) {
+) telemetrytypes.MetadataStore {
 	return &telemetryMetaStore{
 		telemetrystore:         telemetrystore,
 		tracesDBName:           tracesDBName,
@@ -67,7 +67,7 @@ func NewTelemetryMetaStore(
 		relatedMetadataTblName: relatedMetadataTblName,
 
 		conditionBuilder: NewConditionBuilder(),
-	}, nil
+	}
 }
 
 // tracesTblStatementToFieldKeys returns materialised attribute/resource/scope keys from the traces table

pkg/telemetrymetadata/metadata_test.go

@@ -33,7 +33,7 @@ func TestGetKeys(t *testing.T) {
 	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
 	mock := mockTelemetryStore.Mock()
 
-	metadata, err := NewTelemetryMetaStore(
+	metadata := NewTelemetryMetaStore(
 		mockTelemetryStore,
 		telemetrytraces.DBName,
 		telemetrytraces.TagAttributesV2TableName,
@@ -48,10 +48,6 @@ func TestGetKeys(t *testing.T) {
 		AttributesMetadataLocalTableName,
 	)
 
-	if err != nil {
-		t.Fatalf("Failed to create telemetry metadata store: %v", err)
-	}
-
 	rows := cmock.NewRows([]cmock.ColumnType{
 		{Name: "statement", Type: "String"},
 	}, [][]any{{"CREATE TABLE signoz_traces.signoz_index_v3"}})

pkg/types/telemetrytypes/field.go

@@ -130,7 +130,7 @@ type FieldKeySelector struct {
 }
 
 type FieldValueSelector struct {
-	FieldKeySelector
+	*FieldKeySelector
 	ExistingQuery string `json:"existingQuery"`
 	Value         string `json:"value"`
 	Limit         int    `json:"limit"`