From 3efd9801a1f94460f468ce850cd63cf4cb6f25df Mon Sep 17 00:00:00 2001 From: Raj Kamal Singh <1133322+raj-k-singh@users.noreply.github.com> Date: Wed, 15 May 2024 14:36:52 +0530 Subject: [PATCH] Chore: restrict logs connection test for integrations to use log attributes for identifying logs (#4977) * chore: change logs connection test spec to be based on an attrib value * chore: disallow unknown fields while unmarshalling JSON for an integration * chore: add description field to collected metric spec * chore: update logs connection test for builtin integrations * chore: update logic for calculating logs connection status --- pkg/query-service/app/http_handler.go | 19 +++++++++-- pkg/query-service/app/integrations/builtin.go | 5 ++- .../clickhouse/integration.json | 14 ++------ .../mongo/integration.json | 14 ++------ .../nginx/integration.json | 14 ++------ .../postgres/integration.json | 14 ++------ .../redis/integration.json | 14 ++------ pkg/query-service/app/integrations/manager.go | 17 ++++++---- .../app/integrations/test_utils.go | 32 ++++--------------- 9 files changed, 48 insertions(+), 95 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index c7adc95416..7f8916fa9f 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -2347,13 +2347,28 @@ func (ah *APIHandler) calculateConnectionStatus( func (ah *APIHandler) calculateLogsConnectionStatus( ctx context.Context, - logsConnectionTest *v3.FilterSet, + logsConnectionTest *integrations.LogsConnectionTest, lookbackSeconds int64, ) (*integrations.SignalConnectionStatus, *model.ApiError) { if logsConnectionTest == nil { return nil, nil } + logsConnTestFilter := &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: logsConnectionTest.AttributeKey, + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: "=", + Value: logsConnectionTest.AttributeValue, + }, + }, + } + qrParams := &v3.QueryRangeParamsV3{ Start: time.Now().UnixMilli() - (lookbackSeconds * 1000), End: time.Now().UnixMilli(), @@ -2363,7 +2378,7 @@ func (ah *APIHandler) calculateLogsConnectionStatus( BuilderQueries: map[string]*v3.BuilderQuery{ "A": { PageSize: 1, - Filters: logsConnectionTest, + Filters: logsConnTestFilter, QueryName: "A", DataSource: v3.DataSourceLogs, Expression: "A", diff --git a/pkg/query-service/app/integrations/builtin.go b/pkg/query-service/app/integrations/builtin.go index cf98b3ff9d..00810cacc1 100644 --- a/pkg/query-service/app/integrations/builtin.go +++ b/pkg/query-service/app/integrations/builtin.go @@ -1,6 +1,7 @@ package integrations import ( + "bytes" "context" "embed" "strings" @@ -120,7 +121,9 @@ func readBuiltInIntegration(dirpath string) ( } var integration IntegrationDetails - err = json.Unmarshal(hydratedSpecJson, &integration) + decoder := json.NewDecoder(bytes.NewReader(hydratedSpecJson)) + decoder.DisallowUnknownFields() + err = decoder.Decode(&integration) if err != nil { return nil, fmt.Errorf( "couldn't parse hydrated JSON spec read from %s: %w", diff --git a/pkg/query-service/app/integrations/builtin_integrations/clickhouse/integration.json b/pkg/query-service/app/integrations/builtin_integrations/clickhouse/integration.json index 99bf16c72d..3135ce402e 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/clickhouse/integration.json +++ b/pkg/query-service/app/integrations/builtin_integrations/clickhouse/integration.json @@ -41,18 +41,8 @@ }, "connection_tests": { "logs": { - "op": "AND", - "items": [ - { - "key": { - "type": "tag", - "key": "source", - "dataType": "string" - }, - "op": "=", - "value": "clickhouse" - } - ] + "attribute_key": "source", + "attribute_value": "clickhouse" } }, "data_collected": "file://data-collected.json" diff --git a/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json b/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json index b9543e0757..d5e24eb4fc 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json +++ b/pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json @@ -37,18 +37,8 @@ }, "connection_tests": { "logs": { - "op": "AND", - "items": [ - { - "key": { - "type": "tag", - "key": "source", - "dataType": "string" - }, - "op": "=", - "value": "mongo" - } - ] + "attribute_key": "source", + "attribute_value": "mongodb" } }, "data_collected": { diff --git a/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json b/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json index 16f03bbed3..7789a5ae90 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json +++ b/pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json @@ -32,18 +32,8 @@ }, "connection_tests": { "logs": { - "op": "AND", - "items": [ - { - "key": { - "type": "tag", - "key": "source", - "dataType": "string" - }, - "op": "=", - "value": "nginx" - } - ] + "attribute_key": "source", + "attribute_value": "nginx" } }, "data_collected": { diff --git a/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json b/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json index 823ba61223..2040a5c946 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json +++ b/pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json @@ -37,18 +37,8 @@ }, "connection_tests": { "logs": { - "op": "AND", - "items": [ - { - "key": { - "type": "tag", - "key": "source", - "dataType": "string" - }, - "op": "=", - "value": "postgres" - } - ] + "attribute_key": "source", + "attribute_value": "postgres" } }, "data_collected": { diff --git a/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json b/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json index e3f5ef2e3c..c0da666181 100644 --- a/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json +++ b/pkg/query-service/app/integrations/builtin_integrations/redis/integration.json @@ -37,18 +37,8 @@ }, "connection_tests": { "logs": { - "op": "AND", - "items": [ - { - "key": { - "type": "tag", - "key": "source", - "dataType": "string" - }, - "op": "=", - "value": "redis" - } - ] + "attribute_key": "source", + "attribute_value": "redis" } }, "data_collected": { diff --git a/pkg/query-service/app/integrations/manager.go b/pkg/query-service/app/integrations/manager.go index c3ebd21cc2..56a57ee026 100644 --- a/pkg/query-service/app/integrations/manager.go +++ b/pkg/query-service/app/integrations/manager.go @@ -12,7 +12,6 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/dashboards" "go.signoz.io/signoz/pkg/query-service/app/logparsingpipeline" "go.signoz.io/signoz/pkg/query-service/model" - v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/utils" ) @@ -60,9 +59,10 @@ type CollectedLogAttribute struct { } type CollectedMetric struct { - Name string `json:"name"` - Type string `json:"type"` - Unit string `json:"unit"` + Name string `json:"name"` + Type string `json:"type"` + Unit string `json:"unit"` + Description string `json:"description"` } type SignalConnectionStatus struct { @@ -75,9 +75,14 @@ type IntegrationConnectionStatus struct { Metrics *SignalConnectionStatus `json:"metrics"` } +// log attribute value to use for finding logs for the integration. +type LogsConnectionTest struct { + AttributeKey string `json:"attribute_key"` + AttributeValue string `json:"attribute_value"` +} + type IntegrationConnectionTests struct { - // Filter to use for finding logs for the integration. - Logs *v3.FilterSet `json:"logs"` + Logs *LogsConnectionTest `json:"logs"` // Metric names expected to have been received for the integration. Metrics []string `json:"metrics"` diff --git a/pkg/query-service/app/integrations/test_utils.go b/pkg/query-service/app/integrations/test_utils.go index 1ff964b3e6..adb667b96e 100644 --- a/pkg/query-service/app/integrations/test_utils.go +++ b/pkg/query-service/app/integrations/test_utils.go @@ -96,19 +96,9 @@ func (t *TestAvailableIntegrationsRepo) list( Alerts: []rules.PostableRule{}, }, ConnectionTests: &IntegrationConnectionTests{ - Logs: &v3.FilterSet{ - Operator: "AND", - Items: []v3.FilterItem{ - { - Key: v3.AttributeKey{ - Key: "source", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - }, - Operator: "=", - Value: "nginx", - }, - }, + Logs: &LogsConnectionTest{ + AttributeKey: "source", + AttributeValue: "nginx", }, }, }, { @@ -174,19 +164,9 @@ func (t *TestAvailableIntegrationsRepo) list( Alerts: []rules.PostableRule{}, }, ConnectionTests: &IntegrationConnectionTests{ - Logs: &v3.FilterSet{ - Operator: "AND", - Items: []v3.FilterItem{ - { - Key: v3.AttributeKey{ - Key: "source", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - }, - Operator: "=", - Value: "nginx", - }, - }, + Logs: &LogsConnectionTest{ + AttributeKey: "source", + AttributeValue: "nginx", }, }, },