diff --git a/pkg/query-service/app/logparsingpipeline/controller.go b/pkg/query-service/app/logparsingpipeline/controller.go index c4b1b0e3ff..7880ac27b7 100644 --- a/pkg/query-service/app/logparsingpipeline/controller.go +++ b/pkg/query-service/app/logparsingpipeline/controller.go @@ -133,14 +133,15 @@ type PipelinesPreviewRequest struct { } type PipelinesPreviewResponse struct { - OutputLogs []model.SignozLog `json:"logs"` + OutputLogs []model.SignozLog `json:"logs"` + CollectorLogs []string `json:"collectorLogs"` } func (ic *LogParsingPipelineController) PreviewLogsPipelines( ctx context.Context, request *PipelinesPreviewRequest, ) (*PipelinesPreviewResponse, *model.ApiError) { - result, _, err := SimulatePipelinesProcessing( + result, collectorLogs, err := SimulatePipelinesProcessing( ctx, request.Pipelines, request.Logs, ) @@ -149,7 +150,8 @@ func (ic *LogParsingPipelineController) PreviewLogsPipelines( } return &PipelinesPreviewResponse{ - OutputLogs: result, + OutputLogs: result, + CollectorLogs: collectorLogs, }, nil } diff --git a/pkg/query-service/app/logparsingpipeline/pipelineBuilder_test.go b/pkg/query-service/app/logparsingpipeline/pipelineBuilder_test.go index 34afddccce..47a3e70925 100644 --- a/pkg/query-service/app/logparsingpipeline/pipelineBuilder_test.go +++ b/pkg/query-service/app/logparsingpipeline/pipelineBuilder_test.go @@ -351,13 +351,13 @@ func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) { for _, testCase := range testCases { testPipelines := []Pipeline{makeTestPipeline([]PipelineOperator{testCase.Operator})} - result, collectorErrorLogs, err := SimulatePipelinesProcessing( + result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing( context.Background(), testPipelines, []model.SignozLog{testCase.NonMatchingLog}, ) require.Nil(err) - require.Equal(0, len(collectorErrorLogs), strings.Join(collectorErrorLogs, "\n")) + require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n")) require.Equal(1, len(result)) } } diff --git a/pkg/query-service/app/logparsingpipeline/preview.go b/pkg/query-service/app/logparsingpipeline/preview.go index c1863649e6..4f725cafa5 100644 --- a/pkg/query-service/app/logparsingpipeline/preview.go +++ b/pkg/query-service/app/logparsingpipeline/preview.go @@ -22,7 +22,7 @@ func SimulatePipelinesProcessing( pipelines []Pipeline, logs []model.SignozLog, ) ( - output []model.SignozLog, collectorErrorLogs []string, apiErr *model.ApiError, + output []model.SignozLog, collectorWarnAndErrorLogs []string, apiErr *model.ApiError, ) { if len(pipelines) < 1 { diff --git a/pkg/query-service/app/logparsingpipeline/preview_test.go b/pkg/query-service/app/logparsingpipeline/preview_test.go index 9911fd26c2..a7fa51732b 100644 --- a/pkg/query-service/app/logparsingpipeline/preview_test.go +++ b/pkg/query-service/app/logparsingpipeline/preview_test.go @@ -104,7 +104,7 @@ func TestPipelinePreview(t *testing.T) { }, ) - result, collectorErrorLogs, err := SimulatePipelinesProcessing( + result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing( context.Background(), testPipelines, []model.SignozLog{ @@ -114,7 +114,7 @@ func TestPipelinePreview(t *testing.T) { ) require.Nil(err) - require.Equal(0, len(collectorErrorLogs)) + require.Equal(0, len(collectorWarnAndErrorLogs)) require.Equal(2, len(result)) // matching log should have been modified as expected. @@ -190,7 +190,7 @@ func TestGrokParsingPreview(t *testing.T) { "method": "GET", }, ) - result, collectorErrorLogs, err := SimulatePipelinesProcessing( + result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing( context.Background(), testPipelines, []model.SignozLog{ @@ -199,7 +199,7 @@ func TestGrokParsingPreview(t *testing.T) { ) require.Nil(err) - require.Equal(0, len(collectorErrorLogs)) + require.Equal(0, len(collectorWarnAndErrorLogs)) require.Equal(1, len(result)) processed := result[0] @@ -280,7 +280,7 @@ func TestTraceParsingPreview(t *testing.T) { TraceFlags: 0, } - result, collectorErrorLogs, err := SimulatePipelinesProcessing( + result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing( context.Background(), testPipelines, []model.SignozLog{ @@ -289,7 +289,7 @@ func TestTraceParsingPreview(t *testing.T) { ) require.Nil(err) require.Equal(1, len(result)) - require.Equal(0, len(collectorErrorLogs)) + require.Equal(0, len(collectorWarnAndErrorLogs)) processed := result[0] require.Equal(testTraceId, processed.TraceID) @@ -301,7 +301,7 @@ func TestTraceParsingPreview(t *testing.T) { // trace parser should work even if parse_from value is empty testPipelines[0].Config[0].SpanId.ParseFrom = "" - result, collectorErrorLogs, err = SimulatePipelinesProcessing( + result, collectorWarnAndErrorLogs, err = SimulatePipelinesProcessing( context.Background(), testPipelines, []model.SignozLog{ @@ -310,7 +310,7 @@ func TestTraceParsingPreview(t *testing.T) { ) require.Nil(err) require.Equal(1, len(result)) - require.Equal(0, len(collectorErrorLogs)) + require.Equal(0, len(collectorWarnAndErrorLogs)) require.Equal("", result[0].SpanID) } diff --git a/pkg/query-service/collectorsimulator/collectorsimulator.go b/pkg/query-service/collectorsimulator/collectorsimulator.go index 96a2d9fdf7..4a8236b483 100644 --- a/pkg/query-service/collectorsimulator/collectorsimulator.go +++ b/pkg/query-service/collectorsimulator/collectorsimulator.go @@ -181,14 +181,14 @@ func (l *CollectorSimulator) Shutdown(ctx context.Context) ( simulationErrs = append(simulationErrs, reportedErr.Error()) } - collectorErrorLogs, err := os.ReadFile(l.collectorLogsOutputFilePath) + collectorWarnAndErrorLogs, err := os.ReadFile(l.collectorLogsOutputFilePath) if err != nil { return nil, model.InternalError(fmt.Errorf( "could not read collector logs from tmp file: %w", err, )) } - if len(collectorErrorLogs) > 0 { - errorLines := strings.Split(string(collectorErrorLogs), "\n") + if len(collectorWarnAndErrorLogs) > 0 { + errorLines := strings.Split(string(collectorWarnAndErrorLogs), "\n") simulationErrs = append(simulationErrs, errorLines...) } @@ -219,7 +219,7 @@ func generateSimulationConfig( metrics: level: none logs: - level: error + level: warn output_paths: ["%s"] `, receiverId, exporterId, collectorLogsOutputPath)