chore: return warning logs too from collector simulator (#3888)

* chore: return warning logs too from collector simulator

* chore: also return collector logs in preview API response to help with debugging

---------

Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>

pkg/query-service/app/logparsingpipeline/controller.go

@@ -133,14 +133,15 @@ type PipelinesPreviewRequest struct {
 }
 
 type PipelinesPreviewResponse struct {
-	OutputLogs []model.SignozLog `json:"logs"`
+	OutputLogs    []model.SignozLog `json:"logs"`
+	CollectorLogs []string          `json:"collectorLogs"`
 }
 
 func (ic *LogParsingPipelineController) PreviewLogsPipelines(
 	ctx context.Context,
 	request *PipelinesPreviewRequest,
 ) (*PipelinesPreviewResponse, *model.ApiError) {
-	result, _, err := SimulatePipelinesProcessing(
+	result, collectorLogs, err := SimulatePipelinesProcessing(
 		ctx, request.Pipelines, request.Logs,
 	)
 
@@ -149,7 +150,8 @@ func (ic *LogParsingPipelineController) PreviewLogsPipelines(
 	}
 
 	return &PipelinesPreviewResponse{
-		OutputLogs: result,
+		OutputLogs:    result,
+		CollectorLogs: collectorLogs,
 	}, nil
 }
 

pkg/query-service/app/logparsingpipeline/pipelineBuilder_test.go

@@ -351,13 +351,13 @@ func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) {
 	for _, testCase := range testCases {
 		testPipelines := []Pipeline{makeTestPipeline([]PipelineOperator{testCase.Operator})}
 
-		result, collectorErrorLogs, err := SimulatePipelinesProcessing(
+		result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
 			context.Background(),
 			testPipelines,
 			[]model.SignozLog{testCase.NonMatchingLog},
 		)
 		require.Nil(err)
-		require.Equal(0, len(collectorErrorLogs), strings.Join(collectorErrorLogs, "\n"))
+		require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
 		require.Equal(1, len(result))
 	}
 }

pkg/query-service/app/logparsingpipeline/preview.go

@@ -22,7 +22,7 @@ func SimulatePipelinesProcessing(
 	pipelines []Pipeline,
 	logs []model.SignozLog,
 ) (
-	output []model.SignozLog, collectorErrorLogs []string, apiErr *model.ApiError,
+	output []model.SignozLog, collectorWarnAndErrorLogs []string, apiErr *model.ApiError,
 ) {
 
 	if len(pipelines) < 1 {

pkg/query-service/app/logparsingpipeline/preview_test.go

@@ -104,7 +104,7 @@ func TestPipelinePreview(t *testing.T) {
 		},
 	)
 
-	result, collectorErrorLogs, err := SimulatePipelinesProcessing(
+	result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
 		context.Background(),
 		testPipelines,
 		[]model.SignozLog{
@@ -114,7 +114,7 @@ func TestPipelinePreview(t *testing.T) {
 	)
 
 	require.Nil(err)
-	require.Equal(0, len(collectorErrorLogs))
+	require.Equal(0, len(collectorWarnAndErrorLogs))
 	require.Equal(2, len(result))
 
 	// matching log should have been modified as expected.
@@ -190,7 +190,7 @@ func TestGrokParsingPreview(t *testing.T) {
 			"method": "GET",
 		},
 	)
-	result, collectorErrorLogs, err := SimulatePipelinesProcessing(
+	result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
 		context.Background(),
 		testPipelines,
 		[]model.SignozLog{
@@ -199,7 +199,7 @@ func TestGrokParsingPreview(t *testing.T) {
 	)
 
 	require.Nil(err)
-	require.Equal(0, len(collectorErrorLogs))
+	require.Equal(0, len(collectorWarnAndErrorLogs))
 	require.Equal(1, len(result))
 	processed := result[0]
 
@@ -280,7 +280,7 @@ func TestTraceParsingPreview(t *testing.T) {
 		TraceFlags: 0,
 	}
 
-	result, collectorErrorLogs, err := SimulatePipelinesProcessing(
+	result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
 		context.Background(),
 		testPipelines,
 		[]model.SignozLog{
@@ -289,7 +289,7 @@ func TestTraceParsingPreview(t *testing.T) {
 	)
 	require.Nil(err)
 	require.Equal(1, len(result))
-	require.Equal(0, len(collectorErrorLogs))
+	require.Equal(0, len(collectorWarnAndErrorLogs))
 	processed := result[0]
 
 	require.Equal(testTraceId, processed.TraceID)
@@ -301,7 +301,7 @@ func TestTraceParsingPreview(t *testing.T) {
 
 	// trace parser should work even if parse_from value is empty
 	testPipelines[0].Config[0].SpanId.ParseFrom = ""
-	result, collectorErrorLogs, err = SimulatePipelinesProcessing(
+	result, collectorWarnAndErrorLogs, err = SimulatePipelinesProcessing(
 		context.Background(),
 		testPipelines,
 		[]model.SignozLog{
@@ -310,7 +310,7 @@ func TestTraceParsingPreview(t *testing.T) {
 	)
 	require.Nil(err)
 	require.Equal(1, len(result))
-	require.Equal(0, len(collectorErrorLogs))
+	require.Equal(0, len(collectorWarnAndErrorLogs))
 	require.Equal("", result[0].SpanID)
 }
 

pkg/query-service/collectorsimulator/collectorsimulator.go

@@ -181,14 +181,14 @@ func (l *CollectorSimulator) Shutdown(ctx context.Context) (
 		simulationErrs = append(simulationErrs, reportedErr.Error())
 	}
 
-	collectorErrorLogs, err := os.ReadFile(l.collectorLogsOutputFilePath)
+	collectorWarnAndErrorLogs, err := os.ReadFile(l.collectorLogsOutputFilePath)
 	if err != nil {
 		return nil, model.InternalError(fmt.Errorf(
 			"could not read collector logs from tmp file: %w", err,
 		))
 	}
-	if len(collectorErrorLogs) > 0 {
+	if len(collectorWarnAndErrorLogs) > 0 {
-		errorLines := strings.Split(string(collectorErrorLogs), "\n")
+		errorLines := strings.Split(string(collectorWarnAndErrorLogs), "\n")
 		simulationErrs = append(simulationErrs, errorLines...)
 	}
 
@@ -219,7 +219,7 @@ func generateSimulationConfig(
         metrics:
           level: none
         logs:
-          level: error
+          level: warn
           output_paths: ["%s"]
     `, receiverId, exporterId, collectorLogsOutputPath)