GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-15 01:55:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: return warning logs too from collector simulator (#3888)

Browse Source 
* chore: return warning logs too from collector simulator

* chore: also return collector logs in preview API response to help with debugging

---------

Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>
This commit is contained in:
Raj Kamal Singh 2023-11-03 17:41:09 +05:30 committed by GitHub
parent 0906886e9a
commit 050b866173
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 20 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/query-service/app/logparsingpipeline/controller.go
View File

@ -134,13 +134,14 @@ type PipelinesPreviewRequest struct {
type PipelinesPreviewResponse struct {
OutputLogs []model.SignozLog `json:"logs"`
CollectorLogs []string `json:"collectorLogs"`
}
func (ic *LogParsingPipelineController) PreviewLogsPipelines(
ctx context.Context,
request *PipelinesPreviewRequest,
) (*PipelinesPreviewResponse, *model.ApiError) {
result, _, err := SimulatePipelinesProcessing(
result, collectorLogs, err := SimulatePipelinesProcessing(
ctx, request.Pipelines, request.Logs,
)
@ -150,6 +151,7 @@ func (ic *LogParsingPipelineController) PreviewLogsPipelines(
return &PipelinesPreviewResponse{
OutputLogs: result,
CollectorLogs: collectorLogs,
}, nil
}

4
pkg/query-service/app/logparsingpipeline/pipelineBuilder_test.go
View File

@ -351,13 +351,13 @@ func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) {
for _, testCase := range testCases {
testPipelines := []Pipeline{makeTestPipeline([]PipelineOperator{testCase.Operator})}
result, collectorErrorLogs, err := SimulatePipelinesProcessing(
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
context.Background(),
testPipelines,
[]model.SignozLog{testCase.NonMatchingLog},
)
require.Nil(err)
require.Equal(0, len(collectorErrorLogs), strings.Join(collectorErrorLogs, "\n"))
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
require.Equal(1, len(result))
}
}

2
pkg/query-service/app/logparsingpipeline/preview.go
View File

@ -22,7 +22,7 @@ func SimulatePipelinesProcessing(
pipelines []Pipeline,
logs []model.SignozLog,
) (
output []model.SignozLog, collectorErrorLogs []string, apiErr *model.ApiError,
output []model.SignozLog, collectorWarnAndErrorLogs []string, apiErr *model.ApiError,
) {
if len(pipelines) < 1 {

16
pkg/query-service/app/logparsingpipeline/preview_test.go
View File

@ -104,7 +104,7 @@ func TestPipelinePreview(t *testing.T) {
},
)
result, collectorErrorLogs, err := SimulatePipelinesProcessing(
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
context.Background(),
testPipelines,
[]model.SignozLog{
@ -114,7 +114,7 @@ func TestPipelinePreview(t *testing.T) {
)
require.Nil(err)
require.Equal(0, len(collectorErrorLogs))
require.Equal(0, len(collectorWarnAndErrorLogs))
require.Equal(2, len(result))
// matching log should have been modified as expected.
@ -190,7 +190,7 @@ func TestGrokParsingPreview(t *testing.T) {
"method": "GET",
},
)
result, collectorErrorLogs, err := SimulatePipelinesProcessing(
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
context.Background(),
testPipelines,
[]model.SignozLog{
@ -199,7 +199,7 @@ func TestGrokParsingPreview(t *testing.T) {
)
require.Nil(err)
require.Equal(0, len(collectorErrorLogs))
require.Equal(0, len(collectorWarnAndErrorLogs))
require.Equal(1, len(result))
processed := result[0]
@ -280,7 +280,7 @@ func TestTraceParsingPreview(t *testing.T) {
TraceFlags: 0,
}
result, collectorErrorLogs, err := SimulatePipelinesProcessing(
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
context.Background(),
testPipelines,
[]model.SignozLog{
@ -289,7 +289,7 @@ func TestTraceParsingPreview(t *testing.T) {
)
require.Nil(err)
require.Equal(1, len(result))
require.Equal(0, len(collectorErrorLogs))
require.Equal(0, len(collectorWarnAndErrorLogs))
processed := result[0]
require.Equal(testTraceId, processed.TraceID)
@ -301,7 +301,7 @@ func TestTraceParsingPreview(t *testing.T) {
// trace parser should work even if parse_from value is empty
testPipelines[0].Config[0].SpanId.ParseFrom = ""
result, collectorErrorLogs, err = SimulatePipelinesProcessing(
result, collectorWarnAndErrorLogs, err = SimulatePipelinesProcessing(
context.Background(),
testPipelines,
[]model.SignozLog{
@ -310,7 +310,7 @@ func TestTraceParsingPreview(t *testing.T) {
)
require.Nil(err)
require.Equal(1, len(result))
require.Equal(0, len(collectorErrorLogs))
require.Equal(0, len(collectorWarnAndErrorLogs))
require.Equal("", result[0].SpanID)
}

8
pkg/query-service/collectorsimulator/collectorsimulator.go
View File

@ -181,14 +181,14 @@ func (l *CollectorSimulator) Shutdown(ctx context.Context) (
simulationErrs = append(simulationErrs, reportedErr.Error())
}
collectorErrorLogs, err := os.ReadFile(l.collectorLogsOutputFilePath)
collectorWarnAndErrorLogs, err := os.ReadFile(l.collectorLogsOutputFilePath)
if err != nil {
return nil, model.InternalError(fmt.Errorf(
"could not read collector logs from tmp file: %w", err,
))
}
if len(collectorErrorLogs) > 0 {
errorLines := strings.Split(string(collectorErrorLogs), "\n")
if len(collectorWarnAndErrorLogs) > 0 {
errorLines := strings.Split(string(collectorWarnAndErrorLogs), "\n")
simulationErrs = append(simulationErrs, errorLines...)
}
@ -219,7 +219,7 @@ func generateSimulationConfig(
metrics:
level: none
logs:
level: error
level: warn
output_paths: ["%s"]
`, receiverId, exporterId, collectorLogsOutputPath)