mirror of
https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz
synced 2025-07-08 22:11:50 +08:00
546d98ca9c
* chore: add integration test for log pipelines being recommended to agents on connection * chore: agentConf.Manager as AgentConfigProvider * feat: logparsingpipelines as an AgentFeature * chore: some cleanup * feat: update agentConf init in ee query-service * chore: more cleanup and move opamp/logspipeline -> logparsingpipeline/collector_config.go * chore: some more cleanup * chore: set agent.RemoteConfig after broadcasting conf to all agents * chore: add test scenario for post remote conf application
207 lines
9.9 KiB
Go
207 lines
9.9 KiB
Go
package logparsingpipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
"go.signoz.io/signoz/pkg/query-service/constants"
|
|
)
|
|
|
|
var buildProcessorTestData = []struct {
|
|
Name string
|
|
agentConf map[string]interface{}
|
|
pipelineProcessor map[string]interface{}
|
|
outputConf map[string]interface{}
|
|
}{
|
|
{
|
|
Name: "Add",
|
|
agentConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
pipelineProcessor: map[string]interface{}{
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
},
|
|
outputConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "Remove",
|
|
agentConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
pipelineProcessor: map[string]interface{}{},
|
|
outputConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "remove and upsert 1",
|
|
agentConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
constants.LogsPPLPfx + "a": struct{}{},
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
pipelineProcessor: map[string]interface{}{
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
},
|
|
outputConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "remove and upsert 2",
|
|
agentConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
"memorylimiter": struct{}{},
|
|
constants.LogsPPLPfx + "a": struct{}{},
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
pipelineProcessor: map[string]interface{}{
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
},
|
|
outputConf: map[string]interface{}{
|
|
"processors": map[string]interface{}{
|
|
"memorylimiter": struct{}{},
|
|
constants.LogsPPLPfx + "b": struct{}{},
|
|
"batch": struct{}{},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
func TestBuildLogParsingProcessors(t *testing.T) {
|
|
for _, test := range buildProcessorTestData {
|
|
Convey(test.Name, t, func() {
|
|
err := buildLogParsingProcessors(test.agentConf, test.pipelineProcessor)
|
|
So(err, ShouldBeNil)
|
|
So(test.agentConf, ShouldResemble, test.outputConf)
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
var BuildLogsPipelineTestData = []struct {
|
|
Name string
|
|
currentPipeline []string
|
|
logsPipeline []string
|
|
expectedPipeline []string
|
|
}{
|
|
{
|
|
Name: "Add new pipelines",
|
|
currentPipeline: []string{"processor1", "processor2"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "b"},
|
|
expectedPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "b", "processor1", "processor2"},
|
|
},
|
|
{
|
|
Name: "Add new pipeline and respect custom processors",
|
|
currentPipeline: []string{constants.LogsPPLPfx + "a", "processor1", constants.LogsPPLPfx + "b", "processor2"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "b", constants.LogsPPLPfx + "c"},
|
|
expectedPipeline: []string{constants.LogsPPLPfx + "a", "processor1", constants.LogsPPLPfx + "b", constants.LogsPPLPfx + "c", "processor2"},
|
|
},
|
|
{
|
|
Name: "Add new pipeline and respect custom processors",
|
|
currentPipeline: []string{constants.LogsPPLPfx + "a", "processor1", constants.LogsPPLPfx + "b", "processor2"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "b", constants.LogsPPLPfx + "c", constants.LogsPPLPfx + "d"},
|
|
expectedPipeline: []string{constants.LogsPPLPfx + "a", "processor1", constants.LogsPPLPfx + "b", constants.LogsPPLPfx + "c", constants.LogsPPLPfx + "d", "processor2"},
|
|
},
|
|
{
|
|
Name: "Add new pipeline and respect custom processors in the beginning and middle",
|
|
currentPipeline: []string{"processor1", constants.LogsPPLPfx + "a", "processor2", constants.LogsPPLPfx + "b", "batch"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "b", constants.LogsPPLPfx + "c"},
|
|
expectedPipeline: []string{"processor1", constants.LogsPPLPfx + "a", "processor2", constants.LogsPPLPfx + "b", constants.LogsPPLPfx + "c", "batch"},
|
|
},
|
|
{
|
|
Name: "Remove old pipeline add add new",
|
|
currentPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "b", "processor1", "processor2"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a"},
|
|
expectedPipeline: []string{constants.LogsPPLPfx + "a", "processor1", "processor2"},
|
|
},
|
|
{
|
|
Name: "Remove old pipeline from middle",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "a", "processor3", constants.LogsPPLPfx + "b", "batch"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a"},
|
|
expectedPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "a", "processor3", "batch"},
|
|
},
|
|
{
|
|
Name: "Remove old pipeline from middle and add new pipeline",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "a", "processor3", constants.LogsPPLPfx + "b", "batch"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "c"},
|
|
expectedPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "c", "processor3", "batch"},
|
|
},
|
|
{
|
|
Name: "Remove multiple old pipelines from middle and add multiple new ones",
|
|
currentPipeline: []string{"processor1", constants.LogsPPLPfx + "a", "processor2", constants.LogsPPLPfx + "b", "processor3", constants.LogsPPLPfx + "c", "processor4", constants.LogsPPLPfx + "d", "processor5", "batch"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "a1", constants.LogsPPLPfx + "c", constants.LogsPPLPfx + "c1"},
|
|
expectedPipeline: []string{"processor1", constants.LogsPPLPfx + "a", constants.LogsPPLPfx + "a1", "processor2", "processor3", constants.LogsPPLPfx + "c", constants.LogsPPLPfx + "c1", "processor4", "processor5", "batch"},
|
|
},
|
|
|
|
// working
|
|
{
|
|
Name: "rearrange pipelines",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_a", "processor3", constants.LogsPPLPfx + "_b", "batch"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a"},
|
|
expectedPipeline: []string{"processor1", "processor2", "processor3", constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", "batch"},
|
|
},
|
|
{
|
|
Name: "rearrange pipelines with new processor",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_a", "processor3", constants.LogsPPLPfx + "_b", "batch"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_c"},
|
|
expectedPipeline: []string{"processor1", "processor2", "processor3", constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_c", "batch"},
|
|
// expectedPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_b", "processor3", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_c", "batch"},
|
|
},
|
|
{
|
|
Name: "delete processor",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_a", "processor3", constants.LogsPPLPfx + "_b", "batch"},
|
|
logsPipeline: []string{},
|
|
expectedPipeline: []string{"processor1", "processor2", "processor3", "batch"},
|
|
},
|
|
{
|
|
Name: "last to first",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_a", "processor3", "processor4", constants.LogsPPLPfx + "_b", "batch", constants.LogsPPLPfx + "_c"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "_c", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_b"},
|
|
expectedPipeline: []string{"processor1", "processor2", "processor3", "processor4", "batch", constants.LogsPPLPfx + "_c", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_b"},
|
|
},
|
|
{
|
|
Name: "multiple rearrange pipelines",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_a", "processor3", constants.LogsPPLPfx + "_b", "batch", constants.LogsPPLPfx + "_c", "processor4", "processor5", constants.LogsPPLPfx + "_d", "processor6", "processor7"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_d", constants.LogsPPLPfx + "_c", constants.LogsPPLPfx + "_e"},
|
|
expectedPipeline: []string{"processor1", "processor2", "processor3", constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", "batch", "processor4", "processor5", constants.LogsPPLPfx + "_d", constants.LogsPPLPfx + "_c", constants.LogsPPLPfx + "_e", "processor6", "processor7"},
|
|
},
|
|
{
|
|
Name: "multiple rearrange with new pipelines",
|
|
currentPipeline: []string{"processor1", "processor2", constants.LogsPPLPfx + "_a", "processor3", constants.LogsPPLPfx + "_b", "batch", constants.LogsPPLPfx + "_c", "processor4", "processor5", constants.LogsPPLPfx + "_d", "processor6", "processor7"},
|
|
logsPipeline: []string{constants.LogsPPLPfx + "_z", constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", constants.LogsPPLPfx + "_d", constants.LogsPPLPfx + "_c", constants.LogsPPLPfx + "_e"},
|
|
expectedPipeline: []string{constants.LogsPPLPfx + "_z", "processor1", "processor2", "processor3", constants.LogsPPLPfx + "_b", constants.LogsPPLPfx + "_a", "batch", "processor4", "processor5", constants.LogsPPLPfx + "_d", constants.LogsPPLPfx + "_c", constants.LogsPPLPfx + "_e", "processor6", "processor7"},
|
|
},
|
|
}
|
|
|
|
func TestBuildLogsPipeline(t *testing.T) {
|
|
for _, test := range BuildLogsPipelineTestData {
|
|
Convey(test.Name, t, func() {
|
|
v, err := buildLogsProcessors(test.currentPipeline, test.logsPipeline)
|
|
So(err, ShouldBeNil)
|
|
fmt.Println(test.Name, "\n", test.currentPipeline, "\n", v, "\n", test.expectedPipeline)
|
|
So(v, ShouldResemble, test.expectedPipeline)
|
|
})
|
|
}
|
|
}
|