Skip to content

feat: option for ignoring missed fields in plugin configs #18648

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mialinx wants to merge 1 commit into from
Closed

feat: option for ignoring missed fields in plugin configs #18648

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 67 additions & 33 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,11 @@ type AgentConfig struct {
// metrics buffered in the last `flush_interval` in the event of a power
// cut.
BufferDiskSync *bool `toml:"buffer_disk_sync"`

// AllowPluginMissingFields, when true, logs a warning for configuration
// keys that are not used by the target table (agent, tags, or plugin) and
// continues loading instead of returning an error.
AllowPluginMissingFields bool `toml:"allow_plugin_missing_fields"`
}

// InputNames returns a list of strings of the configured inputs.
Expand Down Expand Up @@ -680,11 +685,13 @@ func (c *Config) LoadConfigData(data []byte, path string) error {
}
}

if len(c.UnusedFields) > 0 {
return fmt.Errorf(
"line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
tbl.Line, keys(c.UnusedFields))
err = c.unusedFieldsErrorOrWarn(
"line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
tbl.Line,
)
if err != nil {
return err
}

// Initialize the file-sorting slices
Expand Down Expand Up @@ -718,11 +725,13 @@ func (c *Config) LoadConfigData(data []byte, path string) error {
return fmt.Errorf("unsupported config format: %s",
pluginName)
}
if len(c.UnusedFields) > 0 {
return fmt.Errorf(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line, keys(c.UnusedFields))
err = c.unusedFieldsErrorOrWarn(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line,
)
if err != nil {
return err
}
}
case "inputs", "plugins":
Expand All @@ -743,11 +752,13 @@ func (c *Config) LoadConfigData(data []byte, path string) error {
return fmt.Errorf("unsupported config format: %s",
pluginName)
}
if len(c.UnusedFields) > 0 {
return fmt.Errorf(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line, keys(c.UnusedFields))
err = c.unusedFieldsErrorOrWarn(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line,
)
if err != nil {
return err
}
}
case "processors":
Expand All @@ -763,15 +774,13 @@ func (c *Config) LoadConfigData(data []byte, path string) error {
return fmt.Errorf("unsupported config format: %s",
pluginName)
}
if len(c.UnusedFields) > 0 {
return fmt.Errorf(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name,
pluginName,
subTable.Line,
keys(c.UnusedFields),
)
err = c.unusedFieldsErrorOrWarn(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line,
)
if err != nil {
return err
}
}
case "aggregators":
Expand All @@ -787,11 +796,13 @@ func (c *Config) LoadConfigData(data []byte, path string) error {
return fmt.Errorf("unsupported config format: %s",
pluginName)
}
if len(c.UnusedFields) > 0 {
return fmt.Errorf(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line, keys(c.UnusedFields))
err = c.unusedFieldsErrorOrWarn(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line,
)
if err != nil {
return err
}
}
case "secretstores":
Expand All @@ -806,10 +817,13 @@ func (c *Config) LoadConfigData(data []byte, path string) error {
default:
return fmt.Errorf("unsupported config format: %s", pluginName)
}
if len(c.UnusedFields) > 0 {
msg := "plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; " +
"this is either a typo or this config option does not exist in this version"
return fmt.Errorf(msg, name, pluginName, subTable.Line, keys(c.UnusedFields))
err = c.unusedFieldsErrorOrWarn(
"plugin %s.%s: line %d: configuration specified the fields %q, but they were not used; "+
"this is either a typo or this config option does not exist in this version",
name, pluginName, subTable.Line,
)
if err != nil {
return err
}
}

Expand Down Expand Up @@ -2026,6 +2040,26 @@ func (c *Config) matchesLabelSelection(tbl *ast.Table) (bool, error) {
return pluginLabelSelector.matches(labels), nil
}

// unusedFieldsErrorOrWarn fails load with a formatted error when UnusedFields
// is non-empty and AllowPluginMissingFields is false. When AllowPluginMissingFields
// is true, it logs a warning and returns nil.
func (c *Config) unusedFieldsErrorOrWarn(format string, args ...any) error {
c.unusedFieldsMutex.Lock()
if len(c.UnusedFields) == 0 {
c.unusedFieldsMutex.Unlock()
return nil
}
fieldKeys := keys(c.UnusedFields)
c.unusedFieldsMutex.Unlock()
allow := c.Agent != nil && c.Agent.AllowPluginMissingFields
fmtArgs := append(append([]any(nil), args...), fieldKeys)
if allow {
log.Printf("W! "+format, fmtArgs...)
return nil
}
return fmt.Errorf(format, fmtArgs...)
}

func keys(m map[string]bool) []string {
result := make([]string, 0, len(m))
for k := range m {
Expand Down
71 changes: 71 additions & 0 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,77 @@ func TestConfig_FieldNotDefined(t *testing.T) {
}
}

func TestConfig_FieldNotDefined_AllowPluginMissingFields(t *testing.T) {
tests := []struct {
name string
filename string
minInputs int
minProcessors int
}{
{
name: "in input plugin without parser",
filename: "./testdata/invalid_field.toml",
minInputs: 1,
},
{
name: "in input plugin with parser",
filename: "./testdata/invalid_field_with_parser.toml",
minInputs: 1,
},
{
name: "in input plugin with parser func",
filename: "./testdata/invalid_field_with_parserfunc.toml",
minInputs: 1,
},
{
name: "in parser of input plugin",
filename: "./testdata/invalid_field_in_parser_table.toml",
minInputs: 1,
},
{
name: "in parser of input plugin with parser-func",
filename: "./testdata/invalid_field_in_parserfunc_table.toml",
minInputs: 1,
},
{
name: "in processor plugin without parser",
filename: "./testdata/invalid_field_processor.toml",
minProcessors: 1,
},
{
name: "in processor plugin with parser",
filename: "./testdata/invalid_field_processor_with_parser.toml",
minProcessors: 1,
},
{
name: "in processor plugin with parser func",
filename: "./testdata/invalid_field_processor_with_parserfunc.toml",
minProcessors: 1,
},
{
name: "in parser of processor plugin",
filename: "./testdata/invalid_field_processor_in_parser_table.toml",
minProcessors: 1,
},
{
name: "in parser of processor plugin with parser-func",
filename: "./testdata/invalid_field_processor_in_parserfunc_table.toml",
minInputs: 1,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := config.NewConfig()
c.Agent.AllowPluginMissingFields = true
err := c.LoadConfig(tt.filename)
require.NoError(t, err)
require.GreaterOrEqual(t, len(c.Inputs), tt.minInputs)
require.GreaterOrEqual(t, len(c.Processors), tt.minProcessors)
})
}
}

func TestConfig_WrongFieldType(t *testing.T) {
c := config.NewConfig()
err := c.LoadConfig("./testdata/wrong_field_type.toml")
Expand Down
Loading