diff --git a/cmd/root.go b/cmd/root.go index 449be02518..d2f72bfa0b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,6 +212,7 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md new file mode 100644 index 0000000000..a58fa7f30d --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -0,0 +1,23 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. + +## Usage + +```bash +yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--security-enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md new file mode 100644 index 0000000000..65501b2516 --- /dev/null +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -0,0 +1,72 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. + +## Usage + +```bash +yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--security-enable-system-operator` flag to enable it. + +## system operator returns null when disabled +Use `--security-enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: null +``` + +## Run a command with an argument +Use `--security-enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: test +``` + +## Run a command without arguments +Omit the semicolon and args to run the command with no extra arguments. + +Given a sample.yml file of: +```yaml +a: hello +``` +then +```bash +yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml +``` +will output +```yaml +a: "" +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 866b736cb4..938b8a717a 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{ simpleOp("load_?str|str_?load", loadStringOpType), {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, + simpleOp("system", systemOpType), + {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, simpleOp("select", selectOpType), diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index f36054eec9..cbea1d7b9c 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} +var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator} + var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go new file mode 100644 index 0000000000..5f22508df5 --- /dev/null +++ b/pkg/yqlib/operator_system.go @@ -0,0 +1,148 @@ +package yqlib + +import ( + "bytes" + "container/list" + "fmt" + "os/exec" + "strings" +) + +func resolveSystemArgs(argsNode *CandidateNode) []string { + if argsNode == nil { + return nil + } + + if argsNode.Kind == SequenceNode { + args := make([]string, 0, len(argsNode.Content)) + for _, child := range argsNode.Content { + // Only non-null scalar children are valid arguments. + if child == nil { + continue + } + if child.Kind != ScalarNode || child.Tag == "!!null" { + log.Warningf("system operator: argument must be a non-null scalar; got kind=%v tag=%v - ignoring", child.Kind, child.Tag) + continue + } + args = append(args, child.Value) + } + if len(args) == 0 { + return nil + } + return args + } + + // Single-argument case: only accept a non-null scalar node. + if argsNode.Tag == "!!null" { + return nil + } + if argsNode.Kind != ScalarNode { + log.Warningf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v - ignoring", argsNode.Kind, argsNode.Tag) + return nil + } + return []string{argsNode.Value} +} + +func resolveCommandNode(commandNodes Context) (string, error) { + if commandNodes.MatchingNodes.Front() == nil { + return "", fmt.Errorf("system operator: command expression returned no results") + } + if commandNodes.MatchingNodes.Len() > 1 { + log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len()) + } + cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode) + if cmdNode.Kind != ScalarNode || cmdNode.Tag == "!!null" { + return "", fmt.Errorf("system operator: command must be a string scalar") + } + if cmdNode.Value == "" { + return "", fmt.Errorf("system operator: command must be a non-empty string") + } + return cmdNode.Value, nil +} + +func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + if !ConfiguredSecurityPreferences.EnableSystemOps { + log.Warning("system operator is disabled, use --security-enable-system-operator flag to enable") + results := list.New() + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + results.PushBack(candidate.CreateReplacement(ScalarNode, "!!null", "null")) + } + return context.ChildContext(results), nil + } + + // determine at parse time whether we have (command; args) or just (command) + hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + nodeContext := context.SingleReadonlyChildContext(candidate) + + var command string + var args []string + + if hasArgs { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS) + if err != nil { + return Context{}, err + } + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err + } + + argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Front() != nil { + args = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode)) + } + } else { + commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS) + if err != nil { + return Context{}, err + } + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err + } + } + + var stdin bytes.Buffer + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err + } + stdin.WriteString(encoded) + + // #nosec G204 - intentional: user must explicitly enable this operator + cmd := exec.Command(command, args...) + cmd.Stdin = &stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + + output, err := cmd.Output() + if err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if stderrStr != "" { + return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr) + } + return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) + } + + result := string(output) + if strings.HasSuffix(result, "\r\n") { + result = result[:len(result)-2] + } else if strings.HasSuffix(result, "\n") { + result = result[:len(result)-1] + } + newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) + results.PushBack(newNode) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go new file mode 100644 index 0000000000..4f0e4da8b2 --- /dev/null +++ b/pkg/yqlib/operator_system_test.go @@ -0,0 +1,114 @@ +package yqlib + +import ( + "os/exec" + "testing" +) + +func findExec(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + t.Skipf("skipping: %v not found: %v", name, err) + } + return path +} + +var systemOperatorDisabledScenarios = []expressionScenario{ + { + description: "system operator returns null when disabled", + subdescription: "Use `--security-enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: null\n", + }, + }, +} + +func TestSystemOperatorDisabledScenarios(t *testing.T) { + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = false + + for _, tt := range systemOperatorDisabledScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios) +} + +func TestSystemOperatorEnabledScenarios(t *testing.T) { + echoPath := findExec(t, "echo") + falsePath := findExec(t, "false") + + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = true + + scenarios := []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--security-enable-system-operator` to enable the system operator.", + yqFlags: "--security-enable-system-operator", + document: "country: Australia", + expression: `.country = system("` + echoPath + `"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + yqFlags: "--security-enable-system-operator", + document: "a: hello", + expression: `.a = system("` + echoPath + `")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command and args are evaluated per matched node", + skipDoc: true, + document: "cmd: " + echoPath + "\narg: hello", + expression: `.result = system(.cmd; .arg)`, + expected: []string{ + "D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + falsePath + `")`, + expectedError: "system command '" + falsePath + "' failed: exit status 1", + }, + { + description: "Null command returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system(null)`, + expectedError: "system operator: command must be a string scalar", + }, + } + + for _, tt := range scenarios { + testScenario(t, &tt) + } + appendOperatorDocumentScenario(t, "system-operators", scenarios) +} diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 1646fb9b4a..b4d4301e5a 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -31,6 +31,7 @@ type expressionScenario struct { dontFormatInputForDoc bool // dont format input doc for documentation generation requiresFormat string skipForGoccy bool + yqFlags string // extra yq flags to include in generated doc command snippets } var goccyTesting = false @@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { writeOrPanic(w, "then\n") + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) } else { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files)) } } else { writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression)) + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression)) } return formattedDoc, formattedDoc2 } diff --git a/pkg/yqlib/security_prefs.go b/pkg/yqlib/security_prefs.go index 3e2fe6b49a..3c203014f2 100644 --- a/pkg/yqlib/security_prefs.go +++ b/pkg/yqlib/security_prefs.go @@ -1,11 +1,13 @@ package yqlib type SecurityPreferences struct { - DisableEnvOps bool - DisableFileOps bool + DisableEnvOps bool + DisableFileOps bool + EnableSystemOps bool } var ConfiguredSecurityPreferences = SecurityPreferences{ - DisableEnvOps: false, - DisableFileOps: false, + DisableEnvOps: false, + DisableFileOps: false, + EnableSystemOps: false, }