diff --git a/pyproject.toml b/pyproject.toml index 57237268..563784e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "cryptography>=40.0.0", "typing-extensions>=4.15.0", "python-benedict==0.30", + "ruamel-yaml>=0.17.0", ] [dependency-groups] diff --git a/riocli/configtree/import_keys.py b/riocli/configtree/import_keys.py index bee57f6f..b924327e 100644 --- a/riocli/configtree/import_keys.py +++ b/riocli/configtree/import_keys.py @@ -17,6 +17,7 @@ import click from benedict import benedict from click_help_colors import HelpColorsCommand +from ruamel.yaml import YAML as _YAML from yaspin.core import Yaspin from riocli.config import new_v2_client @@ -94,7 +95,7 @@ "--override", "overrides", type=click.Path(exists=True), - default=None, + default=(), multiple=True, help="Override values for keys in the imported files.", ) @@ -277,6 +278,13 @@ def split_metadata(data: Iterable) -> (Iterable, Iterable): return content, metadata +def _load_yaml_file(path: str) -> dict: + """Load a YAML file using YAML 1.2 (ruamel.yaml) to avoid implicit type coercions.""" + _yaml = _YAML(typ="safe") + with open(path) as f: + return _yaml.load(f) or {} + + def _process_files_with_overrides( files: Iterable[str], overrides: Iterable[str], @@ -295,7 +303,10 @@ def _process_files_with_overrides( if f.endswith("json"): file_format = "json" - data[file_prefix] = benedict(f, format=file_format) + if file_format == "json": + data[file_prefix] = benedict(f, format="json") + else: + data[file_prefix] = benedict(_load_yaml_file(f)) spinner.write( click.style( f"{Symbols.SUCCESS} File {f} processed.", @@ -313,7 +324,11 @@ def _process_files_with_overrides( if f.endswith("json"): file_format = "json" - override.merge(benedict(f, format=file_format).unflatten(separator="/")) + if file_format == "json": + loaded = benedict(f, format="json") + else: + loaded = benedict(_load_yaml_file(f)) + override.merge(loaded.unflatten(separator="/")) spinner.write( click.style( diff --git a/riocli/configtree/revision.py b/riocli/configtree/revision.py index 7a4c068a..e3d182c7 100644 --- a/riocli/configtree/revision.py +++ b/riocli/configtree/revision.py @@ -29,6 +29,7 @@ MILESTONE_LABEL_KEY, display_config_tree_keys, get_revision_from_state, + parse_configtree_value, save_revision, serialize_value, ) @@ -430,7 +431,7 @@ def put_key_in_revision( with Revision( tree_name=tree_name, spinner=spinner, client=client, with_org=with_org ) as rev: - rev.store(key=key, value=value) + rev.store(key=key, value=parse_configtree_value(value)) spinner.write(click.style(f"\t{Symbols.SUCCESS} Key {key} added.")) except Exception as e: spinner.text = click.style( diff --git a/riocli/configtree/util.py b/riocli/configtree/util.py index ce917f4f..b3b5f667 100644 --- a/riocli/configtree/util.py +++ b/riocli/configtree/util.py @@ -19,10 +19,10 @@ from datetime import date, datetime from typing import TYPE_CHECKING, Any -import yaml from benedict import benedict from munch import Munch, munchify, unmunchify from rapyuta_io_sdk_v2 import walk_pages +from ruamel.yaml import YAML, YAMLError from riocli.config import new_v2_client from riocli.utils import tabulate_data @@ -31,6 +31,11 @@ if TYPE_CHECKING: from collections.abc import Iterable + +_yaml_reader = YAML(typ="safe") +_yaml_writer = YAML() +_yaml_writer.default_flow_style = False + MILESTONE_LABEL_KEY = "rapyuta.io/milestone" TOP_KEYS_FILE = "top-keys" @@ -219,6 +224,33 @@ def serialize_value(value: Any) -> str: ) +def parse_configtree_value(value: str) -> Any: + """Parse a CLI string value using YAML 1.2 semantics before storage. + + Gives put-key the same type detection as the import path so that + '{a: 1}' → {"a": 1} (JSON), '[1,2]' → [1, 2] (list), 'true' → bool, + 'yes' → str (YAML 1.2, not coerced to bool), etc. + Falls back to the raw string when the value is not parseable as YAML. + """ + try: + return _yaml_reader.load(value) + except YAMLError: + return value + + +def _to_plain(obj: Any) -> Any: + """Recursively convert dict/list subclasses to plain Python containers. + + ruamel.yaml's representer only handles exact dict/list types, not subclasses + like benedict or CommentedMap, so we normalise before dumping. + """ + if isinstance(obj, dict): + return {k: _to_plain(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_to_plain(v) for v in obj] + return obj + + def export_to_files(base_dir: str, data: dict, file_format: str = "yaml") -> None: base_dir = os.path.abspath(base_dir) @@ -228,7 +260,8 @@ def export_to_files(base_dir: str, data: dict, file_format: str = "yaml") -> Non file_path = os.path.join(base_dir, f"{file_name}.{file_format}") final_data = benedict(file_data) if file_format == "yaml": - final_data.to_yaml(filepath=file_path) + with open(file_path, "w") as fh: + _yaml_writer.dump(_to_plain(final_data), fh) elif file_format == "json": final_data.to_json(filepath=file_path, indent=4) else: @@ -313,8 +346,8 @@ def combine_metadata(keys: dict) -> dict: # appropriate data-type in Python (as well in exports), we are # passing it through YAML parser. try: - data = yaml.safe_load(data) - except yaml.YAMLError: + data = _yaml_reader.load(data) + except YAMLError: # Values are not guaranteed to be valid YAML, e.g. logging # format strings like "[%(levelname)s] ...". Keep the raw # string as-is when parsing fails. diff --git a/tests/unit/configtree/test_util.py b/tests/unit/configtree/test_util.py index 3d5a498a..dcec6485 100644 --- a/tests/unit/configtree/test_util.py +++ b/tests/unit/configtree/test_util.py @@ -18,7 +18,11 @@ from base64 import b64encode -from riocli.configtree.util import combine_metadata +from riocli.configtree.util import ( + combine_metadata, + parse_configtree_value, + serialize_value, +) def _encode(value: str) -> str: @@ -41,6 +45,31 @@ def test_yaml_scalars_are_typed(self): assert result["a/bool"] is True assert result["a/str"] == "hello" + def test_yaml11_booleans_are_preserved_as_strings(self): + # YAML 1.1 coerces yes/no/on/off to bool; YAML 1.2 (ruamel) keeps them + # as strings — the core fix for the configtree import/export pipeline. + keys = { + "a/yes": {"data": _encode("yes")}, + "a/no": {"data": _encode("no")}, + "a/on": {"data": _encode("on")}, + "a/off": {"data": _encode("off")}, + } + + result = combine_metadata(keys) + + assert result["a/yes"] == "yes" + assert result["a/no"] == "no" + assert result["a/on"] == "on" + assert result["a/off"] == "off" + + def test_octal_0777_is_decimal_777(self): + # YAML 1.1 (PyYAML) parses 0777 as octal 511; YAML 1.2 treats it as decimal. + keys = {"cfg/mode": {"data": _encode("0777")}} + + result = combine_metadata(keys) + + assert result["cfg/mode"] == 777 + def test_non_yaml_value_is_kept_as_raw_string(self): # Logging format strings are not valid YAML: the leading '[' starts a # flow sequence and '%' cannot start a token. They must survive as-is @@ -52,6 +81,50 @@ def test_non_yaml_value_is_kept_as_raw_string(self): assert result["wms/logging/format"] == fmt + +class TestParseConfigtreeValue: + """Tests for parse_configtree_value() — ensures put-key matches import types.""" + + def test_integer_string_becomes_int(self): + assert parse_configtree_value("300") == 300 + assert isinstance(parse_configtree_value("300"), int) + + def test_float_string_becomes_float(self): + assert parse_configtree_value("3.14") == 3.14 + + def test_bool_true_becomes_bool(self): + assert parse_configtree_value("true") is True + assert parse_configtree_value("false") is False + + def test_null_becomes_none(self): + assert parse_configtree_value("null") is None + + def test_yaml11_booleans_stay_as_strings(self): + # YAML 1.2: yes/no/on/off are NOT booleans — strings preserved. + assert parse_configtree_value("yes") == "yes" + assert parse_configtree_value("no") == "no" + assert parse_configtree_value("on") == "on" + assert parse_configtree_value("off") == "off" + + def test_plain_string_passes_through(self): + assert parse_configtree_value("hello") == "hello" + + def test_dict_normalizes_to_json_form(self): + # YAML shorthand {a: 1} → Python dict → json.dumps → '{"a": 1}' + parsed = parse_configtree_value("{a: 1}") + stored = serialize_value(parsed) + assert stored == '{"a": 1}' + + def test_list_normalizes_spacing(self): + # '[1,2]' (no space) → list [1,2] → json.dumps → '[1, 2]' + parsed = parse_configtree_value("[1,2]") + stored = serialize_value(parsed) + assert stored == "[1, 2]" + + def test_invalid_yaml_falls_back_to_raw_string(self): + fmt = "[%(levelname)s] [%(asctime)s]: %(message)s" + assert parse_configtree_value(fmt) == fmt + def test_metadata_is_combined_with_value(self): keys = { "a/key": { diff --git a/uv.lock b/uv.lock index de6a2e2f..0ae6ed48 100644 --- a/uv.lock +++ b/uv.lock @@ -1554,6 +1554,7 @@ dependencies = [ { name = "rapyuta-io" }, { name = "rapyuta-io-sdk-v2" }, { name = "requests" }, + { name = "ruamel-yaml" }, { name = "semver" }, { name = "setuptools" }, { name = "six" }, @@ -1611,6 +1612,7 @@ requires-dist = [ { name = "rapyuta-io", specifier = ">=3.1.0" }, { name = "rapyuta-io-sdk-v2", specifier = ">=0.8.0" }, { name = "requests", specifier = ">=2.20.0" }, + { name = "ruamel-yaml", specifier = ">=0.17.0" }, { name = "semver", specifier = ">=3.0.0" }, { name = "setuptools" }, { name = "six", specifier = ">=1.13.0" }, @@ -1979,6 +1981,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ea/e7b0251441da9adfeaebcf29601d10f2a1455fcf0772fae9e7e19032bd96/rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8c43a8a973270fd173bf48cdf80bbe66312421cba68d40845034f174f2389049", size = 586326, upload-time = "2026-05-28T12:02:11.47Z" }, ] +[[package]] +name = "ruamel-yaml" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload-time = "2026-01-02T16:50:29.201Z" }, +] + [[package]] name = "semver" version = "3.0.4"