Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ dependencies = [
"cryptography>=40.0.0",
"typing-extensions>=4.15.0",
"python-benedict==0.30",
"ruamel-yaml>=0.17.0",
]

[dependency-groups]
Expand Down
21 changes: 18 additions & 3 deletions riocli/configtree/import_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import click
from benedict import benedict
from click_help_colors import HelpColorsCommand
from ruamel.yaml import YAML as _YAML
from yaspin.core import Yaspin

from riocli.config import new_v2_client
Expand Down Expand Up @@ -94,7 +95,7 @@
"--override",
"overrides",
type=click.Path(exists=True),
default=None,
default=(),
multiple=True,
help="Override values for keys in the imported files.",
)
Expand Down Expand Up @@ -277,6 +278,13 @@ def split_metadata(data: Iterable) -> (Iterable, Iterable):
return content, metadata


def _load_yaml_file(path: str) -> dict:
"""Load a YAML file using YAML 1.2 (ruamel.yaml) to avoid implicit type coercions."""
_yaml = _YAML(typ="safe")
with open(path) as f:
return _yaml.load(f) or {}


def _process_files_with_overrides(
files: Iterable[str],
overrides: Iterable[str],
Expand All @@ -295,7 +303,10 @@ def _process_files_with_overrides(
if f.endswith("json"):
file_format = "json"

data[file_prefix] = benedict(f, format=file_format)
if file_format == "json":
data[file_prefix] = benedict(f, format="json")
else:
data[file_prefix] = benedict(_load_yaml_file(f))
spinner.write(
click.style(
f"{Symbols.SUCCESS} File {f} processed.",
Expand All @@ -313,7 +324,11 @@ def _process_files_with_overrides(
if f.endswith("json"):
file_format = "json"

override.merge(benedict(f, format=file_format).unflatten(separator="/"))
if file_format == "json":
loaded = benedict(f, format="json")
else:
loaded = benedict(_load_yaml_file(f))
override.merge(loaded.unflatten(separator="/"))
Comment on lines +327 to +331

spinner.write(
click.style(
Expand Down
3 changes: 2 additions & 1 deletion riocli/configtree/revision.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
MILESTONE_LABEL_KEY,
display_config_tree_keys,
get_revision_from_state,
parse_configtree_value,
save_revision,
serialize_value,
)
Expand Down Expand Up @@ -430,7 +431,7 @@ def put_key_in_revision(
with Revision(
tree_name=tree_name, spinner=spinner, client=client, with_org=with_org
) as rev:
rev.store(key=key, value=value)
rev.store(key=key, value=parse_configtree_value(value))
spinner.write(click.style(f"\t{Symbols.SUCCESS} Key {key} added."))
except Exception as e:
spinner.text = click.style(
Expand Down
41 changes: 37 additions & 4 deletions riocli/configtree/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@
from datetime import date, datetime
from typing import TYPE_CHECKING, Any

import yaml
from benedict import benedict
from munch import Munch, munchify, unmunchify
from rapyuta_io_sdk_v2 import walk_pages
from ruamel.yaml import YAML, YAMLError

from riocli.config import new_v2_client
from riocli.utils import tabulate_data
Expand All @@ -31,6 +31,11 @@

if TYPE_CHECKING:
from collections.abc import Iterable

_yaml_reader = YAML(typ="safe")
_yaml_writer = YAML()
_yaml_writer.default_flow_style = False

MILESTONE_LABEL_KEY = "rapyuta.io/milestone"
TOP_KEYS_FILE = "top-keys"

Expand Down Expand Up @@ -219,6 +224,33 @@ def serialize_value(value: Any) -> str:
)


def parse_configtree_value(value: str) -> Any:
"""Parse a CLI string value using YAML 1.2 semantics before storage.

Gives put-key the same type detection as the import path so that
'{a: 1}' → {"a": 1} (JSON), '[1,2]' → [1, 2] (list), 'true' → bool,
'yes' → str (YAML 1.2, not coerced to bool), etc.
Falls back to the raw string when the value is not parseable as YAML.
"""
try:
return _yaml_reader.load(value)
except YAMLError:
return value


def _to_plain(obj: Any) -> Any:
"""Recursively convert dict/list subclasses to plain Python containers.

ruamel.yaml's representer only handles exact dict/list types, not subclasses
like benedict or CommentedMap, so we normalise before dumping.
"""
if isinstance(obj, dict):
return {k: _to_plain(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_to_plain(v) for v in obj]
return obj


def export_to_files(base_dir: str, data: dict, file_format: str = "yaml") -> None:
base_dir = os.path.abspath(base_dir)

Expand All @@ -228,7 +260,8 @@ def export_to_files(base_dir: str, data: dict, file_format: str = "yaml") -> Non
file_path = os.path.join(base_dir, f"{file_name}.{file_format}")
final_data = benedict(file_data)
if file_format == "yaml":
final_data.to_yaml(filepath=file_path)
with open(file_path, "w") as fh:
_yaml_writer.dump(_to_plain(final_data), fh)
elif file_format == "json":
final_data.to_json(filepath=file_path, indent=4)
else:
Expand Down Expand Up @@ -313,8 +346,8 @@ def combine_metadata(keys: dict) -> dict:
# appropriate data-type in Python (as well in exports), we are
# passing it through YAML parser.
try:
data = yaml.safe_load(data)
except yaml.YAMLError:
data = _yaml_reader.load(data)
except YAMLError:
# Values are not guaranteed to be valid YAML, e.g. logging
# format strings like "[%(levelname)s] ...". Keep the raw
# string as-is when parsing fails.
Expand Down
75 changes: 74 additions & 1 deletion tests/unit/configtree/test_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@

from base64 import b64encode

from riocli.configtree.util import combine_metadata
from riocli.configtree.util import (
combine_metadata,
parse_configtree_value,
serialize_value,
)


def _encode(value: str) -> str:
Expand All @@ -41,6 +45,31 @@ def test_yaml_scalars_are_typed(self):
assert result["a/bool"] is True
assert result["a/str"] == "hello"

def test_yaml11_booleans_are_preserved_as_strings(self):
# YAML 1.1 coerces yes/no/on/off to bool; YAML 1.2 (ruamel) keeps them
# as strings — the core fix for the configtree import/export pipeline.
keys = {
"a/yes": {"data": _encode("yes")},
"a/no": {"data": _encode("no")},
"a/on": {"data": _encode("on")},
"a/off": {"data": _encode("off")},
}

result = combine_metadata(keys)

assert result["a/yes"] == "yes"
assert result["a/no"] == "no"
assert result["a/on"] == "on"
assert result["a/off"] == "off"

def test_octal_0777_is_decimal_777(self):
# YAML 1.1 (PyYAML) parses 0777 as octal 511; YAML 1.2 treats it as decimal.
keys = {"cfg/mode": {"data": _encode("0777")}}

result = combine_metadata(keys)

assert result["cfg/mode"] == 777

def test_non_yaml_value_is_kept_as_raw_string(self):
# Logging format strings are not valid YAML: the leading '[' starts a
# flow sequence and '%' cannot start a token. They must survive as-is
Expand All @@ -52,6 +81,50 @@ def test_non_yaml_value_is_kept_as_raw_string(self):

assert result["wms/logging/format"] == fmt


class TestParseConfigtreeValue:
"""Tests for parse_configtree_value() — ensures put-key matches import types."""

def test_integer_string_becomes_int(self):
assert parse_configtree_value("300") == 300
assert isinstance(parse_configtree_value("300"), int)

def test_float_string_becomes_float(self):
assert parse_configtree_value("3.14") == 3.14

def test_bool_true_becomes_bool(self):
assert parse_configtree_value("true") is True
assert parse_configtree_value("false") is False

def test_null_becomes_none(self):
assert parse_configtree_value("null") is None

def test_yaml11_booleans_stay_as_strings(self):
# YAML 1.2: yes/no/on/off are NOT booleans — strings preserved.
assert parse_configtree_value("yes") == "yes"
assert parse_configtree_value("no") == "no"
assert parse_configtree_value("on") == "on"
assert parse_configtree_value("off") == "off"

def test_plain_string_passes_through(self):
assert parse_configtree_value("hello") == "hello"

def test_dict_normalizes_to_json_form(self):
# YAML shorthand {a: 1} → Python dict → json.dumps → '{"a": 1}'
parsed = parse_configtree_value("{a: 1}")
stored = serialize_value(parsed)
assert stored == '{"a": 1}'

def test_list_normalizes_spacing(self):
# '[1,2]' (no space) → list [1,2] → json.dumps → '[1, 2]'
parsed = parse_configtree_value("[1,2]")
stored = serialize_value(parsed)
assert stored == "[1, 2]"

def test_invalid_yaml_falls_back_to_raw_string(self):
fmt = "[%(levelname)s] [%(asctime)s]: %(message)s"
assert parse_configtree_value(fmt) == fmt

def test_metadata_is_combined_with_value(self):
keys = {
"a/key": {
Expand Down
11 changes: 11 additions & 0 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading