From 87384f143d492dfaeb03a174a6e17560ae11d012 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Tue, 10 Mar 2026 20:18:10 +0000 Subject: [PATCH 1/6] Fix non base 10 number parsing issues in YAML --- CHANGELOG.md | 9 ++++----- parsing/yaml/yaml_reader.go | 4 ++-- parsing/yaml/yaml_test.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e6c7f6..3496cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- Nothing yet. +### Fixed + +- Fixed a bug in YAML parser that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). ## [v3.4.1] - 2026-03-30 @@ -792,10 +794,7 @@ See [documentation](https://daseldocs.tomwright.me) for all changes. - Everything! -[unreleased]: https://github.com/TomWright/dasel/compare/v3.4.1...HEAD -[v3.4.1]: https://github.com/TomWright/dasel/compare/v3.4.0...v3.4.1 -[v3.4.0]: https://github.com/TomWright/dasel/compare/v3.3.2...v3.4.0 -[v3.3.2]: https://github.com/TomWright/dasel/compare/v3.3.1...v3.3.2 +[unreleased]: https://github.com/TomWright/dasel/compare/v3.3.1...HEAD [v3.3.1]: https://github.com/TomWright/dasel/compare/v3.3.0...v3.3.1 [v3.3.0]: https://github.com/TomWright/dasel/compare/v3.2.3...v3.3.0 [v3.2.3]: https://github.com/TomWright/dasel/compare/v3.2.2...v3.2.3 diff --git a/parsing/yaml/yaml_reader.go b/parsing/yaml/yaml_reader.go index ed82d8f..29af53f 100644 --- a/parsing/yaml/yaml_reader.go +++ b/parsing/yaml/yaml_reader.go @@ -96,11 +96,11 @@ func (yv *yamlValue) UnmarshalYAML(value *yaml.Node) error { case "!!bool": yv.value = model.NewBoolValue(value.Value == "true") case "!!int": - i, err := strconv.Atoi(value.Value) + i, err := strconv.ParseInt(value.Value, 0, 64) if err != nil { return err } - yv.value = model.NewIntValue(int64(i)) + yv.value = model.NewIntValue(i) case "!!float": f, err := strconv.ParseFloat(value.Value, 64) if err != nil { diff --git a/parsing/yaml/yaml_test.go b/parsing/yaml/yaml_test.go index 9093d4d..93b0786 100644 --- a/parsing/yaml/yaml_test.go +++ b/parsing/yaml/yaml_test.go @@ -204,6 +204,43 @@ name2: Tom `, }.run) + t.Run("base numbers", func(t *testing.T) { + t.Run("standard", rwTestCase{ + in: `10 +`, + out: `10 +`, + }.run) + + t.Run("hex", rwTestCase{ + in: `0x10 +`, + out: `16 +`, + }.run) + + t.Run("octal", rwTestCase{ + in: `0o10 +`, + out: `8 +`, + }.run) + + t.Run("binary", rwTestCase{ + in: `0b10 +`, + out: `2 +`, + }.run) + + t.Run("leading zero", rwTestCase{ + in: `010 +`, + out: `8 +`, + }.run) + }) + t.Run("bounded yaml expansion", func(t *testing.T) { in := `a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] From 5cf1594cbd11b2cfe4877f4f59a9e78ea62373f7 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Tue, 10 Mar 2026 20:21:18 +0000 Subject: [PATCH 2/6] Fix integer parsing in various parsers and functions --- CHANGELOG.md | 2 ++ execution/func_to_int.go | 2 +- parsing/toml/toml_reader.go | 2 +- selector/parser/parse_literal.go | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3496cce..edfdc62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed a bug in YAML parser that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). +- Fixed a bug in YAML, TOML and dasel query parser that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). +- Fixed a bug in the `toInt` function that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). ## [v3.4.1] - 2026-03-30 diff --git a/execution/func_to_int.go b/execution/func_to_int.go index d8c3872..578d334 100644 --- a/execution/func_to_int.go +++ b/execution/func_to_int.go @@ -19,7 +19,7 @@ var FuncToInt = NewFunc( return nil, err } - i, err := strconv.ParseInt(stringValue, 10, 64) + i, err := strconv.ParseInt(stringValue, 0, 64) if err != nil { return nil, err } diff --git a/parsing/toml/toml_reader.go b/parsing/toml/toml_reader.go index 56f56f3..55d3aec 100644 --- a/parsing/toml/toml_reader.go +++ b/parsing/toml/toml_reader.go @@ -150,7 +150,7 @@ func (j *tomlReader) readNode(p *unstable.Parser, n *unstable.Node) (string, *mo } return "", model.NewFloatValue(f), nil case unstable.Integer: - i64, err := strconv.ParseInt(string(n.Data), 10, 64) + i64, err := strconv.ParseInt(string(n.Data), 0, 64) if err != nil { return "", nil, err } diff --git a/selector/parser/parse_literal.go b/selector/parser/parse_literal.go index b155906..11656a6 100644 --- a/selector/parser/parse_literal.go +++ b/selector/parser/parse_literal.go @@ -68,7 +68,7 @@ func parseNumberLiteral(p *Parser) (ast.Expr, error) { }, nil default: - value, err := strconv.ParseInt(token.Value, 10, 64) + value, err := strconv.ParseInt(token.Value, 0, 64) if err != nil { return nil, err } From 01434f50047fe6d6663d4dedd3104dcf999701d5 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:40:36 +0100 Subject: [PATCH 3/6] Improve YAML number handling --- parsing/yaml/yaml_reader.go | 22 ++++++++++++++- parsing/yaml/yaml_test.go | 54 ++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/parsing/yaml/yaml_reader.go b/parsing/yaml/yaml_reader.go index 29af53f..1c5d7a2 100644 --- a/parsing/yaml/yaml_reader.go +++ b/parsing/yaml/yaml_reader.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "strconv" + "strings" "github.com/tomwright/dasel/v3/model" "github.com/tomwright/dasel/v3/parsing" @@ -96,7 +97,7 @@ func (yv *yamlValue) UnmarshalYAML(value *yaml.Node) error { case "!!bool": yv.value = model.NewBoolValue(value.Value == "true") case "!!int": - i, err := strconv.ParseInt(value.Value, 0, 64) + i, err := parseYAMLInt(value.Value) if err != nil { return err } @@ -186,3 +187,22 @@ func (yv *yamlValue) UnmarshalYAML(value *yaml.Node) error { } return nil } + +func parseYAMLInt(s string) (int64, error) { + // Strip leading sign for prefix detection. + clean := s + if len(clean) > 0 && (clean[0] == '+' || clean[0] == '-') { + clean = clean[1:] + } + + switch { + case strings.HasPrefix(clean, "0x") || strings.HasPrefix(clean, "0X"): + return strconv.ParseInt(s, 0, 64) + case strings.HasPrefix(clean, "0o") || strings.HasPrefix(clean, "0O"): + return strconv.ParseInt(s, 0, 64) + case strings.HasPrefix(clean, "0b") || strings.HasPrefix(clean, "0B"): + return strconv.ParseInt(s, 0, 64) + default: + return strconv.ParseInt(s, 10, 64) + } +} \ No newline at end of file diff --git a/parsing/yaml/yaml_test.go b/parsing/yaml/yaml_test.go index 93b0786..0db37b2 100644 --- a/parsing/yaml/yaml_test.go +++ b/parsing/yaml/yaml_test.go @@ -212,13 +212,34 @@ name2: Tom `, }.run) - t.Run("hex", rwTestCase{ + t.Run("zero", rwTestCase{ + in: `0 +`, + out: `0 +`, + }.run) + + t.Run("negative", rwTestCase{ + in: `-42 +`, + out: `-42 +`, + }.run) + + t.Run("hex lowercase", rwTestCase{ in: `0x10 `, out: `16 `, }.run) + t.Run("hex uppercase letters", rwTestCase{ + in: `0xff +`, + out: `255 +`, + }.run) + t.Run("octal", rwTestCase{ in: `0o10 `, @@ -233,10 +254,37 @@ name2: Tom `, }.run) - t.Run("leading zero", rwTestCase{ + t.Run("leading zero is decimal", rwTestCase{ in: `010 `, - out: `8 + out: `10 +`, + }.run) + + t.Run("hex in map", rwTestCase{ + in: `val: 0x10 +`, + out: `val: 16 +`, + }.run) + + t.Run("octal in map", rwTestCase{ + in: `val: 0o77 +`, + out: `val: 63 +`, + }.run) + + t.Run("mixed types in map", rwTestCase{ + in: `dec: 42 +hex: 0xff +oct: 0o77 +bin: 0b1010 +`, + out: `dec: 42 +hex: 255 +oct: 63 +bin: 10 `, }.run) }) From 16c95d4087ca2c30f420b1d1366b3f79411f67a9 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Mon, 30 Mar 2026 19:46:28 +0100 Subject: [PATCH 4/6] Fix a bad rebase --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edfdc62..436ab14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed a bug in YAML parser that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). - Fixed a bug in YAML, TOML and dasel query parser that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). - Fixed a bug in the `toInt` function that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). @@ -796,7 +795,10 @@ See [documentation](https://daseldocs.tomwright.me) for all changes. - Everything! -[unreleased]: https://github.com/TomWright/dasel/compare/v3.3.1...HEAD +[unreleased]: https://github.com/TomWright/dasel/compare/v3.4.1...HEAD +[v3.4.1]: https://github.com/TomWright/dasel/compare/v3.4.0...v3.4.1 +[v3.4.0]: https://github.com/TomWright/dasel/compare/v3.3.2...v3.4.0 +[v3.3.2]: https://github.com/TomWright/dasel/compare/v3.3.1...v3.3.2 [v3.3.1]: https://github.com/TomWright/dasel/compare/v3.3.0...v3.3.1 [v3.3.0]: https://github.com/TomWright/dasel/compare/v3.2.3...v3.3.0 [v3.2.3]: https://github.com/TomWright/dasel/compare/v3.2.2...v3.2.3 From 85a8410a42f802411005732b981cda2cb8db8699 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:55:52 +0100 Subject: [PATCH 5/6] fix: improve YAML/TOML number parsing and revert parser base-0 regression - Handle underscores in YAML decimal integers (e.g. 1_000) - Handle signed prefixed numbers in YAML (+0x10, -0o10, etc.) - Add TOML test coverage for hex, octal, binary, and underscore numbers - Revert parse_literal.go to base 10 to prevent 010 being parsed as octal --- parsing/toml/toml_reader_test.go | 38 +++++++++++++++++ parsing/yaml/yaml_reader.go | 5 ++- parsing/yaml/yaml_test.go | 70 ++++++++++++++++++++++++++++++++ selector/parser/parse_literal.go | 2 +- selector/parser/parser_test.go | 4 ++ 5 files changed, 117 insertions(+), 2 deletions(-) diff --git a/parsing/toml/toml_reader_test.go b/parsing/toml/toml_reader_test.go index 42ae43d..911aa0c 100644 --- a/parsing/toml/toml_reader_test.go +++ b/parsing/toml/toml_reader_test.go @@ -514,6 +514,44 @@ func TestTomlReader_EdgeCases(t *testing.T) { }) } +func TestTomlReader_NonBase10Numbers(t *testing.T) { + t.Run("hex", tomlReaderTest([]byte(`val = 0x12`), func() *model.Value { + res := model.NewMapValue() + _ = res.SetMapKey("val", model.NewIntValue(18)) + return res + })) + + t.Run("octal", tomlReaderTest([]byte(`val = 0o7`), func() *model.Value { + res := model.NewMapValue() + _ = res.SetMapKey("val", model.NewIntValue(7)) + return res + })) + + t.Run("binary", tomlReaderTest([]byte(`val = 0b111`), func() *model.Value { + res := model.NewMapValue() + _ = res.SetMapKey("val", model.NewIntValue(7)) + return res + })) + + t.Run("underscore decimal", tomlReaderTest([]byte(`val = 12_000`), func() *model.Value { + res := model.NewMapValue() + _ = res.SetMapKey("val", model.NewIntValue(12000)) + return res + })) + + t.Run("negative underscore decimal", tomlReaderTest([]byte(`val = -12_000`), func() *model.Value { + res := model.NewMapValue() + _ = res.SetMapKey("val", model.NewIntValue(-12000)) + return res + })) + + t.Run("negative int", tomlReaderTest([]byte(`val = -42`), func() *model.Value { + res := model.NewMapValue() + _ = res.SetMapKey("val", model.NewIntValue(-42)) + return res + })) +} + func TestTomlReader_TimeStrings(t *testing.T) { // Local date t.Run("local date string", func(t *testing.T) { diff --git a/parsing/yaml/yaml_reader.go b/parsing/yaml/yaml_reader.go index 1c5d7a2..9f1be6f 100644 --- a/parsing/yaml/yaml_reader.go +++ b/parsing/yaml/yaml_reader.go @@ -203,6 +203,9 @@ func parseYAMLInt(s string) (int64, error) { case strings.HasPrefix(clean, "0b") || strings.HasPrefix(clean, "0B"): return strconv.ParseInt(s, 0, 64) default: - return strconv.ParseInt(s, 10, 64) + // YAML 1.2 allows underscores in decimal integers (e.g. 1_000). + // strconv.ParseInt with base 10 does not support underscores, + // so we strip them before parsing. + return strconv.ParseInt(strings.ReplaceAll(s, "_", ""), 10, 64) } } \ No newline at end of file diff --git a/parsing/yaml/yaml_test.go b/parsing/yaml/yaml_test.go index 0db37b2..131ab8e 100644 --- a/parsing/yaml/yaml_test.go +++ b/parsing/yaml/yaml_test.go @@ -285,6 +285,76 @@ bin: 0b1010 hex: 255 oct: 63 bin: 10 +`, + }.run) + + t.Run("positive sign", rwTestCase{ + in: `+42 +`, + out: `42 +`, + }.run) + + t.Run("positive hex", rwTestCase{ + in: `+0x10 +`, + out: `16 +`, + }.run) + + t.Run("positive octal", rwTestCase{ + in: `+0o10 +`, + out: `8 +`, + }.run) + + t.Run("positive binary", rwTestCase{ + in: `+0b10 +`, + out: `2 +`, + }.run) + + t.Run("negative hex", rwTestCase{ + in: `-0x10 +`, + out: `-16 +`, + }.run) + + t.Run("negative octal", rwTestCase{ + in: `-0o10 +`, + out: `-8 +`, + }.run) + + t.Run("negative binary", rwTestCase{ + in: `-0b10 +`, + out: `-2 +`, + }.run) + + t.Run("underscore decimal", rwTestCase{ + in: `1_000 +`, + out: `1000 +`, + }.run) + + t.Run("underscore hex", rwTestCase{ + in: `0xFF_FF +`, + out: `65535 +`, + }.run) + + t.Run("underscore binary", rwTestCase{ + in: `0b1010_1010 +`, + out: `170 `, }.run) }) diff --git a/selector/parser/parse_literal.go b/selector/parser/parse_literal.go index 11656a6..b155906 100644 --- a/selector/parser/parse_literal.go +++ b/selector/parser/parse_literal.go @@ -68,7 +68,7 @@ func parseNumberLiteral(p *Parser) (ast.Expr, error) { }, nil default: - value, err := strconv.ParseInt(token.Value, 0, 64) + value, err := strconv.ParseInt(token.Value, 10, 64) if err != nil { return nil, err } diff --git a/selector/parser/parser_test.go b/selector/parser/parser_test.go index 0d87503..0851b72 100644 --- a/selector/parser/parser_test.go +++ b/selector/parser/parser_test.go @@ -61,6 +61,10 @@ func TestParser_Parse_HappyPath(t *testing.T) { input: "42", expected: ast.NumberIntExpr{Value: 42}, }.run) + t.Run("leading zero decimal", happyTestCase{ + input: "010", + expected: ast.NumberIntExpr{Value: 10}, + }.run) t.Run("float", happyTestCase{ input: "42.1", expected: ast.NumberFloatExpr{Value: 42.1}, From 5bb67d51b309a137a11e4e816a6ded62df27f525 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:56:25 +0100 Subject: [PATCH 6/6] fix: correct changelog wording and add missing newline in yaml_reader --- CHANGELOG.md | 2 +- parsing/yaml/yaml_reader.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a912ecc..f99f9bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed a bug in YAML, TOML and dasel query parser that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). +- Fixed a bug in YAML and TOML parsers that caused them to fail when parsing non-base10 numbers (e.g. hex, binary, octal). - Fixed a bug in the `toInt` function that caused it to fail when parsing non-base10 numbers (e.g. hex, binary, octal). - XML child element ordering now has more comprehensive round-trip handling. Thanks @takeokunn. diff --git a/parsing/yaml/yaml_reader.go b/parsing/yaml/yaml_reader.go index 9f1be6f..6cef7fa 100644 --- a/parsing/yaml/yaml_reader.go +++ b/parsing/yaml/yaml_reader.go @@ -208,4 +208,4 @@ func parseYAMLInt(s string) (int64, error) { // so we strip them before parsing. return strconv.ParseInt(strings.ReplaceAll(s, "_", ""), 10, 64) } -} \ No newline at end of file +}