Specify code unit adaptor conversion rule in terms of width, not sizeof

Per SG16 feedback, compare the width of the source and target types
([basic.fundamental]: value bits plus sign bit, excluding padding)
rather than sizeof, so that padding bits cannot hide a narrowing
conversion. The implementation computes width from numeric_limits
digits and is_signed, using unsigned char for std::byte.

Update the wording to define width for integer types, character types,
and byte via the underlying type. Replace the design section's single
conversion table with two: one covering all integral type sizes the
standard permits (assuming CHAR_BIT == 8), and one covering typical
x86_64/aarch64 sizes, using "same width" wherever LP64 and LLP64
targets differ.

Also size the exposition-only transcoding buffer as 4, 2, or 1 elements
for char8_t, char16_t, or char32_t respectively, rather than
4 / sizeof(ToType), which under-sizes the buffer on an implementation
whose char16_t has padding bits.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: ednolan

include/beman/utf_view/code_unit_view.hpp

@@ -22,6 +22,7 @@ import beman.utf_view;
 #include <beman/transform_view/transform_view.hpp>
 #include <concepts>
 #include <cstddef>
+#include <limits>
 #include <ranges>
 #include <type_traits>
 #endif
@@ -47,18 +48,32 @@ struct exposition_only_cast_to {
 
 namespace detail {
 
+  // The width of an integral type is the number of bits in its value
+  // representation ([basic.fundamental]), i.e. value bits plus the sign bit
+  // and excluding padding bits. For std::byte it is the width of its
+  // underlying type, unsigned char.
+  template <class T>
+  using code_unit_width_type =
+    std::conditional_t<std::same_as<T, std::byte>, unsigned char, T>;
+
+  template <class T>
+  constexpr int code_unit_width =
+    std::numeric_limits<code_unit_width_type<T>>::digits +
+    std::numeric_limits<code_unit_width_type<T>>::is_signed;
+
   // A source type U may be cast to the code unit type Char if it is Char
   // itself, or if it is std::byte or a non-bool integral type of the same
-  // width as Char. This rules out narrowing, widening (e.g. accidentally
-  // treating UTF-8 code units as code points), floating-point sources, bool,
-  // enumerations other than std::byte, and class types such as proxy
-  // references.
+  // width as Char. Width rather than sizeof is compared so that padding bits
+  // cannot hide a narrowing conversion. This rules out narrowing, widening
+  // (e.g. accidentally treating UTF-8 code units as code points),
+  // floating-point sources, bool, enumerations other than std::byte, and class
+  // types such as proxy references.
   template <class U, class Char>
   concept code_unit_castable =
     std::same_as<U, Char> ||
     ((std::same_as<U, std::byte> ||
       (std::is_integral_v<U> && !std::same_as<U, bool>)) &&
-     sizeof(U) == sizeof(Char));
+     code_unit_width<U> == code_unit_width<Char>);
 
   template <typename Char>
   struct as_code_unit_impl

include/beman/utf_view/to_utf_view.hpp

@@ -294,8 +294,10 @@ struct to_utf_view<V, E, ToType>::exposition_only_iterator
   [[no_unique_address]] std::ranges::sentinel_t<exposition_only_Base> exposition_only_end_{}; // @*exposition only*@
 /* PAPER:   sentinel_t<exposition_only_Base> @*end_*@; // @*exposition only*@ */
 
-  detail::fake_inplace_vector<value_type, 4 / sizeof(ToType)> exposition_only_buf_{}; // @*exposition only*@
-/* PAPER:   inplace_vector<value_type, 4 / sizeof(ToType)> @*buf_*@{}; // @*exposition only*@ */
+  detail::fake_inplace_vector<value_type,
+                              std::same_as<ToType, char8_t> ? 4 : std::same_as<ToType, char16_t> ? 2 : 1>
+      exposition_only_buf_{}; // @*exposition only*@
+/* PAPER:   inplace_vector<value_type, same_as<ToType, char8_t> ? 4 : same_as<ToType, char16_t> ? 2 : 1> @*buf_*@{}; // @*exposition only*@ */
 /* PAPER */
 
   std::int8_t exposition_only_buf_index_{}; // @*exposition only*@

papers/P2728.md

@@ -748,7 +748,7 @@ private:
   iterator_t<@*Base*@> @*current_*@{}; // @*exposition only*@
   sentinel_t<@*Base*@> @*end_*@; // @*exposition only*@
 
-  inplace_vector<value_type, 4 / sizeof(ToType)> @*buf_*@{}; // @*exposition only*@
+  inplace_vector<value_type, same_as<ToType, char8_t> ? 4 : same_as<ToType, char16_t> ? 2 : 1> @*buf_*@{}; // @*exposition only*@
 
   int8_t @*buf_index_*@{}; // @*exposition only*@
   uint8_t @*to_increment_*@{}; // @*exposition only*@
@@ -1109,11 +1109,13 @@ The names `as_char`, `as_wchar_t`, `as_char8_t`, `as_char16_t`, and
 `as_char16_t`, and `as_char32_t`. Let `Char` be the corresponding character
 type for `as_charT`, let `E` be an expression, let `T` be
 `remove_cvref_t<decltype((E))>`, and let `U` be
-`remove_cvref_t<ranges::range_reference_t<T>>`. `as_charT(E)` is ill-formed if
-`T` is an array, or if `U` is not `Char` and `U` is not a type `X` such that
-`sizeof(X) == sizeof(Char)` and either `X` is `byte` or `X` is an integral type
-other than `bool`. Otherwise, the expression `as_charT(E)` is
-expression-equivalent to:
+`remove_cvref_t<ranges::range_reference_t<T>>`. For a type `X`, let the width
+of `X` be the width ([basic.fundamental]) of `X` if `X` is a signed or unsigned
+integer type, and otherwise the width of the underlying type of `X`.
+`as_charT(E)` is ill-formed if `T` is an array, or if `U` is not `Char` and `U`
+is not a type `X` such that the width of `X` equals the width of `Char` and
+either `X` is `byte` or `X` is an integral type other than `bool`. Otherwise,
+the expression `as_charT(E)` is expression-equivalent to:
 
 - If `T` is a specialization of `empty_view` ([range.empty.view]), then
   `empty_view<Char>{}`.
@@ -1330,7 +1332,7 @@ For the `as_charT` casting views, the rule we use to determine whether the eleme
 is:
 
 - `U` is `Char`, or
-- `U` is `std::byte` or a non-`bool` integral type, and `sizeof(U) == sizeof(Char)`.
+- `U` is `std::byte` or a non-`bool` integral type, and `U` has the same width as `Char`.
 
 This accomplishes the following design goals:
 
@@ -1352,23 +1354,32 @@ out a `transform_view` if, for example, they want to cast between `char16_t` and
 merits an explicit carveout here since there's no justification based on backwards
 compatibility).
 
+### All integral type sizes permitted by the standard, assuming `CHAR_BIT == 8`
+
+"Same width" marks pairs whose relative widths the standard does not fix.
+
+| Source | `as_char` | `as_char8_t` | `as_char16_t` | `as_char32_t` | `as_wchar_t` |
+|---|---|---|---|---|---|
+| `char`, `signed char`, `unsigned char`, `char8_t`, `std::byte` | yes | yes | no | no | same width |
+| `char16_t` | no | no | yes | same width | same width |
+| `char32_t` | no | no | same width | yes | same width |
+| `wchar_t` | same width | same width | same width | same width | yes |
+| `short`, `int`, `long`, `long long`, and their `unsigned` counterparts | no | no | same width | same width | same width |
+| `bool`, floating-point types, enumerations other than `std::byte`, class types | no | no | no | no | no |
+
+### Typical x86_64/aarch64 integral type sizes
+
 | Source | `as_char` | `as_char8_t` | `as_char16_t` | `as_char32_t` | `as_wchar_t` |
 |---|---|---|---|---|---|
-| `char` | yes | yes | no | no | no |
-| `signed char`, `unsigned char` | yes | yes | no | no | no |
-| `std::byte` | yes | yes | no | no | no |
-| `char8_t` | yes | yes | no | no | no |
+| `char`, `signed char`, `unsigned char`, `char8_t`, `std::byte` | yes | yes | no | no | no |
 | `char16_t` | no | no | yes | no | same width |
 | `char32_t` | no | no | no | yes | same width |
 | `wchar_t` | no | no | same width | same width | yes |
 | `short`, `unsigned short` | no | no | yes | no | same width |
-| `int`, `unsigned int` | no | no | no | same width | same width |
-| `long`, `unsigned long` | no | no | no | same width | same width |
+| `int`, `unsigned int` | no | no | no | yes | same width |
+| `long`, `unsigned long` | no | no | no | same width | no |
 | `long long`, `unsigned long long` | no | no | no | no | no |
-| `bool` | no | no | no | no | no |
-| floating-point types | no | no | no | no | no |
-| enumerations other than `std::byte` | no | no | no | no | no |
-| class types | no | no | no | no | no |
+| `bool`, floating-point types, enumerations other than `std::byte`, class types | no | no | no | no | no |
 
 ## Why `to_utf_view` doesn't have a deduction guide like other views do
 
@@ -1481,6 +1492,11 @@ adjacent_view, and adjacent_transform_view.
 - Remove `to_utf_tag` types and the `to_utf_view` deduction guide; add discussion section
   on dropping support for CTAD per
   `elements_view`/`adjacent_view`/`adjacent_transform_view` precedent.
+- Specify the code unit adaptor conversion rule in terms of width rather than `sizeof`,
+  and split its table into one reflecting only the standard's guarantees and one
+  reflecting typical platforms, per SG16 feedback.
+- Size the exposition-only transcoding buffer by target type rather than by
+  `sizeof(ToType)`.
 
 ## Changes since R14
 

tests/beman/utf_view/code_unit_view.test.cpp

@@ -17,6 +17,7 @@ import std;
 #include <cstddef>
 #include <cstdint>
 #include <ranges>
+#include <limits>
 #include <span>
 #include <string>
 #include <string_view>
@@ -108,6 +109,22 @@ static_assert(
 
 // Conversion rule: a source type is accepted if it is the target type, or if it
 // is std::byte or a non-bool integral type of the same width as the target.
+// Width counts value and sign bits and excludes padding, so signed and
+// unsigned types of the same size have the same width.
+constexpr int char_bit{std::numeric_limits<unsigned char>::digits};
+static_assert(detail::code_unit_width<char> == char_bit);
+static_assert(detail::code_unit_width<signed char> == char_bit);
+static_assert(detail::code_unit_width<unsigned char> == char_bit);
+static_assert(detail::code_unit_width<std::byte> == char_bit);
+static_assert(detail::code_unit_width<char8_t> == char_bit);
+static_assert(detail::code_unit_width<short> ==
+              detail::code_unit_width<unsigned short>);
+static_assert(detail::code_unit_width<int> == detail::code_unit_width<unsigned>);
+static_assert(detail::code_unit_width<char16_t> >= 16);
+static_assert(detail::code_unit_width<char32_t> >= 32);
+static_assert(detail::code_unit_width<wchar_t> ==
+              std::numeric_limits<wchar_t>::digits +
+                  std::numeric_limits<wchar_t>::is_signed);
 template <class R, auto const& Adaptor>
 constexpr bool adaptable = requires { std::declval<R>() | Adaptor; };
 
@@ -153,7 +170,7 @@ static_assert(adaptable<std::span<short const>, as_char16_t>);
 static_assert(adaptable<std::span<unsigned short const>, as_char16_t>);
 static_assert(adaptable<std::span<std::uint16_t const>, as_char16_t>);
 static_assert(adaptable<std::wstring_view, as_char16_t> ==
-              (sizeof(wchar_t) == sizeof(char16_t)));
+              (detail::code_unit_width<wchar_t> == detail::code_unit_width<char16_t>));
 static_assert(!adaptable<std::u8string_view, as_char16_t>);
 static_assert(!adaptable<std::u32string_view, as_char16_t>);
 static_assert(!adaptable<std::string_view, as_char16_t>);
@@ -164,13 +181,13 @@ static_assert(adaptable<std::u32string_view, as_char32_t>);
 static_assert(adaptable<std::span<std::uint32_t const>, as_char32_t>);
 static_assert(adaptable<std::span<std::int32_t const>, as_char32_t>);
 static_assert(adaptable<std::span<int const>, as_char32_t> ==
-              (sizeof(int) == sizeof(char32_t)));
+              (detail::code_unit_width<int> == detail::code_unit_width<char32_t>));
 static_assert(adaptable<std::span<unsigned const>, as_char32_t> ==
-              (sizeof(unsigned) == sizeof(char32_t)));
+              (detail::code_unit_width<unsigned> == detail::code_unit_width<char32_t>));
 static_assert(adaptable<std::span<long const>, as_char32_t> ==
-              (sizeof(long) == sizeof(char32_t)));
+              (detail::code_unit_width<long> == detail::code_unit_width<char32_t>));
 static_assert(adaptable<std::wstring_view, as_char32_t> ==
-              (sizeof(wchar_t) == sizeof(char32_t)));
+              (detail::code_unit_width<wchar_t> == detail::code_unit_width<char32_t>));
 static_assert(!adaptable<std::u8string_view, as_char32_t>);
 static_assert(!adaptable<std::u16string_view, as_char32_t>);
 static_assert(!adaptable<std::span<long long const>, as_char32_t>);
@@ -179,11 +196,11 @@ static_assert(!adaptable<std::span<double const>, as_char32_t>);
 // as_wchar_t
 static_assert(adaptable<std::wstring_view, as_wchar_t>);
 static_assert(adaptable<std::u16string_view, as_wchar_t> ==
-              (sizeof(wchar_t) == sizeof(char16_t)));
+              (detail::code_unit_width<wchar_t> == detail::code_unit_width<char16_t>));
 static_assert(adaptable<std::u32string_view, as_wchar_t> ==
-              (sizeof(wchar_t) == sizeof(char32_t)));
+              (detail::code_unit_width<wchar_t> == detail::code_unit_width<char32_t>));
 static_assert(adaptable<std::span<unsigned short const>, as_wchar_t> ==
-              (sizeof(wchar_t) == sizeof(unsigned short)));
+              (detail::code_unit_width<wchar_t> == detail::code_unit_width<unsigned short>));
 static_assert(!adaptable<std::u8string_view, as_wchar_t>);
 static_assert(!adaptable<std::string_view, as_wchar_t>);
 
@@ -239,7 +256,7 @@ constexpr bool signed_source_test() {
 }
 
 constexpr bool int_source_test() {
-  if constexpr (sizeof(int) == sizeof(char32_t)) {
+  if constexpr (detail::code_unit_width<int> == detail::code_unit_width<char32_t>) {
     std::array<int, 2> const code_points{0x1F574, 0xFFFD};
     auto const as_utf32{code_points | as_char32_t |
                         std::ranges::to<std::u32string>()};