[VARIANT] Add support for the json_to_variant API

parquet-variant/Cargo.toml

@@ -37,7 +37,7 @@ arrow-schema = { workspace = true }
 chrono = { workspace = true }
 # "arbitrary_precision" allows us to manually parse numbers. "preserve_order" does not automatically
 # sort object keys
-serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] }
+serde_json = { version = "1.0" }
 rust_decimal = "1.37.2"
 base64 = "0.22"
 

parquet-variant/src/from_json.rs

@@ -89,36 +89,15 @@ fn variant_from_number<'a, 'b>(n: &Number) -> Result<Variant<'a, 'b>, ArrowError
             Ok(i.into())
         }
     } else {
-        // Try decimal
-        // TODO: Replace with custom decimal parsing as the rust_decimal library only supports
-        // a max unscaled value of 2^96.
-        match Decimal::from_str_exact(n.as_str()) {
-            Ok(dec) => {
-                let unscaled: i128 = dec.mantissa();
-                let scale = dec.scale() as u8;
-                if unscaled.abs() <= VariantDecimal4::MAX_UNSCALED_VALUE as i128
-                    && scale <= VariantDecimal4::MAX_PRECISION as u8
-                {
-                    (unscaled as i32, scale).try_into()
-                } else if unscaled.abs() <= VariantDecimal8::MAX_UNSCALED_VALUE as i128
-                    && scale <= VariantDecimal8::MAX_PRECISION as u8
-                {
-                    (unscaled as i64, scale).try_into()
-                } else {
-                    (unscaled, scale).try_into()
-                }
-            }
-            Err(_) => {
-                // Try double
-                match n.as_f64() {
-                    Some(f) => return Ok(f.into()),
-                    None => Err(ArrowError::InvalidArgumentError(format!(
-                        "Failed to parse {} as number",
-                        n.as_str()
-                    ))),
-                }?
-            }
-        }
+        // Todo: Try decimal once we implement custom JSON parsing where we have access to strings
+        // Try double - currently json_to_variant does not produce decimal
+        match n.as_f64() {
+            Some(f) => return Ok(f.into()),
+            None => Err(ArrowError::InvalidArgumentError(format!(
+                "Failed to parse {} as number",
+                n.to_string()
+            ))),
+        }?
     }
 }
 

parquet-variant/src/to_json.rs

@@ -18,14 +18,10 @@
 //! Module for converting Variant data to JSON format
 use arrow_schema::ArrowError;
 use base64::{engine::general_purpose, Engine as _};
-use rust_decimal::prelude::*;
-use serde_json::{Number, Value};
+use serde_json::Value;
 use std::io::Write;
 
-use crate::{
-    variant::{Variant, VariantList, VariantObject},
-    VariantDecimal,
-};
+use crate::variant::{Variant, VariantList, VariantObject};
 
 // Format string constants to avoid duplication and reduce errors
 const DATE_FORMAT: &str = "%Y-%m-%d";
@@ -249,27 +245,6 @@ pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError> {
         .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8 conversion error: {e}")))
 }
 
-fn variant_decimal_to_json_value(decimal: &impl VariantDecimal) -> Result<Value, ArrowError> {
-    let integer = decimal.integer();
-    let scale = decimal.scale();
-    if scale == 0 {
-        return Ok(Value::from(integer));
-    }
-    let divisor = 10_i128.pow(scale);
-    if integer % divisor != 0 {
-        // try decimal
-        if let Ok(dec) = Decimal::try_from_i128_with_scale(integer, scale) {
-            if let Ok(value) = serde_json::from_str::<Number>(&dec.to_string()) {
-                return Ok(serde_json::Value::Number(value));
-            }
-        }
-        // fall back to floating point
-        return Ok(Value::from(integer as f64 / divisor as f64));
-    }
-    // integer perfect division
-    Ok(Value::from(integer / divisor))
-}
-
 /// Convert [`Variant`] to [`serde_json::Value`]
 ///
 /// This function converts a Variant to a [`serde_json::Value`], which is useful
@@ -311,9 +286,60 @@ pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
         Variant::Double(f) => serde_json::Number::from_f64(*f)
             .map(Value::Number)
             .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid double value".to_string())),
-        Variant::Decimal4(decimal4) => variant_decimal_to_json_value(decimal4),
-        Variant::Decimal8(decimal8) => variant_decimal_to_json_value(decimal8),
-        Variant::Decimal16(decimal16) => variant_decimal_to_json_value(decimal16),
+        Variant::Decimal4(decimal4) => {
+            let scale = decimal4.scale();
+            let integer = decimal4.integer();
+
+            let integer = if scale == 0 {
+                integer
+            } else {
+                let divisor = 10_i32.pow(scale as u32);
+                if integer % divisor != 0 {
+                    // fall back to floating point
+                    return Ok(Value::from(integer as f64 / divisor as f64));
+                }
+                integer / divisor
+            };
+            Ok(Value::from(integer))
+        }
+        Variant::Decimal8(decimal8) => {
+            let scale = decimal8.scale();
+            let integer = decimal8.integer();
+
+            let integer = if scale == 0 {
+                integer
+            } else {
+                let divisor = 10_i64.pow(scale as u32);
+                if integer % divisor != 0 {
+                    // fall back to floating point
+                    return Ok(Value::from(integer as f64 / divisor as f64));
+                }
+                integer / divisor
+            };
+            Ok(Value::from(integer))
+        }
+        Variant::Decimal16(decimal16) => {
+            let scale = decimal16.scale();
+            let integer = decimal16.integer();
+
+            let integer = if scale == 0 {
+                integer
+            } else {
+                let divisor = 10_i128.pow(scale as u32);
+                if integer % divisor != 0 {
+                    // fall back to floating point
+                    return Ok(Value::from(integer as f64 / divisor as f64));
+                }
+                integer / divisor
+            };
+            // i128 has higher precision than any 64-bit type. Try a lossless narrowing cast to
+            // i64 or u64 first, falling back to a lossy narrowing cast to f64 if necessary.
+            let value = i64::try_from(integer)
+                .map(Value::from)
+                .or_else(|_| u64::try_from(integer).map(Value::from))
+                .unwrap_or_else(|_| Value::from(integer as f64));
+            Ok(value)
+        }
         Variant::Date(date) => Ok(Value::String(format_date_string(date))),
         Variant::TimestampMicros(ts) => Ok(Value::String(ts.to_rfc3339())),
         Variant::TimestampNtzMicros(ts) => Ok(Value::String(format_timestamp_ntz_string(ts))),

parquet-variant/src/variant.rs

@@ -16,7 +16,7 @@ use std::ops::Deref;
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-pub use self::decimal::{VariantDecimal, VariantDecimal16, VariantDecimal4, VariantDecimal8};
+pub use self::decimal::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
 pub use self::list::VariantList;
 pub use self::metadata::VariantMetadata;
 pub use self::object::VariantObject;

parquet-variant/src/variant/decimal.rs

@@ -40,12 +40,6 @@ macro_rules! format_decimal {
     }};
 }
 
-// Common trait to classify VariantDecimalXXX types
-pub trait VariantDecimal {
-    fn integer(&self) -> i128;
-    fn scale(&self) -> u32;
-}
-
 /// Represents a 4-byte decimal value in the Variant format.
 ///
 /// This struct stores a decimal number using a 32-bit signed integer for the coefficient
@@ -107,16 +101,6 @@ impl VariantDecimal4 {
     }
 }
 
-impl VariantDecimal for VariantDecimal4 {
-    fn integer(&self) -> i128 {
-        self.integer() as i128
-    }
-
-    fn scale(&self) -> u32 {
-        self.scale() as u32
-    }
-}
-
 impl fmt::Display for VariantDecimal4 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         format_decimal!(f, self.integer, self.scale, i32)
@@ -185,16 +169,6 @@ impl VariantDecimal8 {
     }
 }
 
-impl VariantDecimal for VariantDecimal8 {
-    fn integer(&self) -> i128 {
-        self.integer() as i128
-    }
-
-    fn scale(&self) -> u32 {
-        self.scale() as u32
-    }
-}
-
 impl fmt::Display for VariantDecimal8 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         format_decimal!(f, self.integer, self.scale, i64)
@@ -263,16 +237,6 @@ impl VariantDecimal16 {
     }
 }
 
-impl VariantDecimal for VariantDecimal16 {
-    fn integer(&self) -> i128 {
-        self.integer()
-    }
-
-    fn scale(&self) -> u32 {
-        self.scale() as u32
-    }
-}
-
 impl fmt::Display for VariantDecimal16 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         format_decimal!(f, self.integer, self.scale, i128)