[VARIANT] Add support for the json_to_variant API

parquet-variant/Cargo.toml

@@ -35,7 +35,10 @@ rust-version = "1.83"
 [dependencies]
 arrow-schema = { workspace = true }
 chrono = { workspace = true }
-serde_json = "1.0"
+# "arbitrary_precision" allows us to manually parse numbers. "preserve_order" does not automatically
+# sort object keys
+serde_json = { version = "1.0", features = ["arbitrary_precision", "preserve_order"] }
+rust_decimal = "1.37.2"
 base64 = "0.21"
 
 [lib]

parquet-variant/examples/variant_from_json_examples.rs

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Example showing how to convert Variant values to JSON
+
+use parquet_variant::{
+    json_to_variant, variant_to_json, variant_to_json_string, variant_to_json_value,
+    SampleBoxBasedVariantBufferManager, SampleVecBasedVariantBufferManager, VariantBufferManager,
+};
+
+fn from_json_example<T: VariantBufferManager>(
+    variant_buffer_manager: &mut T,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let person_string = "{\"name\":\"Alice\", \"age\":30, ".to_string()
+        + "\"email\":\"alice@example.com\", \"is_active\": true, \"score\": 95.7,"
+        + "\"additional_info\": null}";
+    let (metadata_size, value_size) = json_to_variant(&person_string, variant_buffer_manager)?;
+
+    let variant = parquet_variant::Variant::try_new(
+        &variant_buffer_manager.get_immutable_metadata_buffer()[..metadata_size],
+        &variant_buffer_manager.get_immutable_value_buffer()[..value_size],
+    )?;
+
+    let json_string = variant_to_json_string(&variant)?;
+    let json_value = variant_to_json_value(&variant)?;
+    let pretty_json = serde_json::to_string_pretty(&json_value)?;
+    println!("{}", pretty_json);
+
+    let mut buffer = Vec::new();
+    variant_to_json(&mut buffer, &variant)?;
+    let buffer_result = String::from_utf8(buffer)?;
+
+    // Verify all methods produce the same result
+    assert_eq!(json_string, buffer_result);
+    assert_eq!(json_string, serde_json::to_string(&json_value)?);
+
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // The caller must provide an object implementing the `VariantBufferManager` trait to the library.
+    // This allows the library to write the constructed variant to buffers provided by the caller.
+    // This way, the caller has direct control over the output buffers.
+    let mut box_based_buffer_manager = SampleBoxBasedVariantBufferManager {
+        value_buffer: vec![0u8; 1].into_boxed_slice(),
+        metadata_buffer: vec![0u8; 1].into_boxed_slice(),
+    };
+
+    let mut vec_based_buffer_manager = SampleVecBasedVariantBufferManager {
+        value_buffer: vec![0u8; 1],
+        metadata_buffer: vec![0u8; 1],
+    };
+
+    from_json_example(&mut box_based_buffer_manager)?;
+    from_json_example(&mut vec_based_buffer_manager)?;
+    Ok(())
+}

parquet-variant/src/decoder.rs

@@ -26,6 +26,11 @@ use std::num::TryFromIntError;
 // Makes the code a bit more readable
 pub(crate) const VARIANT_VALUE_HEADER_BYTES: usize = 1;
 
+pub(crate) const MAX_UNSCALED_DECIMAL_4: i32 = 999999999;
+pub(crate) const MAX_PRECISION_DECIMAL_4: u8 = 9;
+pub(crate) const MAX_UNSCALED_DECIMAL_8: i64 = 999999999999999999i64;
+pub(crate) const MAX_PRECISION_DECIMAL_8: u8 = 18;
+
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum VariantBasicType {
     Primitive = 0,

parquet-variant/src/from_json.rs

@@ -0,0 +1,102 @@
+use crate::variant_buffer_manager::VariantBufferManager;
+use crate::{ListBuilder, ObjectBuilder, Variant, VariantBuilder};
+use arrow_schema::ArrowError;
+use serde_json::{Map, Value};
+
+/// Eventually, internal writes should also be performed using VariantBufferManager instead of
+/// ValueBuffer and MetadataBuffer so the caller has control of the memory.
+/// Returns a pair <value_size, metadata_size>
+pub fn json_to_variant<T: VariantBufferManager>(
+    json: &str,
+    variant_buffer_manager: &mut T,
+) -> Result<(usize, usize), ArrowError> {
+    let mut builder = VariantBuilder::new();
+    let json: Value = serde_json::from_str(json)
+        .map_err(|e| ArrowError::InvalidArgumentError(format!("JSON format error: {}", e)))?;
+
+    build_json(&json, &mut builder)?;
+    let (metadata, value) = builder.finish();
+    let value_size = value.len();
+    let metadata_size = metadata.len();
+
+    // Write to caller's buffers - Remove this when the library internally writes to the caller's
+    // buffers anyway
+    variant_buffer_manager.ensure_metadata_buffer_size(metadata_size)?;
+    variant_buffer_manager.ensure_value_buffer_size(value_size)?;
+
+    let caller_metadata_buffer = variant_buffer_manager.borrow_metadata_buffer();
+    caller_metadata_buffer[..metadata_size].copy_from_slice(metadata.as_slice());
+    let caller_value_buffer = variant_buffer_manager.borrow_value_buffer();
+    caller_value_buffer[..value_size].copy_from_slice(value.as_slice());
+    Ok((metadata_size, value_size))
+}
+
+fn build_json(json: &Value, builder: &mut VariantBuilder) -> Result<(), ArrowError> {
+    match json {
+        Value::Null => builder.append_value(Variant::Null),
+        Value::Bool(b) => builder.append_value(*b),
+        Value::Number(n) => {
+            let v: Variant = n.try_into()?;
+            builder.append_value(v)
+        }
+        Value::String(s) => builder.append_value(s.as_str()),
+        Value::Array(arr) => {
+            let mut list_builder = builder.new_list();
+            build_list(arr, &mut list_builder)?;
+            list_builder.finish();
+        }
+        Value::Object(obj) => {
+            let mut obj_builder = builder.new_object();
+            build_object(obj, &mut obj_builder)?;
+            obj_builder.finish();
+        }
+    };
+    Ok(())
+}
+
+fn build_list(arr: &[Value], builder: &mut ListBuilder) -> Result<(), ArrowError> {
+    for val in arr {
+        match val {
+            Value::Null => builder.append_value(Variant::Null),
+            Value::Bool(b) => builder.append_value(*b),
+            Value::Number(n) => builder.append_value(Variant::try_from(n)?),
+            Value::String(s) => builder.append_value(s.as_str()),
+            Value::Array(arr) => {
+                let mut list_builder = builder.new_list();
+                build_list(arr, &mut list_builder)?;
+                list_builder.finish()
+            }
+            Value::Object(obj) => {
+                let mut obj_builder = builder.new_object();
+                build_object(obj, &mut obj_builder)?;
+                obj_builder.finish();
+            }
+        }
+    }
+    Ok(())
+}
+
+fn build_object<'a, 'b>(
+    obj: &'b Map<String, Value>,
+    builder: &mut ObjectBuilder<'a, 'b>,
+) -> Result<(), ArrowError> {
+    for (key, value) in obj.iter() {
+        match value {
+            Value::Null => builder.insert(key, Variant::Null),
+            Value::Bool(b) => builder.insert(key, *b),
+            Value::Number(n) => builder.insert(key, Variant::try_from(n)?),
+            Value::String(s) => builder.insert(key, s.as_str()),
+            Value::Array(arr) => {
+                let mut list_builder = builder.new_list(key);
+                build_list(arr, &mut list_builder)?;
+                list_builder.finish()
+            }
+            Value::Object(obj) => {
+                let mut obj_builder = builder.new_object(key);
+                build_object(obj, &mut obj_builder)?;
+                obj_builder.finish();
+            }
+        }
+    }
+    Ok(())
+}

parquet-variant/src/lib.rs

@@ -33,10 +33,16 @@ mod decoder;
 mod variant;
 // TODO: dead code removal
 mod builder;
+mod from_json;
 mod to_json;
 #[allow(dead_code)]
 mod utils;
+mod variant_buffer_manager;
 
 pub use builder::*;
+pub use from_json::json_to_variant;
 pub use to_json::{variant_to_json, variant_to_json_string, variant_to_json_value};
 pub use variant::*;
+pub use variant_buffer_manager::{
+    SampleBoxBasedVariantBufferManager, SampleVecBasedVariantBufferManager, VariantBufferManager,
+};

parquet-variant/src/variant.rs

@@ -32,6 +32,8 @@ mod decimal;
 mod list;
 mod metadata;
 mod object;
+use rust_decimal::prelude::*;
+use serde_json::Number;
 
 const MAX_SHORT_STRING_BYTES: usize = 0x3F;
 
@@ -941,6 +943,56 @@ impl From<VariantDecimal16> for Variant<'_, '_> {
     }
 }
 
+impl TryFrom<&Number> for Variant<'_, '_> {
+    type Error = ArrowError;
+
+    fn try_from(n: &Number) -> Result<Self, Self::Error> {
+        if let Some(i) = n.as_i64() {
+            // Find minimum Integer width to fit
+            if i as i8 as i64 == i {
+                Ok((i as i8).into())
+            } else if i as i16 as i64 == i {
+                Ok((i as i16).into())
+            } else if i as i32 as i64 == i {
+                Ok((i as i32).into())
+            } else {
+                Ok(i.into())
+            }
+        } else {
+            // Try decimal
+            // TODO: Replace with custom decimal parsing as the rust_decimal library only supports
+            // a max unscaled value of 2^96.
+            match Decimal::from_str_exact(n.as_str()) {
+                Ok(dec) => {
+                    let unscaled: i128 = dec.mantissa();
+                    let scale = dec.scale() as u8;
+                    if unscaled.abs() <= decoder::MAX_UNSCALED_DECIMAL_4 as i128
+                        && scale <= decoder::MAX_PRECISION_DECIMAL_4
+                    {
+                        (unscaled as i32, scale).try_into()
+                    } else if unscaled.abs() <= decoder::MAX_UNSCALED_DECIMAL_8 as i128
+                        && scale <= decoder::MAX_PRECISION_DECIMAL_8
+                    {
+                        (unscaled as i64, scale).try_into()
+                    } else {
+                        (unscaled, scale).try_into()
+                    }
+                }
+                Err(_) => {
+                    // Try double
+                    match n.as_f64() {
+                        Some(f) => return Ok(f.into()),
+                        None => Err(ArrowError::InvalidArgumentError(format!(
+                            "Failed to parse {} as number",
+                            n.as_str()
+                        ))),
+                    }?
+                }
+            }
+        }
+    }
+}
+
 impl From<f32> for Variant<'_, '_> {
     fn from(value: f32) -> Self {
         Variant::Float(value)