feat: writing/reading to parquet

.github/workflows/pre-commit.yml

@@ -29,6 +29,7 @@ jobs:
             --preset ci-build \
             -DENABLE_JULIA=ON \
             -DENABLE_ARROW=ON \
+            -DENABLE_PARQUET=ON \
             -DUSE_EXTERNAL_CATCH2=OFF
           ln -s build/compile_commands.json compile_commands.json
           echo "::endgroup::"

.github/workflows/test.yml

@@ -38,6 +38,7 @@ jobs:
           cmake --preset ci-build \
             -DENABLE_JULIA=ON \
             -DENABLE_ARROW=ON \
+            -DENABLE_PARQUET=$([[ ${{ matrix.LCG }} == dev* ]] && echo "ON" || echo "OFF") \
             -DENABLE_RNTUPLE=$([[ ${{ matrix.LCG }} == LCG_104/* ]] && echo "OFF" || echo "ON") \
             -DPODIO_RUN_STRACE_TEST=$([[ ${{ matrix.LCG }} == LCG_104/* ]] && echo "OFF" || echo "ON") \
             -DCMAKE_INSTALL_PREFIX=$(pwd)/install \

.gitignore

@@ -22,6 +22,8 @@ tests/unittests/Manifest.toml
 *.root
 *.dat
 *.sio
+*.parquet
+*.podio_arrow
 
 # Spack build folders
 spack*

CMakeLists.txt

@@ -68,6 +68,7 @@ ADD_CLANG_TIDY()
 option(CREATE_DOC        "Whether or not to create doxygen doc target." OFF)
 option(ENABLE_SIO        "Build SIO I/O support" OFF)
 option(ENABLE_ARROW      "Build Arrow I/O support" OFF)
+option(ENABLE_PARQUET    "Build Parquet support" OFF)
 option(PODIO_RELAX_PYVER "Do not require exact python version match with ROOT" OFF)
 option(ENABLE_RNTUPLE    "Build with support for the new ROOT NTtuple format" OFF)
 option(ENABLE_DATASOURCE "Build podio's ROOT DataSource" OFF)
@@ -147,6 +148,10 @@ if(ENABLE_SIO)
   endif()
 endif()
 
+if(ENABLE_PARQUET AND NOT ENABLE_ARROW)
+  message(FATAL_ERROR "ENABLE_PARQUET requires ENABLE_ARROW to be ON")
+endif()
+
 # optionally build with Arrow -----------------------------------------------
 if(ENABLE_ARROW)
   set(lz4Alt_FIND_QUIETLY TRUE)
@@ -159,7 +164,19 @@ if(ENABLE_ARROW)
     message(FATAL_ERROR "Found Arrow, but no usable Arrow CMake target was exported")
   endif()
 
-  message(STATUS "Found Arrow library - will build Arrow I/O support")
+  if(ENABLE_PARQUET)
+    find_package(nlohmann_json 3.10 REQUIRED)
+    find_package(Parquet REQUIRED)
+    if(TARGET Parquet::parquet_shared)
+      set(PODIO_PARQUET_TARGET Parquet::parquet_shared)
+    else()
+      set(PODIO_PARQUET_TARGET Parquet::parquet_static)
+    endif()
+    message(STATUS "Found Arrow and Parquet libraries - will build Arrow I/O support")
+  else()
+    message(STATUS "Found Arrow library - will build Arrow I/O support without Parquet")
+  endif()
+
   list(APPEND PODIO_IO_HANDLERS ARROW)
 endif()
 

cmake/podioConfig.cmake.in

@@ -47,9 +47,14 @@ if(PODIO_ENABLE_SIO)
 endif()
 
 SET(PODIO_ENABLE_ARROW @ENABLE_ARROW@)
+SET(PODIO_ENABLE_PARQUET @ENABLE_PARQUET@)
 if(PODIO_ENABLE_ARROW)
   set(lz4Alt_FIND_QUIETLY TRUE)
   find_dependency(Arrow)
+  if(PODIO_ENABLE_PARQUET)
+    find_dependency(nlohmann_json)
+    find_dependency(Parquet)
+  endif()
   set(PODIO_ARROW_TARGET @PODIO_ARROW_TARGET@)
 endif()
 

cmake/podioMacros.cmake

@@ -362,7 +362,7 @@ endif()
   list(FILTER SOURCES INCLUDE REGEX .*ArrowMapper.cc)
 
   add_library(${CORE_LIB}PodioArrow SHARED ${SOURCES})
-  target_link_libraries(${CORE_LIB}PodioArrow PUBLIC ${CORE_LIB} podio::podio ${PODIO_ARROW_TARGET})
+  target_link_libraries(${CORE_LIB}PodioArrow PUBLIC ${CORE_LIB} podio::podio podio::podioArrow)
   target_include_directories(${CORE_LIB}PodioArrow PUBLIC
     $<BUILD_INTERFACE:${ARG_OUTPUT_FOLDER}>
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)

doc/reading_writing.md

@@ -1,12 +1,12 @@
-# ROOT Support in Podio
+# Reading and writing podio files
 
-Podio supports ROOT file I/O through multiple backends in both C++ and Python
-for ROOT TTrees and ROOT RNTuples. Below are the recommended approaches for
-reading and writing these files, as well as additional notes to guide usage.
+Podio supports file I/O through multiple backends in both C++ and Python. Below
+are the recommended approaches for reading and writing these files, as well as
+additional notes to guide usage.
 
 ## C++
 
-Podio provides generic and format-specific I/O interfaces for ROOT files.
+Podio provides generic and format-specific I/O interfaces.
 
 ### Reading
 
@@ -55,8 +55,9 @@ By default, TTrees are written. To explicitly select an output backend, provide
 the type:
 
 ```cpp
-auto ttreeWriter = podio::makeWriter(filename, "root");       // Use TTree
-auto rntupleWriter = podio::makeWriter(filename, "rntuple");  // Use RNTuple
+auto ttreeWriter = podio::makeWriter(filename, "root");         // Use TTree
+auto rntupleWriter = podio::makeWriter(filename, "rntuple");    // Use RNTuple
+auto parquetWriter = podio::makeWriter(filename, "parquet");    // Use Arrow/Parquet
 ```
 
 The format can also be set by the environment variable `PODIO_DEFAULT_WRITE_RNTUPLE`. If
@@ -66,18 +67,58 @@ the environment variable is set **to a non-empty string**, RNTuples will be the
 - `.root`: Uses the default backend (TTree or RNTuple if `PODIO_DEFAULT_WRITE_RNTUPLE`
   is set to a non-empty string), unless specified.
 - `.sio`: Uses the SIO writer.
+- `.podio_parquet`: Uses the Arrow/Parquet writer.
 - Other extensions are not allowed.
 
 Specific writers for each backend are also available:
 
 ```cpp
 #include <podio/ROOTWriter.h>     // For TTree output
 #include <podio/RNTupleWriter.h>  // For RNTuple output
+#include <podio/ArrowWriter.h>    // For Arrow/Parquet output
 
-podio::ROOTWriter writer(filename);           // For TTree output
-podio::RNTupleWriter rntupleWriter(filename); // For RNTuple output
+podio::ROOTWriter writer(filename);              // For TTree output
+podio::RNTupleWriter rntupleWriter(filename);    // For RNTuple output
+podio::ArrowWriter parquetWriter(directoryName); // For Arrow/Parquet output
 ```
 
+### Arrow/Parquet I/O
+
+The Arrow/Parquet backend writes a directory with the `.podio_parquet`
+extension. The directory contains one Parquet file per category and a
+`metadata.json` file with the podio metadata needed for reading.
+
+```cpp
+#include <podio/Reader.h>
+#include <podio/Writer.h>
+
+auto writer = podio::makeWriter("events.podio_parquet", "parquet");
+writer.writeFrame(frame, podio::Category::Event);
+writer.finish();
+
+auto reader = podio::makeReader("events.podio_parquet");
+auto event = reader.readEvent(0);
+```
+
+The backend-specific classes can also be used directly:
+
+```cpp
+#include <podio/ArrowReader.h>
+#include <podio/ArrowWriter.h>
+
+podio::ArrowWriter writer("events.podio_parquet");
+writer.writeFrame(frame, podio::Category::Event);
+writer.finish();
+
+podio::ArrowReader reader;
+reader.openFile("events.podio_parquet");
+auto event = podio::Frame(reader.readEntry(podio::Category::Event, 0));
+```
+
+The default compression for Arrow/Parquet output is controlled at configure
+time with the `PODIO_ARROW_DEFAULT_COMPRESSION` CMake option. Supported values
+are `UNCOMPRESSED`, `SNAPPY`, and `ZSTD`.
+
 ```{note}
 Note that the generic readers and writers have methods that are not available in
 the backend-specific classes. For example, the generic reader has a
@@ -111,9 +152,11 @@ Alternatively, instantiate backend-specific readers explicitly:
 ```python
 from podio.root_io import Reader         # For TTrees
 from podio.root_io import RNTupleReader  # For RNTuples
+from podio.arrow_io import Reader as ArrowReader
 
 reader = Reader(filename)                # For TTree files (.root)
 rntuple_reader = RNTupleReader(filename) # For RNTuple files (.root)
+arrow_reader = ArrowReader(directory)    # For Arrow/Parquet directories (.podio_parquet)
 ```
 
 ### Writing
@@ -123,9 +166,12 @@ Similarly, use the appropriate writer class for the file format:
 ```python
 from podio.root_io import Writer         # For TTrees
 from podio.root_io import RNTupleWriter  # For RNTuples
+from podio.arrow_io import Writer as ArrowWriter
 
 writer = Writer(filename)                # For TTree output (.root)
 rntuple_writer = RNTupleWriter(filename) # For RNTuple output (.root)
+arrow_writer = ArrowWriter(directory)    # For Arrow/Parquet output (.podio_parquet)
 
 writer.write_frame(frame, category)
+arrow_writer.write_frame(frame, category)
 ```

doc/storage_details.md

@@ -104,3 +104,24 @@ this record.
 Schematically an SIO file written by podio looks like this
 
 <img src="figures/file_layout_sio.svg" alt="SIO file layout schematic" width=167.75px align=center>
+
+## Arrow/Parquet
+
+The Arrow/Parquet backend stores a podio dataset as a directory, usually using
+the `.podio_parquet` extension. The directory contains one Parquet file per
+category and one `metadata.json` file with dataset-level metadata.
+
+For a category named `events`, the category data is stored in `events.parquet`.
+Each Frame in that category corresponds to one row in the Parquet file. Each
+collection is stored as one Arrow column, and Frame parameters are stored in a
+special `frame_parameters` column.
+
+The `metadata.json` file records the podio format marker, the podio version,
+the available categories, the category file names, the number of entries, and
+the datamodel definitions needed to read the stored collections.
+
+```{note}
+For Arrow/Parquet output all entries of a category have to have the same
+collection contents. This content is defined by the first entry that is written
+for a category.
+```

include/podio/ArrowReader.h

@@ -0,0 +1,70 @@
+#ifndef PODIO_ARROWREADER_H
+#define PODIO_ARROWREADER_H
+
+#include "podio/podioVersion.h"
+#include "podio/utilities/ArrowFrameData.h"
+#include "podio/utilities/ReaderCommon.h"
+
+#include <cstddef>
+#include <filesystem>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace arrow {
+class Table;
+}
+
+namespace podio {
+
+/// Arrow backend reader for PODIO
+///
+/// Reads data from a directory structure containing one Parquet file per category
+/// and a metadata.json file containing metadata for reading.
+class ArrowReader : public ReaderCommon {
+public:
+  /// Create an ArrowReader
+  ArrowReader();
+
+  /// Open the passed directory for reading.
+  ///
+  /// @param directory The path to the directory to read from
+  void openFile(const std::string& directory);
+
+  ~ArrowReader() = default;
+
+  ArrowReader(const ArrowReader&) = delete;
+  ArrowReader& operator=(const ArrowReader&) = delete;
+  ArrowReader(ArrowReader&&) = delete;
+  ArrowReader& operator=(ArrowReader&&) = delete;
+
+  /// Read the next entry for the given category
+  std::unique_ptr<podio::ArrowFrameData> readNextEntry(std::string_view name,
+                                                       const std::vector<std::string>& collsToRead = {});
+
+  /// Read the specific entry for the given category
+  std::unique_ptr<podio::ArrowFrameData> readEntry(std::string_view name, size_t index,
+                                                   const std::vector<std::string>& collsToRead = {});
+
+  /// Get the number of entries for a category
+  size_t getEntries(std::string_view name) const;
+
+private:
+  struct CategoryInfo {
+    std::string filePath{};
+    size_t entries = 0;
+    size_t currentIndex = 0;
+    std::shared_ptr<arrow::Table> table{nullptr};
+  };
+
+  void loadCategoryTable(CategoryInfo& catInfo);
+
+  std::string m_directory{};
+  std::map<std::string, CategoryInfo> m_categories{};
+};
+
+} // namespace podio
+
+#endif // PODIO_ARROWREADER_H