Switch to using blockcache with size 4MB by default for h5netcdf (#11216)

Changes the  default cache type when passing the path as a string to `open_dataset` and `open_datatree` with the `h5netcdf` engine. This represents a significant improvement when opening optimized hdf5 files. This idea is based off the work that @betolink has been doing in earthaccess (earthaccess-dev/earthaccess#1061)

Author: jsignell

doc/whats-new.rst

@@ -27,6 +27,10 @@ Breaking Changes
   :py:meth:`open_dataset`. This finalizes the deprecation cycle initiated in
   xarray version 2025.01.2 (:pull:`11173`). By `Spencer Clark
   <https://github.com/spencerkclark>`_.
+- When using ``h5netcdf`` engine and passing the path as a string to
+  ``open_dataset`` and ``open_datatree`` the default behavior of fsspec is now to
+  use block caching with a 4MB block size (:pull:`11216`). By `Julia Signell
+  <https://github.com/jsignell>`_.
 
 Deprecations
 ~~~~~~~~~~~~

xarray/backends/common.py

@@ -214,13 +214,16 @@ def getbuffer(self) -> memoryview:
         return self.getvalue()
 
 
-def _open_remote_file(file, mode, storage_options=None):
+def _open_remote_file(file, mode, storage_options=None, open_kwargs=None):
     import fsspec
 
     fs, _, paths = fsspec.get_fs_token_paths(
         file, mode=mode, storage_options=storage_options
     )
-    return fs.open(paths[0], mode=mode)
+
+    open_kwargs = open_kwargs or {}
+
+    return fs.open(paths[0], mode=mode, **open_kwargs)
 
 
 def _encode_variable_name(name):

xarray/backends/h5netcdf_.py

@@ -183,13 +183,29 @@ def open(
         driver=None,
         driver_kwds=None,
         storage_options: dict[str, Any] | None = None,
+        open_kwargs: dict[str, Any] | None = None,
     ):
         import h5netcdf
 
         if isinstance(filename, str) and is_remote_uri(filename) and driver is None:
             mode_ = "rb" if mode == "r" else mode
+
+            open_kwargs = open_kwargs or {}
+
+            # Use blockcache with size 4MB by default
+            if "cache_type" not in open_kwargs:
+                open_kwargs["cache_type"] = "blockcache"
+            if (
+                open_kwargs["cache_type"] == "blockcache"
+                and "block_size" not in open_kwargs
+            ):
+                open_kwargs["block_size"] = 4 * 1024 * 1024
+
             filename = _open_remote_file(
-                filename, mode=mode_, storage_options=storage_options
+                filename,
+                mode=mode_,
+                storage_options=storage_options,
+                open_kwargs=open_kwargs,
             )
 
         if isinstance(filename, BytesIOProxy):
@@ -531,6 +547,7 @@ def open_dataset(
         driver=None,
         driver_kwds=None,
         storage_options: dict[str, Any] | None = None,
+        open_kwargs: dict[str, Any] | None = None,
     ) -> Dataset:
         # Keep this message for some versions
         # remove and set phony_dims="access" above
@@ -548,6 +565,7 @@ def open_dataset(
             driver=driver,
             driver_kwds=driver_kwds,
             storage_options=storage_options,
+            open_kwargs=open_kwargs,
         )
 
         store_entrypoint = StoreBackendEntrypoint()
@@ -633,6 +651,8 @@ def open_groups_as_dict(
         decode_vlen_strings=True,
         driver=None,
         driver_kwds=None,
+        storage_options: dict[str, Any] | None = None,
+        open_kwargs: dict[str, Any] | None = None,
         **kwargs,
     ) -> dict[str, Dataset]:
         from xarray.backends.common import _iter_nc_groups
@@ -654,6 +674,8 @@ def open_groups_as_dict(
             decode_vlen_strings=decode_vlen_strings,
             driver=driver,
             driver_kwds=driver_kwds,
+            storage_options=storage_options,
+            open_kwargs=open_kwargs,
         )
 
         # Check for a group and make it a parent if it exists

xarray/tests/test_backends.py

@@ -44,7 +44,7 @@
     open_mfdataset,
     save_mfdataset,
 )
-from xarray.backends.common import robust_getitem
+from xarray.backends.common import _open_remote_file, robust_getitem
 from xarray.backends.h5netcdf_ import H5netcdfBackendEntrypoint
 from xarray.backends.netcdf3 import _nc3_dtype_coercions
 from xarray.backends.netCDF4_ import (
@@ -5298,6 +5298,62 @@ def test_write_inconsistent_chunks(self) -> None:
             assert actual["y"].encoding["chunksizes"] == (100, 50)
 
 
+@requires_h5netcdf
+@requires_fsspec
+@pytest.mark.parametrize(
+    "open_kwargs, expected_cache_type, expected_block_size",
+    [
+        # Default: blockcache with 4MB block size
+        (None, "blockcache", 4 * 1024 * 1024),
+        ({}, "blockcache", 4 * 1024 * 1024),
+        # Custom block_size still uses blockcache
+        ({"block_size": 8 * 1024 * 1024}, "blockcache", 8 * 1024 * 1024),
+        # Explicit blockcache with default block_size
+        ({"cache_type": "blockcache"}, "blockcache", 4 * 1024 * 1024),
+        # Custom cache_type: no block_size default injected
+        ({"cache_type": "readahead"}, "readahead", None),
+    ],
+    ids=["default", "empty-dict", "8mb", "blockcache", "readahead"],
+)
+def test_h5netcdf_open_kwargs(
+    open_kwargs, expected_cache_type, expected_block_size
+) -> None:
+    """Test that open_kwargs are forwarded to the remote file opener."""
+    expected = create_test_data()
+    with create_tmp_file() as tmp_file:
+        expected.to_netcdf(tmp_file, engine="h5netcdf")
+
+        captured = {}
+
+        def capturing_open_remote_file(
+            file, mode, storage_options=None, open_kwargs=None
+        ):
+            captured["open_kwargs"] = open_kwargs
+            return _open_remote_file(
+                file,
+                mode=mode,
+                storage_options=storage_options,
+                open_kwargs=open_kwargs,
+            )
+
+        with patch(
+            "xarray.backends.h5netcdf_._open_remote_file",
+            side_effect=capturing_open_remote_file,
+        ):
+            # Use a file:// URI so is_remote_uri returns True and _open_remote_file is called
+            file_uri = f"file://{tmp_file}"
+            with open_dataset(
+                file_uri, engine="h5netcdf", open_kwargs=open_kwargs
+            ) as actual:
+                assert_identical(actual, expected)
+
+        assert captured["open_kwargs"]["cache_type"] == expected_cache_type
+        if expected_block_size is None:
+            assert "block_size" not in captured["open_kwargs"]
+        else:
+            assert captured["open_kwargs"]["block_size"] == expected_block_size
+
+
 @requires_netCDF4
 @requires_h5netcdf
 def test_memoryview_write_h5netcdf_read_netcdf4() -> None: