HDF5 error with netcdf4 1.7.3 and h5py on linux with pypi #1438
Description
Our CI broke today-ish with the error below with python 3.12 and 3.13. We do not test with 3.14.
I cannot reproduce the error on windows. The error only appears when h5py is installed and imported.
Limiting the version to "netcdf4<1.7.3" is our current workaround.
A minimal reproducer that leverages xarray is
uv pip install xarray netcdf4==1.7.3 h5py==3.15.0import xarray as xr
import h5py
ds = xr.Dataset({"rootvar": ("x", [100, 200])})
ds.to_netcdf('./test_out.nc', engine='netcdf4', format="NETCDF4")CI error message
dt = <xarray.DataTree 'first'>
Group: /
│ Attributes:
│ elicit_md: b'{"gate":{"gate_1":{"data":0.1,"time":176... uuid: 0d796dfb-60ef-42bc-9a7c-0f8aa5ca4e99
uuid_root: ded3e81e-260f-47c5-b860-a0e233cc93fd
filepath = '/tmp/pytest-of-root/pytest-0/test_simple_experiment_with_xr0/2025-10-15 10_19_22_63 first.nc'
mode = 'w', encoding = {'/foo': {}}, unlimited_dims = {}, format = 'NETCDF4'
engine = 'netcdf4', group = None, write_inherited_coords = False, compute = True
invalid_netcdf = False, auto_complex = None
def _datatree_to_netcdf(
dt: DataTree,
filepath: str | PathLike | io.IOBase | None = None,
mode: NetcdfWriteModes = "w",
encoding: Mapping[str, Any] | None = None,
unlimited_dims: Mapping | None = None,
format: T_DataTreeNetcdfTypes | None = None,
engine: T_DataTreeNetcdfEngine | None = None,
group: str | None = None,
write_inherited_coords: bool = False,
compute: bool = True,
invalid_netcdf: bool = False,
auto_complex: bool | None = None,
) -> None | memoryview | Delayed:
"""Implementation of `DataTree.to_netcdf`."""
if format not in [None, *get_args(T_DataTreeNetcdfTypes)]:
raise ValueError("DataTree.to_netcdf only supports the NETCDF4 format")
if engine not in [None, *get_args(T_DataTreeNetcdfEngine)]:
raise ValueError(
"DataTree.to_netcdf only supports the netcdf4 and h5netcdf engines"
)
normalized_path = _normalize_path(filepath)
if engine is None:
engine = get_default_netcdf_write_engine(
path_or_file=normalized_path,
format="NETCDF4", # required for supporting groups
) # type: ignore[assignment]
if group is not None:
raise NotImplementedError(
"specifying a root group for the tree has not been implemented"
)
if encoding is None:
encoding = {}
# In the future, we may want to expand this check to insure all the provided encoding
# options are valid. For now, this simply checks that all provided encoding keys are
# groups in the datatree.
if set(encoding) - set(dt.groups):
raise ValueError(
f"unexpected encoding group name(s) provided: {set(encoding) - set(dt.groups)}"
)
if normalized_path is None:
if not compute:
raise NotImplementedError(
"to_netcdf() with compute=False is not yet implemented when "
"returning a memoryview"
)
target = BytesIOProxy()
else:
target = normalized_path # type: ignore[assignment]
if unlimited_dims is None:
unlimited_dims = {}
scheduler = get_dask_scheduler()
have_chunks = any(
v.chunks is not None for node in dt.subtree for v in node.variables.values()
)
autoclose = have_chunks and scheduler in ["distributed", "multiprocessing"]
root_store = get_writable_netcdf_store(
target,
engine, # type: ignore[arg-type]
mode=mode,
format=format,
autoclose=autoclose,
invalid_netcdf=invalid_netcdf,
auto_complex=auto_complex,
)
writer = ArrayWriter()
# TODO: allow this work (setting up the file for writing array data)
# to be parallelized with dask
try:
for node in dt.subtree:
at_root = node is dt
dataset = node.to_dataset(inherit=write_inherited_coords or at_root)
node_store = (
root_store if at_root else root_store.get_child_store(node.path)
)
> dump_to_store(
dataset,
node_store,
writer,
encoding=encoding.get(node.path),
unlimited_dims=unlimited_dims.get(node.path),
)
$VENV/lib/python3.12/site-packages/xarray/backends/writers.py:899:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
$VENV/lib/python3.12/site-packages/xarray/backends/writers.py:491: in dump_to_store
store.store(variables, attrs, check_encoding, writer, unlimited_dims=unlimited_dims)
$VENV/lib/python3.12/site-packages/xarray/backends/common.py:533: in store
self.set_variables(
$VENV/lib/python3.12/site-packages/xarray/backends/common.py:575: in set_variables
writer.add(source, target)
$VENV/lib/python3.12/site-packages/xarray/backends/common.py:403: in add
target[...] = source
^^^^^^^^^^^
$VENV/lib/python3.12/site-packages/xarray/backends/netCDF4_.py:95: in __setitem__
data[key] = value
^^^^^^^^^
src/netCDF4/_netCDF4.pyx:5645: in netCDF4._netCDF4.Variable.__setitem__
???
src/netCDF4/_netCDF4.pyx:5932: in netCDF4._netCDF4.Variable._put
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E RuntimeError: NetCDF: HDF error
src/netCDF4/_netCDF4.pyx:2160: RuntimeError