From 5ca9be1d774dbb88c2965832d37621f985694710 Mon Sep 17 00:00:00 2001 From: Tai Sakuma Date: Tue, 21 Jul 2026 18:00:39 -0400 Subject: [PATCH 1/3] test: add property-based roundtrip test for `to_parquet`/`from_parquet` Generate arrays restricted to what the parquet path can roundtrip: no unions, no unknown types, and dtypes limited to those pyarrow can write and read back unchanged (complex excluded; datetime64 only "ns"; timedelta64 only "s"/"ms"/"us"/"ns"). A `parquet_writable` predicate excludes layouts that `to_arrow`/`from_parquet` currently mishandle, including a silent offsets-shift corruption for nullable var-length lists. Tests write to fsspec's in-memory filesystem, so no disk I/O. A second test covers option-type layouts (returned as bit-masked, so content classes are not compared), and a third checks that `attrs` survive the roundtrip. Co-Authored-By: Claude Fable 5 --- .../operations/test_to_from_parquet.py | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 tests/properties/operations/test_to_from_parquet.py diff --git a/tests/properties/operations/test_to_from_parquet.py b/tests/properties/operations/test_to_from_parquet.py new file mode 100644 index 0000000000..f90abef57d --- /dev/null +++ b/tests/properties/operations/test_to_from_parquet.py @@ -0,0 +1,182 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/awkward/blob/main/LICENSE + +from __future__ import annotations + +import threading + +import hypothesis_awkward.strategies as st_ak +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st +from hypothesis_awkward.util.dtype import SUPPORTED_DTYPES + +import awkward as ak + +# The first example pays one-time lazy imports (pyarrow.parquet, fsspec's +# memory filesystem) of ~350 ms, exceeding hypothesis' default 200 ms +# deadline; later examples take ~1 ms. +no_deadline = settings(deadline=None) + + +def parquet_path() -> str: + """A fixed path in fsspec's in-memory filesystem, overwritten by each + example. + + Under pytest-run-parallel the test body runs concurrently on multiple + threads sharing the process-global in-memory filesystem, so use a + per-thread filename to avoid one thread reading the file while another + is still writing it. + """ + return f"memory://test-to-from-parquet/{threading.get_ident()}.parquet" + + +def parquet_dtype(dtype: np.dtype) -> bool: + if dtype.kind == "c": + # pyarrow cannot write complex numbers (ArrowNotImplementedError) + return False + if dtype.kind == "M": + # Only "ns" roundtrips: coarser units are unsupported or coerced + # ("s" -> "ms"; "D" corrupts values), and out-of-range values such + # as NaT in "ms"/"us" crash `from_parquet` reading row-group + # statistics (OverflowError in pyarrow's TimestampScalar.as_py). + return np.datetime_data(dtype)[0] == "ns" + if dtype.kind == "m": + # pyarrow durations support only seconds and finer units + return np.datetime_data(dtype)[0] in {"s", "ms", "us", "ns"} + return True + + +SUPPORTED_PARQUET_DTYPES = tuple(d for d in SUPPORTED_DTYPES if parquet_dtype(d)) + + +def parquet_dtypes() -> st.SearchStrategy[np.dtype]: + """Strategy for dtypes that survive the parquet roundtrip.""" + return st.sampled_from(SUPPORTED_PARQUET_DTYPES) + + +def parquet_writable(a: ak.Array) -> bool: + layout, is_option = a.layout, False + while layout.is_option or layout.is_indexed: + is_option = is_option or layout.is_option + layout = layout.content + if layout.is_record: + if layout.fields == [""]: + # collides with the anonymous column that `to_parquet` uses + # for non-record arrays; reads back unwrapped + return False + if is_option and all(x.is_option for x in layout.contents): + # a valid row whose fields are all null reads back as a null + # row (outermost struct validity is not stored) + return False + return _nodes_writable(a.layout) + + +def _nodes_writable(layout: ak.contents.Content, nullable: bool = False) -> bool: + """Exclude layouts that `to_arrow` currently mishandles. + + `nullable` tracks whether Arrow validity bytes flow into this node from + an enclosing option: they start at option nodes, pass through records + and indexed nodes, and stop below lists. A var-length list receiving + validity bytes whose offsets do not start at zero is compacted against + unshifted content in `ListOffsetArray._to_arrow`, shifting every list + by `offsets[0]` (data corruption); indexed nodes can recreate such + offsets from anywhere in their content, so they are excluded outright + when nullable. + """ + if layout.is_record: + if len(layout.fields) == 0: + # named records lose their length; tuples cannot be written + # (pyarrow: "Cannot write struct type '' with no child field") + return False + return all(_nodes_writable(x, nullable) for x in layout.contents) + if layout.is_regular: + # `to_arrow` drops the length of a size-0 `RegularArray` + # (pyarrow: "Expected all lists to be of size=0") + return layout.size > 0 and _nodes_writable(layout.content, False) + if layout.is_option: + if isinstance(layout, ak.contents.IndexedOptionArray): + if layout.length > 0 and layout.content.length == 0: + # `to_arrow` crashes projecting nulls over a zero-length + # content (IndexError in `ListOffsetArray._to_arrow`) + return False + if nullable and layout.content.is_list: + return False + return _nodes_writable(layout.content, True) + if layout.is_indexed: + if nullable and layout.content.is_list: + return False + return _nodes_writable(layout.content, nullable) + if layout.is_list: + if nullable and layout.length > 0 and layout.starts[0] != 0: + return False + return _nodes_writable(layout.content, False) + return True + + +@no_deadline +@given( + a=st_ak.constructors.arrays( + dtypes=parquet_dtypes(), + # a bare unknown type cannot be written ("A null type field may + # not be non-nullable"), and one nested in a record or regular + # list reads back as an option + allow_empty=False, + # pyarrow cannot write unions to parquet + allow_union=False, + allow_indexed_option=False, + allow_byte_masked=False, + allow_bit_masked=False, + allow_unmasked=False, + ).filter(parquet_writable) +) +def test_roundtrip(a: ak.Array) -> None: + """`to_parquet` followed by `from_parquet` reconstructs the array.""" + path = parquet_path() + ak.to_parquet(a, path) + returned = ak.from_parquet(path) + assert ak.array_equal(a, returned, equal_nan=True) + + +@no_deadline +@given( + a=st_ak.constructors.arrays( + dtypes=parquet_dtypes(), + allow_empty=False, + # an option directly over a `RegularArray` cannot be written + # (ARROW-14547: "Lists with non-zero length null components are + # not supported") + allow_regular=False, + allow_union=False, + ).filter(parquet_writable) +) +def test_roundtrip_masked(a: ak.Array) -> None: + """Option-type arrays roundtrip through nullable parquet columns. + + `from_parquet` returns every option layout as a `BitMaskedArray` (or + `UnmaskedArray`), so content classes are not compared. + """ + path = parquet_path() + ak.to_parquet(a, path) + returned = ak.from_parquet(path) + assert ak.array_equal(a, returned, equal_nan=True, same_content_types=False) + + +@no_deadline +@given( + attrs=st.dictionaries( + # keys starting with "@" are transient and intentionally not + # written to parquet + st.text().filter(lambda k: not k.startswith("@")), + st.none() + | st.booleans() + | st.integers() + | st.floats(allow_nan=False) + | st.text(), + ) +) +def test_roundtrip_attrs(attrs: dict) -> None: + """`attrs` survive the parquet roundtrip.""" + a = ak.Array([1.1, 2.2, 3.3], attrs=attrs) + path = parquet_path() + ak.to_parquet(a, path) + assert ak.from_parquet(path).attrs == attrs From 9103557419e245e99409d616e4126dd920c9ebe4 Mon Sep 17 00:00:00 2001 From: Tai Sakuma Date: Tue, 21 Jul 2026 18:37:55 -0400 Subject: [PATCH 2/3] test: cite issues for the parquet roundtrip exclusions The four defects excluded by `parquet_dtype` and `parquet_writable` are now reported: #4219 (datetime64[D] value corruption), #4220 (row-group statistics OverflowError), #4221 (zero-length-content IndexError), and #4222 (offsets-shift corruption for nullable lists). Co-Authored-By: Claude Fable 5 --- tests/properties/operations/test_to_from_parquet.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/properties/operations/test_to_from_parquet.py b/tests/properties/operations/test_to_from_parquet.py index f90abef57d..4c59fd4c5d 100644 --- a/tests/properties/operations/test_to_from_parquet.py +++ b/tests/properties/operations/test_to_from_parquet.py @@ -36,9 +36,9 @@ def parquet_dtype(dtype: np.dtype) -> bool: return False if dtype.kind == "M": # Only "ns" roundtrips: coarser units are unsupported or coerced - # ("s" -> "ms"; "D" corrupts values), and out-of-range values such - # as NaT in "ms"/"us" crash `from_parquet` reading row-group - # statistics (OverflowError in pyarrow's TimestampScalar.as_py). + # ("s" -> "ms"; "D" corrupts values, #4219), and out-of-range + # values such as NaT in "ms"/"us" crash `from_parquet` reading + # row-group statistics (OverflowError, #4220). return np.datetime_data(dtype)[0] == "ns" if dtype.kind == "m": # pyarrow durations support only seconds and finer units @@ -79,7 +79,7 @@ def _nodes_writable(layout: ak.contents.Content, nullable: bool = False) -> bool and indexed nodes, and stop below lists. A var-length list receiving validity bytes whose offsets do not start at zero is compacted against unshifted content in `ListOffsetArray._to_arrow`, shifting every list - by `offsets[0]` (data corruption); indexed nodes can recreate such + by `offsets[0]` (data corruption, #4222); indexed nodes can recreate such offsets from anywhere in their content, so they are excluded outright when nullable. """ @@ -97,7 +97,7 @@ def _nodes_writable(layout: ak.contents.Content, nullable: bool = False) -> bool if isinstance(layout, ak.contents.IndexedOptionArray): if layout.length > 0 and layout.content.length == 0: # `to_arrow` crashes projecting nulls over a zero-length - # content (IndexError in `ListOffsetArray._to_arrow`) + # content (IndexError in `ListOffsetArray._to_arrow`, #4221) return False if nullable and layout.content.is_list: return False From dc3a661ff5dfe50fbbb6bd3ea99da3d4fb22ad9e Mon Sep 17 00:00:00 2001 From: Tai Sakuma Date: Tue, 21 Jul 2026 18:38:02 -0400 Subject: [PATCH 3/3] test: fix parquet roundtrip test on Linux and Windows CI Skip when pyarrow is not installed (Windows CI), matching the other parquet tests, and exclude extended-precision floats: hypothesis-awkward builds its dtype list from the platform's NumPy, so Linux generates float128, which pyarrow cannot write. Co-Authored-By: Claude Fable 5 --- tests/properties/operations/test_to_from_parquet.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/properties/operations/test_to_from_parquet.py b/tests/properties/operations/test_to_from_parquet.py index 4c59fd4c5d..fb00dafaa8 100644 --- a/tests/properties/operations/test_to_from_parquet.py +++ b/tests/properties/operations/test_to_from_parquet.py @@ -6,12 +6,15 @@ import hypothesis_awkward.strategies as st_ak import numpy as np +import pytest from hypothesis import given, settings from hypothesis import strategies as st from hypothesis_awkward.util.dtype import SUPPORTED_DTYPES import awkward as ak +pytest.importorskip("pyarrow.parquet") + # The first example pays one-time lazy imports (pyarrow.parquet, fsspec's # memory filesystem) of ~350 ms, exceeding hypothesis' default 200 ms # deadline; later examples take ~1 ms. @@ -34,6 +37,10 @@ def parquet_dtype(dtype: np.dtype) -> bool: if dtype.kind == "c": # pyarrow cannot write complex numbers (ArrowNotImplementedError) return False + if dtype.kind == "f" and dtype.itemsize == 16: + # pyarrow cannot write extended-precision floats, which exist on + # some platforms (e.g. float128 on Linux) + return False if dtype.kind == "M": # Only "ns" roundtrips: coarser units are unsupported or coerced # ("s" -> "ms"; "D" corrupts values, #4219), and out-of-range