Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions src/awkward/operations/ak_metadata_from_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,7 @@ def metadata_from_parquet(
ignore_metadata=False,
scan_files=True,
):
"""
Args:
path (str): Local filename or remote URL, passed to fsspec for resolution.
May contain glob patterns. A list of paths is also allowed, but they
must be data files, not directories.
storage_options: Passed to `fsspec.parquet.open_parquet_file`.
row_groups (None or set of int): Row groups to read; must be non-negative.
Order is ignored: the output array is presented in the order specified
by Parquet metadata. If None, all row groups/all rows are read.
ignore_metadata (bool): ignore the dedicated _metadata file if found
and instead derive metadata from the first data file.
scan_files (bool): TODO
"""Reads metadata from a Parquet file or dataset without reading the data.

This function differs from ak.from_parquet._metadata as follows:

Expand All @@ -47,7 +36,7 @@ def metadata_from_parquet(
the first data file
* the total number of rows is always known

Returns dict containing
A dict containing

* `form`: an Awkward Form representing the low-level type of the data
(use `.type` to get a high-level type),
Expand All @@ -60,6 +49,22 @@ def metadata_from_parquet(
argument).

See also #ak.from_parquet, #ak.to_parquet.

Args:
path (str): Local filename or remote URL, passed to fsspec for resolution.
May contain glob patterns. A list of paths is also allowed, but they
must be data files, not directories.
storage_options: Passed to `fsspec.parquet.open_parquet_file`.
row_groups (None or set of int): Row groups to read; must be non-negative.
Order is ignored: the output array is presented in the order specified
by Parquet metadata. If None, all row groups/all rows are read.
ignore_metadata (bool): ignore the dedicated _metadata file if found
and instead derive metadata from the first data file.
scan_files (bool): TODO

Returns:
A dict of metadata describing the Parquet dataset (its form, filesystem,
paths, row counts, and columns), read without reading the array data.
"""
import awkward._connect.pyarrow # noqa: F401

Expand Down
34 changes: 18 additions & 16 deletions src/awkward/operations/ak_to_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,22 @@ def to_arrow(
extensionarray=True,
count_nulls=True,
):
"""
"""Converts an Awkward Array into an Apache Arrow array.

This produces arrays of type `pyarrow.Array`. You might need to further
manipulations (using the pyarrow library) to build a `pyarrow.ChunkedArray`,
a `pyarrow.RecordBatch`, or a `pyarrow.Table`. For the latter, see #ak.to_arrow_table.

This function always preserves the values of a dataset; i.e. the Python objects
returned by #ak.to_list are identical to the Python objects returned by Arrow's
`to_pylist` method. With `extensionarray=True`, this function also preserves the
data type (high-level #ak.types.Type, though not the low-level #ak.forms.Form),
even through Parquet, making Parquet a good way to save Awkward Arrays for later
use. If any third-party tools don't recognize Arrow's extension arrays, set this
option to False for plain Arrow arrays.

See also #ak.from_arrow, #ak.to_arrow_table, #ak.to_parquet, #ak.from_arrow_schema.

Args:
array: Array-like data (anything #ak.to_layout recognizes).
list_to32 (bool): If True, convert Awkward lists into 32-bit Arrow lists
Expand Down Expand Up @@ -53,21 +68,8 @@ def to_arrow(
and include these in the resulting Arrow array, which makes some downstream
applications faster. If False, skip the up-front cost of counting them.

Converts an Awkward Array into an Apache Arrow array.

This produces arrays of type `pyarrow.Array`. You might need to further
manipulations (using the pyarrow library) to build a `pyarrow.ChunkedArray`,
a `pyarrow.RecordBatch`, or a `pyarrow.Table`. For the latter, see #ak.to_arrow_table.

This function always preserves the values of a dataset; i.e. the Python objects
returned by #ak.to_list are identical to the Python objects returned by Arrow's
`to_pylist` method. With `extensionarray=True`, this function also preserves the
data type (high-level #ak.types.Type, though not the low-level #ak.forms.Form),
even through Parquet, making Parquet a good way to save Awkward Arrays for later
use. If any third-party tools don't recognize Arrow's extension arrays, set this
option to False for plain Arrow arrays.

See also #ak.from_arrow, #ak.to_arrow_table, #ak.to_parquet, #ak.from_arrow_schema.
Returns:
An Apache Arrow array (`pyarrow.Array`) with the same data as `array`.
"""
# Dispatch
yield (array,)
Expand Down
32 changes: 17 additions & 15 deletions src/awkward/operations/ak_to_arrow_table.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,21 @@ def to_arrow_table(
extensionarray=True,
count_nulls=True,
):
"""
"""Converts an Awkward Array into an Apache Arrow table.

This produces arrays of type `pyarrow.Table`. If you want an Arrow array,
see #ak.to_arrow.

This function always preserves the values of a dataset; i.e. the Python objects
returned by #ak.to_list are identical to the Python objects returned by Arrow's
`to_pylist` method. With `extensionarray=True`, this function also preserves the
data type (high-level #ak.types.Type, though not the low-level #ak.forms.Form),
even through Parquet, making Parquet a good way to save Awkward Arrays for later
use. If any third-party tools don't recognize Arrow's extension arrays, set this
option to False for plain Arrow arrays.

See also #ak.from_arrow, #ak.to_arrow, #ak.to_parquet.

Args:
array: Array-like data (anything #ak.to_layout recognizes).
list_to32 (bool): If True, convert Awkward lists into 32-bit Arrow lists
Expand Down Expand Up @@ -56,20 +70,8 @@ def to_arrow_table(
and include these in the resulting Arrow array, which makes some downstream
applications faster. If False, skip the up-front cost of counting them.

Converts an Awkward Array into an Apache Arrow table.

This produces arrays of type `pyarrow.Table`. If you want an Arrow array,
see #ak.to_arrow.

This function always preserves the values of a dataset; i.e. the Python objects
returned by #ak.to_list are identical to the Python objects returned by Arrow's
`to_pylist` method. With `extensionarray=True`, this function also preserves the
data type (high-level #ak.types.Type, though not the low-level #ak.forms.Form),
even through Parquet, making Parquet a good way to save Awkward Arrays for later
use. If any third-party tools don't recognize Arrow's extension arrays, set this
option to False for plain Arrow arrays.

See also #ak.from_arrow, #ak.to_arrow, #ak.to_parquet.
Returns:
An Apache Arrow table (`pyarrow.Table`) with the same data as `array`.
"""
# Dispatch
yield (array,)
Expand Down
36 changes: 19 additions & 17 deletions src/awkward/operations/ak_to_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,7 @@

@high_level_function()
def to_backend(array, backend, *, highlevel=True, behavior=None, attrs=None):
"""
Args:
array: Array-like data (anything #ak.to_layout recognizes).
backend (`"cpu"`, `"cuda"`, `"jax"`, or `"typetracer"`): If `"cpu"`, the array structure is
recursively copied (if need be) to main memory for use with
the default Numpy backend; if `"cuda"`, the structure is copied
to the GPU(s) for use with CuPy. If `"jax"`, the structure is
copied to the CPU for use with JAX.
highlevel (bool): If True, return an #ak.Array; otherwise, return
a low-level #ak.contents.Content subclass.
behavior (None or dict): Custom #ak.behavior for the output array, if
high-level.
attrs (None or dict): Custom attributes for the output array, if
high-level.

Converts an array from `"cpu"`, `"cuda"`, `"jax"` kernels to `"cpu"`,
`"cuda"`, `"jax"`, or `"typetracer"` .
"""Returns an array on a different backend (kernel set).

Any components that are already in the desired backend are viewed,
rather than copied, so this operation can be an inexpensive way to ensure
Expand All @@ -53,6 +37,24 @@ def to_backend(array, backend, *, highlevel=True, behavior=None, attrs=None):
conda install -c conda-forge jax

See #ak.kernels.

Args:
array: Array-like data (anything #ak.to_layout recognizes).
backend (`"cpu"`, `"cuda"`, `"jax"`, or `"typetracer"`): If `"cpu"`, the array structure is
recursively copied (if need be) to main memory for use with
the default Numpy backend; if `"cuda"`, the structure is copied
to the GPU(s) for use with CuPy. If `"jax"`, the structure is
copied to the CPU for use with JAX.
highlevel (bool): If True, return an #ak.Array; otherwise, return
a low-level #ak.contents.Content subclass.
behavior (None or dict): Custom #ak.behavior for the output array, if
high-level.
attrs (None or dict): Custom attributes for the output array, if
high-level.

Returns:
An array with the same data as the input, moved to the requested backend
(kernel set).
"""
# Dispatch
yield (array,)
Expand Down
89 changes: 46 additions & 43 deletions src/awkward/operations/ak_to_buffers.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,42 +23,7 @@ def to_buffers(
backend=None,
byteorder=ak._util.native_byteorder,
):
"""
Args:
array: Array-like data (anything #ak.to_layout recognizes).
container (None or MutableMapping): The str \u2192 NumPy arrays (or
Python buffers) that represent the decomposed Awkward Array. This
`container` is only assumed to have a `__setitem__` method that
accepts strings as keys.
buffer_key (str or callable): Python format string containing
`"{form_key}"` and/or `"{attribute}"` or a function that takes these
(and/or `layout`) as keyword arguments and returns a string to use
as a key for a buffer in the `container`. The `form_key` is the result
of applying `form_key` (below), and the `attribute` is a hard-coded
string representing the buffer's function (e.g. `"data"`, `"offsets"`,
`"index"`).
form_key (str, callable): Python format string containing
`"{id}"` or a function that takes this (and/or `layout`) as a keyword
argument and returns a string to use as a key for a Form node.
Together, the `buffer_key` and `form_key` links attributes of each Form
node to data in the `container`.
id_start (int): Starting `id` to use in `form_key` and hence `buffer_key`.
This integer increases in a depth-first walk over the `array` nodes and
can be used to generate unique keys for each Form.
backend (`"cpu"`, `"cuda"`, `"jax"`, None): Backend to use to
generate values that are put into the `container`. The default,
`"cpu"`, makes NumPy arrays, which are in main memory
(e.g. not GPU) and satisfy Python's Buffer protocol. If all the
buffers in `array` have the same `backend` as this, they won't be
copied. If the backend is None, then the backend of the layout
will be used to generate the buffers.
byteorder (`"<"`, `">"`): Endianness of buffers written to `container`.
If the byteorder does not match the current system byteorder, the
arrays will be copied.

Decomposes an Awkward Array into a Form and a collection of memory buffers,
so that data can be losslessly written to file formats and storage devices
that only map names to binary blobs (such as a filesystem directory).
"""Decomposes an Awkward Array into a Form, length, and memory buffers.

This function returns a 3-tuple::

Expand Down Expand Up @@ -94,7 +59,50 @@ def to_buffers(
Dask, but partition numbers can be emulated by prepending a fixed `"partN-"`
string to the `buffer_key`. The `array` represents exactly one partition.

Here is a simple example:
If you intend to use this function for saving data, you may want to pack it
first with #ak.to_packed.

See also #ak.from_buffers and #ak.to_packed.

Args:
array: Array-like data (anything #ak.to_layout recognizes).
container (None or MutableMapping): The str \u2192 NumPy arrays (or
Python buffers) that represent the decomposed Awkward Array. This
`container` is only assumed to have a `__setitem__` method that
accepts strings as keys.
buffer_key (str or callable): Python format string containing
`"{form_key}"` and/or `"{attribute}"` or a function that takes these
(and/or `layout`) as keyword arguments and returns a string to use
as a key for a buffer in the `container`. The `form_key` is the result
of applying `form_key` (below), and the `attribute` is a hard-coded
string representing the buffer's function (e.g. `"data"`, `"offsets"`,
`"index"`).
form_key (str, callable): Python format string containing
`"{id}"` or a function that takes this (and/or `layout`) as a keyword
argument and returns a string to use as a key for a Form node.
Together, the `buffer_key` and `form_key` links attributes of each Form
node to data in the `container`.
id_start (int): Starting `id` to use in `form_key` and hence `buffer_key`.
This integer increases in a depth-first walk over the `array` nodes and
can be used to generate unique keys for each Form.
backend (`"cpu"`, `"cuda"`, `"jax"`, None): Backend to use to
generate values that are put into the `container`. The default,
`"cpu"`, makes NumPy arrays, which are in main memory
(e.g. not GPU) and satisfy Python's Buffer protocol. If all the
buffers in `array` have the same `backend` as this, they won't be
copied. If the backend is None, then the backend of the layout
will be used to generate the buffers.
byteorder (`"<"`, `">"`): Endianness of buffers written to `container`.
If the byteorder does not match the current system byteorder, the
arrays will be copied.

Returns:
A 3-tuple `(form, length, container)` decomposed from `array`,
so that data can be losslessly written to file formats and storage devices
that only map names to binary blobs (such as a filesystem directory).

Examples:
Here is a simple example:

>>> original = ak.Array([[1, 2, 3], [], [4, 5]])
>>> form, length, container = ak.to_buffers(original)
Expand All @@ -114,15 +122,10 @@ def to_buffers(
>>> container
{'node0-offsets': array([0, 3, 3, 5]), 'node1-data': array([1, 2, 3, 4, 5])}

which may be read back with
which may be read back with

>>> ak.from_buffers(form, length, container)
<Array [[1, 2, 3], [], [4, 5]] type='3 * var * int64'>

If you intend to use this function for saving data, you may want to pack it
first with #ak.to_packed.

See also #ak.from_buffers and #ak.to_packed.
"""
# Dispatch
yield (array,)
Expand Down
28 changes: 16 additions & 12 deletions src/awkward/operations/ak_to_feather.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,18 @@ def to_feather(
chunksize=None,
feather_version=2,
):
"""
"""Writes an Awkward Array to a Feather file (through pyarrow).

If the `array` does not contain records at top-level, the Arrow table will
consist of one field whose name is `""` iff. `extensionarray` is False.

If `extensionarray` is True`, use a custom Arrow extension to store this array.
Otherwise, generic Arrow arrays are used, and if the `array` does not
contain records at top-level, the Arrow table will consist of one field whose
name is `""`. See #ak.to_arrow_table for more details.

See also #ak.from_feather.

Args:
array: Array-like data (anything #ak.to_layout recognizes).
destination (str): Local destination path, passed to
Expand Down Expand Up @@ -73,20 +84,13 @@ def to_feather(
Version 2 is the current. Version 1 is the more limited legacy format. If not
provided, version 2 is used.

Writes an Awkward Array to a Feather file (through pyarrow).
Returns:
None. The contents of `array` are written to the given Feather file
(through pyarrow).

Examples:
>>> array = ak.Array([[1.1, 2.2, 3.3], [], [4.4, 5.5]])
>>> ak.to_feather(array, "filename.feather")

If the `array` does not contain records at top-level, the Arrow table will
consist of one field whose name is `""` iff. `extensionarray` is False.

If `extensionarray` is True`, use a custom Arrow extension to store this array.
Otherwise, generic Arrow arrays are used, and if the `array` does not
contain records at top-level, the Arrow table will consist of one field whose
name is `""`. See #ak.to_arrow_table for more details.

See also #ak.from_feather.
"""

# Dispatch
Expand Down
Loading
Loading