Add Parquet-to-NumPy C entry point with column projection

MANIFEST.in

@@ -1,3 +1,4 @@
 include src/__init__.pyi src/py.typed
 include src/*.h
+include src/*.cpp
 include VERSION

README.rst

@@ -32,6 +32,65 @@ ArrayKit requires the following:
 - numpy>=1.24.3
 
 
+Reading Parquet
+--------------
+
+``parquet_to_arrays(file, *, columns_select=None)`` returns a list of immutable,
+one-dimensional NumPy arrays, one per top-level Parquet column. It uses the native
+Arrow and Parquet C++ libraries, not Python PyArrow or ``arrow_python``.
+
+Parquet support is opt-in at build time. Install Arrow/Parquet >=24 development
+libraries and a C++20 compiler, then build ArrayKit from source::
+
+    ARRAYKIT_WITH_PARQUET=1 pip install --no-binary=arraykit arraykit
+
+The build finds libraries using ``pkg-config``. Alternatively set ``ARROW_HOME``
+to a native installation prefix containing ``include`` and ``lib``, or set both
+``ARROW_INCLUDE_DIR`` and ``ARROW_LIBRARY_DIR``. These settings are build-time
+only; the resulting extension must be able to load the same native libraries at
+runtime. Explicit library directories are embedded as runtime search paths on
+Unix. On Windows, make the Arrow DLL directory available to the loader (for
+example with ``os.add_dll_directory()`` before calling the reader). Keep native
+library installations ABI-compatible with the version used to build the extension.
+Locally built wheels with this backend are not portable unless their native
+dependencies are also supplied.
+
+Normal builds still require only NumPy. Calling ``parquet_to_arrays`` without
+the native backend raises ``ImportError`` with build instructions. PyArrow is
+used only to generate fixtures and reference results in optional tests, never
+for loading Parquet in ArrayKit.
+
+``file`` can be a string, bytes, or path-like filesystem path, or a seekable
+binary file such as an open file or ``io.BytesIO``. Caller-owned files are not
+closed; their position can change. Paths identify individual files, not datasets
+or directories.
+
+``columns_select`` can be a list of unique column names. Only those columns are
+decoded, in selection order. ``None`` loads all columns in file order; ``[]``
+returns an empty list. Missing or ambiguous names and repeated selections raise
+``ValueError``. Names select whole top-level columns (including nested columns),
+not dotted paths to nested fields.
+
+The native extension reads paths through Arrow's random-access file API.
+Open Python files use binary ``read``, ``seek``, and ``tell`` callbacks; the
+Parquet decoder and array conversion remain native. It releases the GIL during
+reading and decoding. Arrow handles compression and page encodings; ArrayKit
+copies decoded columns directly into NumPy-owned arrays through the C API,
+without a Python Arrow table, pandas, or Python loops over columns or values.
+Performance relative to Python PyArrow depends on the file, column types, and
+selection.
+
+Dtypes and null handling preserve numeric types where possible; nullable integers
+become floating-point arrays with ``NaN``, nullable booleans and strings use object
+arrays with ``None``, and timestamps use ``datetime64`` with ``NaT``. Large nullable integers
+can lose precision when converted to floating point. Dictionary columns are
+decoded to values, decimal columns use ``decimal.Decimal`` objects, and nested
+struct/list columns use object arrays. Other Arrow types may not have a supported
+NumPy conversion. Unsupported conversions raise ``NotImplementedError``;
+malformed files raise ``ValueError`` or ``OSError``. Exceptions from Python file
+methods are preserved.
+
+
 What is New in ArrayKit
 -------------------------
 

setup.py

@@ -2,6 +2,8 @@
 from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
 import site, os
+import shlex
+import subprocess
 from pathlib import Path
 
 AK_VERSION = Path('VERSION').read_text(encoding='utf-8').strip()
@@ -33,6 +35,7 @@ def get_ext_dir(*components: tp.Iterable[str]) -> tp.Sequence[str]:
             'src/block_index.c',
             'src/delimited_to_arrays.c',
             'src/methods.c',
+            'src/parquet_to_arrays.c',
             'src/tri_map.c',
             'src/auto_map.c',
         ],
@@ -43,4 +46,57 @@ def get_ext_dir(*components: tp.Iterable[str]) -> tp.Sequence[str]:
     )
 ]
 
+
+def get_parquet_extension() -> Extension:
+    prefix = os.environ.get('ARROW_HOME')
+    include_dir = os.environ.get('ARROW_INCLUDE_DIR')
+    library_dir = os.environ.get('ARROW_LIBRARY_DIR')
+    if prefix:
+        include_dir = include_dir or os.path.join(prefix, 'include')
+        library_dir = library_dir or os.path.join(prefix, 'lib')
+    compile_args = ['/std:c++20'] if os.name == 'nt' else ['-std=c++20']
+    link_args = []
+    library_dirs = []
+    include_dirs = get_ext_dir('numpy', '_core', 'include')
+    if include_dir or library_dir:
+        if not (include_dir and library_dir):
+            raise RuntimeError('Set both ARROW_INCLUDE_DIR and ARROW_LIBRARY_DIR')
+        include_dirs.append(include_dir)
+        library_dirs.append(library_dir)
+    else:
+        try:
+            subprocess.check_call(
+                ['pkg-config', '--atleast-version=24', 'arrow', 'parquet']
+            )
+            compile_args.extend(shlex.split(subprocess.check_output(
+                ['pkg-config', '--cflags', 'arrow', 'parquet'], text=True
+            )))
+            link_args = shlex.split(subprocess.check_output(
+                ['pkg-config', '--libs', 'arrow', 'parquet'], text=True
+            ))
+        except (OSError, subprocess.CalledProcessError) as error:
+            raise RuntimeError(
+                'Native Parquet support requires Arrow/Parquet >=24 development '
+                'libraries. Install their pkg-config files or set ARROW_HOME '
+                '(or ARROW_INCLUDE_DIR and ARROW_LIBRARY_DIR).'
+            ) from error
+    runtime_dirs = library_dirs + [
+        flag[2:] for flag in link_args if flag.startswith('-L')
+    ]
+    return Extension(
+        name='arraykit._parquet',
+        sources=['src/parquet_reader.cpp'],
+        include_dirs=include_dirs,
+        library_dirs=library_dirs,
+        runtime_library_dirs=runtime_dirs if os.name != 'nt' else [],
+        libraries=['parquet', 'arrow'],
+        extra_compile_args=compile_args,
+        extra_link_args=link_args,
+        language='c++',
+    )
+
+
+if os.environ.get('ARRAYKIT_WITH_PARQUET') == '1':
+    ext_modules.append(get_parquet_extension())
+
 setup(ext_modules=ext_modules)

src/__init__.py

@@ -21,6 +21,7 @@
 from ._arraykit import isna_element as isna_element
 from ._arraykit import dtype_from_element as dtype_from_element
 from ._arraykit import delimited_to_arrays as delimited_to_arrays
+from ._arraykit import parquet_to_arrays as parquet_to_arrays
 from ._arraykit import iterable_str_to_array_1d as iterable_str_to_array_1d
 from ._arraykit import split_after_count as split_after_count
 from ._arraykit import get_new_indexers_and_screen as get_new_indexers_and_screen

src/__init__.pyi

@@ -1,12 +1,19 @@
 import typing as tp
 import datetime
+import os
 
 import numpy as np  # type: ignore
 
 _T = tp.TypeVar('_T')
 
 __version__: str
 
+def parquet_to_arrays(
+    file: tp.Union[str, bytes, os.PathLike, tp.BinaryIO],
+    *,
+    columns_select: tp.Optional[tp.List[str]] = None,
+) -> tp.List[np.ndarray]: ...
+
 _TLabel = tp.Union[
     tp.Hashable,
     int,

src/_arraykit.c

@@ -10,6 +10,7 @@
 # include "block_index.h"
 # include "delimited_to_arrays.h"
 # include "methods.h"
+# include "parquet_to_arrays.h"
 # include "tri_map.h"
 # include "auto_map.h"
 
@@ -43,6 +44,10 @@ static PyMethodDef arraykit_methods[] =  {
             (PyCFunction)delimited_to_arrays,
             METH_VARARGS | METH_KEYWORDS,
             NULL},
+    {"parquet_to_arrays",
+            (PyCFunction)parquet_to_arrays,
+            METH_VARARGS | METH_KEYWORDS,
+            "Read a Parquet file into immutable 1D arrays; requires the native backend."},
     {"iterable_str_to_array_1d",
             (PyCFunction)iterable_str_to_array_1d,
             METH_VARARGS | METH_KEYWORDS,