GH-32609: [Python] Add internal type system stubs (_types, error, _stubs_typing)

.github/workflows/python.yml

@@ -239,6 +239,9 @@ jobs:
       - name: Test
         shell: bash
         run: ci/scripts/python_test.sh $(pwd) $(pwd)/build
+      - name: Test annotations
+        shell: bash
+        run: ci/scripts/python_test_type_annotations.sh $(pwd)/python
 
   windows:
     name: AMD64 Windows 2022 Python 3.13
@@ -296,3 +299,7 @@ jobs:
         shell: cmd
         run: |
           call "ci\scripts\python_test.bat" %cd%
+      - name: Test annotations
+        shell: cmd
+        run: |
+          call "ci\scripts\python_test_type_annotations.bat" %cd%\python

ci/scripts/python_test_type_annotations.bat

@@ -0,0 +1,38 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+set PYARROW_DIR=%1
+
+echo Annotation testing on Windows ...
+
+@REM Install library stubs
+%PYTHON_CMD% -m pip install pandas-stubs scipy-stubs sphinx types-cffi types-psutil types-requests types-python-dateutil || exit /B 1
+
+@REM Install other dependencies for type checking
+%PYTHON_CMD% -m pip install fsspec || exit /B 1
+
+@REM Install type checkers
+%PYTHON_CMD% -m pip install mypy pyright ty || exit /B 1
+
+@REM Run type checkers
+pushd %PYARROW_DIR%
+
+mypy
+pyright
+ty check

ci/scripts/python_test_type_annotations.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+pyarrow_dir=${1}
+
+# Install library stubs
+pip install pandas-stubs scipy-stubs sphinx types-cffi types-psutil types-requests types-python-dateutil
+
+# Install type checkers
+pip install mypy pyright ty
+
+# Install other dependencies for type checking
+pip install fsspec
+
+# Run type checkers
+pushd ${pyarrow_dir}
+mypy
+pyright
+ty check;
+fi

ci/scripts/python_wheel_macos_build.sh

@@ -175,6 +175,11 @@ export CMAKE_PREFIX_PATH=${build_dir}/install
 export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION}
 
 pushd ${source_dir}/python
+# We first populate stub docstrings and then build the wheel
+python setup.py build_ext --inplace
+python -m pip install griffe libcst
+python ../dev/update_stub_docstrings.py pyarrow-stubs
+
 python setup.py bdist_wheel
 popd
 

ci/scripts/python_wheel_validate_contents.py

@@ -35,6 +35,11 @@ def validate_wheel(path):
     assert not outliers, f"Unexpected contents in wheel: {sorted(outliers)}"
     print(f"The wheel: {wheels[0]} seems valid.")
 
+    candidates = [info for info in f.filelist if info.filename.endswith('compute.pyi')]
+    assert candidates, "compute.pyi not found in wheel"
+    content = f.read(candidates[0]).decode('utf-8', errors='replace')
+    assert '"""' in content, "compute.pyi missing docstrings (no triple quotes found)"
+
 
 def main():
     parser = argparse.ArgumentParser()

ci/scripts/python_wheel_windows_build.bat

@@ -135,6 +135,11 @@ pushd C:\arrow\python
 @REM Build wheel
 %PYTHON_CMD% setup.py bdist_wheel || exit /B 1
 
+@REM We first populate stub docstrings and then build the wheel
+%PYTHON_CMD% setup.py build_ext --inplace
+%PYTHON_CMD% -m pip install griffe libcst
+%PYTHON_CMD% ..\dev\update_stub_docstrings.py pyarrow-stubs
+
 @REM Repair the wheel with delvewheel
 @REM
 @REM Since we bundled the Arrow C++ libraries ourselves, we only need to

ci/scripts/python_wheel_xlinux_build.sh

@@ -167,6 +167,11 @@ export ARROW_HOME=/tmp/arrow-dist
 export CMAKE_PREFIX_PATH=/tmp/arrow-dist
 
 pushd /arrow/python
+# We first populate stub docstrings and then build the wheel
+python setup.py build_ext --inplace
+python -m pip install griffe libcst
+python ../dev/update_stub_docstrings.py pyarrow-stubs
+
 python setup.py bdist_wheel
 
 echo "=== Strip symbols from wheel ==="

compose.yaml

@@ -924,7 +924,8 @@ services:
       ["
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
-        /arrow/ci/scripts/python_test.sh /arrow"]
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"]
 
   conda-python-emscripten:
     # Usage:
@@ -1008,7 +1009,8 @@ services:
       /bin/bash -c "
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
-        /arrow/ci/scripts/python_test.sh /arrow"
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"
 
   debian-python:
     # Usage:
@@ -1510,7 +1512,8 @@ services:
         /arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         mamba uninstall -y numpy &&
-        /arrow/ci/scripts/python_test.sh /arrow"]
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"]
 
   conda-python-docs:
     # Usage:
@@ -1536,7 +1539,8 @@ services:
         /arrow/ci/scripts/python_build.sh /arrow /build &&
         pip install -e /arrow/dev/archery[numpydoc] &&
         archery numpydoc --allow-rule GL10,PR01,PR03,PR04,PR05,PR10,RT03,YD01 &&
-        /arrow/ci/scripts/python_test.sh /arrow"]
+        /arrow/ci/scripts/python_test.sh /arrow &&
+        /arrow/ci/scripts/python_test_type_annotations.sh /arrow/python"]
 
   conda-python-dask:
     # Possible $DASK parameters:

docs/source/developers/python/development.rst

@@ -42,7 +42,7 @@ Unit Testing
 ============
 
 We are using `pytest <https://docs.pytest.org/en/latest/>`_ to develop our unit
-test suite. After `building the project <build_pyarrow>`_ you can run its unit tests
+test suite. After `building the project <building.html>`_ you can run its unit tests
 like so:
 
 .. code-block::
@@ -101,6 +101,74 @@ The test groups currently include:
 * ``s3``: Tests for Amazon S3
 * ``tensorflow``: Tests that involve TensorFlow
 
+Type Checking
+=============
+
+PyArrow provides type stubs (``*.pyi`` files) for static type checking. These
+stubs are located in the ``pyarrow-stubs/`` directory and are automatically
+included in the distributed wheel packages.
+
+Running Type Checkers
+---------------------
+
+We support multiple type checkers. Their configurations are in
+``pyproject.toml``.
+
+**mypy**
+
+To run mypy on the PyArrow codebase:
+
+.. code-block::
+
+   $ cd arrow/python
+   $ mypy
+
+The mypy configuration is in the ``[tool.mypy]`` section of ``pyproject.toml``.
+
+**pyright**
+
+To run pyright:
+
+.. code-block::
+
+   $ cd arrow/python
+   $ pyright
+
+The pyright configuration is in the ``[tool.pyright]`` section of ``pyproject.toml``.
+
+**ty**
+
+To run ty (note: currently only partially configured):
+
+.. code-block::
+
+   $ cd arrow/python
+   $ ty check
+
+Maintaining Type Stubs
+-----------------------
+
+Type stubs for PyArrow are maintained in the ``pyarrow-stubs/``
+directory. These stubs mirror the structure of the main ``pyarrow/`` package.
+
+When adding or modifying public APIs:
+
+1. **Update the corresponding ``.pyi`` stub file** in ``pyarrow-stubs/``
+   to reflect the new or changed function/class signatures.
+
+2. **Include type annotations** where possible. For Cython modules or
+   dynamically generated APIs such as compute kernels add the corresponding
+   stub in ``pyarrow-stubs/``.
+
+3. **Run type checkers** to ensure the stubs are correct and complete.
+
+The stub files are automatically copied into the built wheel during the build
+process and will be included when users install PyArrow, enabling type checking
+in downstream projects and for users' IDEs.
+
+Note: ``py.typed`` marker file in the ``pyarrow/`` directory indicates to type
+checkers that PyArrow supports type checking according to :pep:`561`.
+
 Doctest
 =======
 

python/MANIFEST.in

@@ -4,6 +4,7 @@ include ../NOTICE.txt
 
 global-include CMakeLists.txt
 graft pyarrow
+graft pyarrow-stubs
 graft cmake_modules
 
 global-exclude *.so

python/pyarrow-stubs/pyarrow/__init__.pyi

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Type stubs for PyArrow.
+
+This is a placeholder stub file.
+Complete type annotations will be added in subsequent PRs.
+"""
+
+from typing import Any
+
+def __getattr__(name: str) -> Any: ...