ci-python.yml

################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

name: CI / Python

on:
  workflow_call:

permissions:
  contents: read

env:
  JDK_VERSION: 8
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true


jobs:
  # Lint + test on selected versions only (3.6 / 3.7 low end, 3.10 / 3.11 / 3.12 / 3.13 current), not every Python version.
  test:
    name: Tests / Python ${{ matrix.python-version }}
    env:
      # Native planning, reading, writing and REST commits run in Native CI.
      PYTEST_ADDOPTS: "-m 'not native_plan'"
    timeout-minutes: 90
    runs-on: ubuntu-latest
    container: "python:${{ matrix.python-version }}-slim"
    strategy:
      fail-fast: false
      matrix:
        python-version: [ '3.6.15', '3.7', '3.10', '3.11', '3.12', '3.13' ]

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Set up JDK ${{ env.JDK_VERSION }}
        uses: actions/setup-java@v5
        with:
          java-version: ${{ env.JDK_VERSION }}
          distribution: 'temurin'

      - name: Cache Maven dependencies
        uses: actions/cache@v5
        with:
          path: ~/.m2/repository
          key: ci-maven-python-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ci-maven-python-${{ runner.os }}-${{ runner.arch }}-

      - name: Install system dependencies
        shell: bash
        run: |
          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
            # The Python 3.6 image uses EOL Debian Bullseye.
            sed -i '/security.debian.org\/debian-security/d' /etc/apt/sources.list
            sed -i 's|deb.debian.org/debian|archive.debian.org/debian|g' /etc/apt/sources.list
            echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99archive
          fi
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            pkg-config \
            libssl-dev \
            libclang-dev \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Set up Maven
        shell: bash
        env:
          MAVEN_VERSION: '3.8.8'
        run: |
          maven_archive="${RUNNER_TEMP}/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
          # Central can return transient 404s even for an existing release.
          curl --fail --location --retry 3 --retry-all-errors --retry-delay 2 \
            --output "$maven_archive" \
            "https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/apache-maven-${MAVEN_VERSION}-bin.tar.gz" \
            || curl --fail --location --retry 3 --retry-all-errors --retry-delay 2 \
              --output "$maven_archive" \
              "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
          tar -xzf "$maven_archive" -C "$RUNNER_TEMP"
          echo "${RUNNER_TEMP}/apache-maven-${MAVEN_VERSION}/bin" >> "$GITHUB_PATH"

      - name: Verify Java and Maven installation
        run: |
          java -version
          mvn -version

      - name: Verify Python version
        run: python --version

      - name: Verify Python distributions
        run: |
          # Build tools and legacy Flake8 require incompatible importlib-metadata versions.
          python -m venv "${RUNNER_TEMP}/paimon-packaging"
          "${RUNNER_TEMP}/paimon-packaging/bin/python" -m pip install --upgrade pip setuptools wheel packaging tomli build
          "${RUNNER_TEMP}/paimon-packaging/bin/python" paimon-python/dev/test_packaging.py

      - name: Verify core package dependencies can be installed
        shell: bash
        run: |
          cd paimon-python
          python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || {
            echo "ERROR: Failed to install core PyPaimon dependencies"
            rm -rf "$TEMP_DIR"
            exit 1
          }
          rm -rf "$TEMP_DIR"
          echo "✓ Core PyPaimon dependencies resolve on this Python"

      - name: Build Java
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B -ntp install -DskipTests \
            -pl paimon-core,paimon-lance,paimon-vortex/paimon-vortex-format,paimon-full-text,paimon-lumina,paimon-vector -am

      - name: Install Python dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          if python -c "import sys; sys.exit(0 if sys.version_info < (3, 8) else 1)"; then
            # Old pip has no --group; read the same TOML with the compatibility bridge.
            python -m pip install tomli
            python paimon-python/dev/install.py legacy
          else
            python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
            dependency_group=python-tests
            if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
              dependency_group=torch
            fi
            python -m pip install -e ./paimon-python --group "./paimon-python/pyproject.toml:${dependency_group}"
          fi
          python -m pip check

      - name: Verify Python test environment
        run: |
          python - <<'PY'
          import importlib.util

          assert importlib.util.find_spec('pypaimon_rust') is None, 'Rust is tested in Native CI'
          PY

      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -e pytest_torch

  s3-delete-compatibility:
    name: Compatibility / PyArrow ${{ matrix.pyarrow }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        pyarrow: ['16.1.0', '20.0.0', '21.0.0', '22.0.0']
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install test dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e ./paimon-python --group './paimon-python/pyproject.toml:test'
          # Exercise each supported PyArrow version.
          python -m pip install --no-deps 'pyarrow==${{ matrix.pyarrow }}'
      - name: Test Arrow compatibility
        working-directory: paimon-python
        run: |
          python -m pytest -q pypaimon/tests/s3_delete_compatibility_test.py \
            pypaimon/tests/s3_client_compatibility_test.py pypaimon/tests/oss_legacy_mode_test.py \
            pypaimon/tests/file_io_test.py pypaimon/tests/filesystem_catalog_test.py \
            pypaimon/tests/table_update_arrow_compatibility_test.py

  rust-plan:
    name: Native CI
    timeout-minutes: 90
    runs-on: ubuntu-latest
    container: "python:3.11-slim"
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        run: |
          apt-get update && apt-get install -y --no-install-recommends \
            build-essential git curl cmake pkg-config libssl-dev libclang-dev tzdata \
            && rm -rf /var/lib/apt/lists/*

      - name: Install Rust toolchain
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH

      - name: Build paimon-rust from main
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade "git+https://github.com/apache/paimon-rust.git@main#subdirectory=bindings/python"

      - name: Install Python test dependencies
        run: |
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:dev
          python -m pip check

      - name: Verify native wheel provenance
        run: |
          python - <<'PY'
          import importlib.metadata as metadata
          import json

          import pypaimon_rust
          import pypaimon_rust.datafusion

          distribution = metadata.distribution('pypaimon-rust')
          source = json.loads(distribution.read_text('direct_url.json'))
          print('pypaimon-rust', distribution.version,
                'commit', source['vcs_info']['commit_id'])
          assert source['vcs_info']['requested_revision'] == 'main'
          assert callable(getattr(pypaimon_rust, 'temporal_match', None)), (
              'Native CI requires the real temporal_match kernel')
          PY

      - name: Run Python tests with native planning, reading, writing, updating and committing
        shell: bash
        env:
          DAFT_RUNNER: 'native'
          PYTEST_ADDOPTS: '--durations=20'
          PYPAIMON_TEST_NATIVE_PLAN: '1'
          PYPAIMON_TEST_NATIVE_READ: '1'
          PYPAIMON_TEST_NATIVE_WRITE: '1'
          PYPAIMON_TEST_NATIVE_UPDATE: '1'
          PYPAIMON_TEST_NATIVE_COMMIT: '1'
        run: |
          bash paimon-python/dev/lint-python.sh -i pytest

      - name: Verify native temporal casts in non-UTC system zones
        shell: bash
        env:
          PYPAIMON_TEST_NATIVE_PLAN: '1'
          PYPAIMON_TEST_NATIVE_READ: '1'
          PYPAIMON_TEST_NATIVE_WRITE: '1'
          PYPAIMON_TEST_NATIVE_COMMIT: '1'
        run: |
          for zone in Asia/Shanghai America/New_York; do
            TZ="$zone" python -m pytest -q \
              paimon-python/pypaimon/tests/native_cast_completion_test.py \
              paimon-python/pypaimon/tests/native_temporal_zone_cast_test.py \
              paimon-python/pypaimon/tests/native_schema_cast_test.py \
              paimon-python/pypaimon/tests/native_partition_metadata_test.py \
              -k 'TIMESTAMP or temporal'
          done

  lerobot-compatibility:
    name: LeRobot ${{ matrix.lerobot }} / Python ${{ matrix.python }} / TorchCodec ${{ matrix.torchcodec }}
    timeout-minutes: 40
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        include:
          - python: "3.10"
            lerobot: "0.4.4"
            torch: "2.7.1"
            torchvision: "0.22.1"
            torchcodec: "0.3.0"
          - python: "3.10"
            lerobot: "0.4.4"
            torch: "2.6.0"
            torchvision: "0.21.0"
            torchcodec: "0.2.1"
          - python: "3.12"
            lerobot: "0.6.0"
            torch: "2.7.1"
            torchvision: "0.22.1"
            torchcodec: "0.3.0"
          - python: "3.12"
            lerobot: "0.6.1"
            torch: "2.7.1"
            torchvision: "0.22.1"
            torchcodec: "0.3.0"
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python }}
      - name: Install FFmpeg
        run: sudo apt-get update && sudo apt-get install -y ffmpeg
      - name: Install LeRobot dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }} torchcodec==${{ matrix.torchcodec }} --index-url https://download.pytorch.org/whl/cpu
          python -m pip install -e './paimon-python[lerobot]' 'lerobot==${{ matrix.lerobot }}' --group ./paimon-python/pyproject.toml:test
          python -m pip check
      - name: Test import, training reads and writer statistics
        if: matrix.torchcodec != '0.2.1'
        working-directory: paimon-python
        run: |
          python -c 'from lerobot.datasets.lerobot_dataset import LeRobotDataset'
          python -m pytest -q \
            pypaimon/tests/multimodal_lerobot_test.py \
            pypaimon/tests/multimodal_lerobot_writer_test.py::PaimonLeRobotWriterTest::test_multiframe_boolean_statistics_preserve_extrema \
            pypaimon/tests/multimodal_lerobot_writer_test.py::PaimonLeRobotWriterTest::test_multiframe_int64_statistics_preserve_extrema

      - name: Test indexed TorchCodec (no skips)
        working-directory: paimon-python
        run: |
          python -c 'import av; from torchcodec.decoders import VideoDecoder'
          python -m pytest -q pypaimon/tests/indexed_torchcodec_test.py --junitxml=indexed-torchcodec.xml
          python - <<'PYTEST_REPORT'
          import xml.etree.ElementTree as ET
          cases = list(ET.parse("indexed-torchcodec.xml").iter("testcase"))
          assert cases, "No indexed TorchCodec tests ran"
          assert not any(case.find("skipped") is not None for case in cases), "Indexed TorchCodec tests must not skip"
          PYTEST_REPORT

  torch:
    name: PyTorch / Python 3.10
    timeout-minutes: 60
    runs-on: ubuntu-latest
    container: "python:3.10-slim"

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Python version
        run: python --version

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:torch
          python -m pip check
      - name: Verify Python test environment
        run: |
          python - <<'PY'
          import importlib.util

          assert importlib.util.find_spec('pypaimon_rust') is None, 'Rust is tested in Native CI'
          PY

      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -i pytest_torch

  # Check core installation across Python versions, then basic Ray compatibility on 3.10.
  compatibility:
    name: Package and Ray compatibility
    timeout-minutes: 60
    # setup-python provides Python 3.9 builds for Ubuntu 24.04, not 26.04.
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        run: |
          sudo apt-get update && sudo apt-get install -y --no-install-recommends \
            build-essential git curl cmake \
            && sudo rm -rf /var/lib/apt/lists/*

      # python:3.8-slim has no C or C++ compiler, so a dependency that has only an
      # sdist for 3.8 fails here instead of being built with build-essential.
      - name: Verify core installation on Python 3.8 without a compiler
        run: |
          docker run --rm -v "$PWD/paimon-python:/src:ro" python:3.8-slim sh -ec '
            cp -r /src /tmp/paimon-python
            python -m pip install --upgrade pip
            python -m pip install /tmp/paimon-python
          '
          echo "✓ Core package OK on Python 3.8"

      - name: Verify core installation on Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Install core package (3.9)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.9"

      - name: Verify core installation on Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install core package (3.10)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.10"

      - name: Verify core installation on Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install core package (3.11)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.11"

      - name: Verify core installation on Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Install core package (3.12)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.12"

      - name: Verify core installation on Python 3.13
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'
      - name: Install core package (3.13)
        run: |
          cd paimon-python && python -m pip install --upgrade pip
          TEMP_DIR=$(mktemp -d)
          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
          rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.13"

      - name: Setup Python 3.10 for Ray test
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install base Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
          # Use only test tools here: the dev group's Ray >=2.50 floor is for full-feature tests.
          python -m pip install './paimon-python[ray,oss,lumina,vindex,full-text,theta-sketch]' \
            --group ./paimon-python/pyproject.toml:test
      - name: Test Ray version compatibility
        run: |
          cd paimon-python
          echo "=========================================="
          echo "Testing Ray version compatibility"
          echo "=========================================="
          for ray_version in 2.44.0 2.48.0 2.50.1 2.53.0; do
            echo "Testing Ray version: $ray_version"
            python -m pip install --no-cache-dir -q ray==$ray_version
            python -c "import ray; print(f'Ray version: {ray.__version__}')"
            python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
            tests="pypaimon/tests/ray_data_test.py::RayDataTest pypaimon/tests/ray_range_join_test.py::RayRangeJoinTest pypaimon/tests/ray_vector_search_test.py::test_raw_search_matches_exact_top_k"
            tests="$tests pypaimon/tests/ray_batch_vector_search_test.py::test_raw_batch_shares_split_tasks"
            if [ "$ray_version" = "2.50.1" ]; then
              tests="$tests pypaimon/tests/ray_update_by_row_id_test.py::RayUpdateByRowIdTest::test_empty_dataset_after_transform_is_noop"
            fi
            if [ "$ray_version" = "2.44.0" ]; then
              tests="$tests pypaimon/tests/ray_partitioning_test.py::RayPartitioningTest::test_ray_legacy_map_batches_cardinality_is_unknown"
            fi
            if [ "$ray_version" = "2.53.0" ]; then
              tests="$tests pypaimon/tests/ray_partitioning_test.py::RayPartitioningTest::test_ray_metadata_respects_transform_cardinality"
            fi
            python -m pytest $tests -v --tb=short || {
              echo "Tests failed for Ray $ray_version"; python -m pip uninstall -y ray; exit 1;
            }
            python -m pip uninstall -y ray
          done
        env:
          PYTHONPATH: ${{ github.workspace }}/paimon-python