unit-test-python.yml

# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven

name: Unit-Test-Py

on:
  push:
    branches:
      - develop
      - iotdb
      - rc/*
    paths-ignore:
      - 'docs/**'
      - 'java/**'
  pull_request:
    branches:
      - develop
      - dev/*
      - iotdb
      - rc/*
    paths-ignore:
      - 'docs/**'
      - 'java/**'
  # Enable manually starting builds, and allow forcing updating of SNAPSHOT dependencies.
  workflow_dispatch:
    inputs:
      forceUpdates:
        description: "Forces a snapshot update"
        required: false
        default: 'false'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
  DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}

jobs:
  unit-test:
    strategy:
      fail-fast: false
      max-parallel: 15
      matrix:
        # Keep Python 3.9 available on Linux as ubuntu-latest advances.
        os: [ ubuntu-24.04, macos-latest, windows-latest ]
        python-version: ["3.9", "3.11", "3.14"]
    runs-on: ${{ matrix.os }}

    steps:

      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: ${{ matrix.python-version }}

      - name: Set up JDK 17
        uses: actions/setup-java@v6.1.0
        with:
          distribution: corretto
          java-version: 17

      # Setup caching of the artifacts in the .m2 directory, so they don't have to
      # all be downloaded again for every build.
      - name: Cache Maven packages
        uses: actions/cache@v6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-

      # On Windows systems the 'mvnw' script needs an additional ".cmd" appended.
      - name: Calculate platform suffix
        id: platform_suffix
        uses: actions/github-script@v9.0.0
        env:
          OS: ${{ matrix.os }}
        with:
          script: |
            const { OS } = process.env
            if (OS.includes("windows")) {
              core.setOutput('platform_suffix', `.cmd`)
            } else {
              core.setOutput('platform_suffix', ``)
            }

      # Install dependencies
      - name: Install dependencies
        shell: bash
        run: |
          if [[ "$RUNNER_OS" == "Linux" ]]; then
            python -m venv "$RUNNER_TEMP/clang-format"
            "$RUNNER_TEMP/clang-format/bin/python" -m pip install --only-binary=:all: clang-format==17.0.6
            "$RUNNER_TEMP/clang-format/bin/clang-format" --version
            echo "$RUNNER_TEMP/clang-format/bin" >> "$GITHUB_PATH"
            sudo apt-get update
            sudo apt-get install -y uuid-dev
          elif [[ "$RUNNER_OS" == "Windows" ]]; then
            choco install llvm --version 17.0.6 --force
          else
            brew install llvm@17
            ln -sf $(brew --prefix llvm@17)/bin/clang-format /opt/homebrew/bin/clang-format
          fi


      # Run the actual maven build including all tests.
      - name: Build and test with Maven
        shell: bash
        run: |
          # On Windows, prepend MinGW bin so CTest/test exe uses the correct
          # runtime DLLs instead of Git-for-Windows mingw runtime from PATH.
          if [[ "$RUNNER_OS" == "Windows" ]]; then
            export PATH="/c/mingw64/bin:$PATH"
          fi
          BLACK_VER_PROP=""
          if [[ "${{ matrix.python-version }}" == "3.9" ]]; then
            BLACK_VER_PROP="-Dblack.version=25.11.0"
          fi
          ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-python -Denable.asan=OFF -Dbuild.type=Release clean verify ${BLACK_VER_PROP}

      - name: Upload whl Artifact
        uses: actions/upload-artifact@v7
        with:
          name: tsfile-${{ runner.os }}-py${{ matrix.python-version }}-whl
          path: python/dist/tsfile-*.whl
          retention-days: 1

  # AddressSanitizer/LeakSanitizer coverage for the complete Python -> Cython
  # -> C wrapper -> C++ chain. The build instruments libtsfile and every
  # extension with the same compiler and check set; the scenarios run in fresh
  # interpreters with leak detection on. See python/SANITIZER.md.
  python-sanitizer:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.11"

      - name: Set up JDK 17
        uses: actions/setup-java@v6.0.1
        with:
          distribution: corretto
          java-version: 17

      - name: Cache Maven packages
        uses: actions/cache@v6
        with:
          path: ~/.m2
          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-m2-

      - name: Install dependencies
        shell: bash
        run: |
          sudo apt-get update
          sudo apt-get install -y uuid-dev binutils

      - name: Build with ASan/LSan
        shell: bash
        run: |
          # One compiler (and therefore one sanitizer runtime) for CMake,
          # distutils and the preload below.
          export CC=gcc CXX=g++
          ./mvnw -P with-python \
            -Dtsfile.sanitizer=address \
            -Dbuild.type=RelWithDebInfo \
            -Dbuild.test=OFF \
            -Dspotless.skip=true \
            clean test-compile

      - name: Verify sanitizer instrumentation
        shell: bash
        run: |
          python/venv/bin/python python/check_sanitizer_build.py python/tsfile

      - name: Run ownership scenarios under ASan/LSan
        shell: bash
        run: |
          set -o pipefail
          # Suppressions must stay scoped to third-party/interpreter frames.
          if grep -v '^#' python/tests/sanitizer/lsan_suppressions.txt \
              | grep -Eqi 'tsfile|dataset/_merge|dataset/_index'; then
            echo "::error::suppression file must not match TsFile modules"
            exit 1
          fi
          ASAN_LIB="$(gcc -print-file-name=libasan.so)"
          LIBSTDCXX="$(gcc -print-file-name=libstdc++.so.6)"
          if [ ! -f "$ASAN_LIB" ] || [ ! -f "$LIBSTDCXX" ]; then
            echo "::error::AddressSanitizer or libstdc++ runtime not found for gcc"
            exit 1
          fi
          # The pytest runner itself does not load tsfile; each scenario child
          # re-enables leak detection (see test_sanitizer_ownership.py).
          #
          # libasan must stay first (runtime-first check); libstdc++ follows so
          # ASan's __cxa_throw interceptor can resolve the real symbol. Without
          # it, any C++ exception aborts with "CHECK failed ... __cxa_throw".
          export TSFILE_SANITIZER=address
          export ASAN_OPTIONS="detect_leaks=0:abort_on_error=1:halt_on_error=1"
          export LSAN_OPTIONS="suppressions=$PWD/python/tests/sanitizer/lsan_suppressions.txt"
          LD_PRELOAD="$ASAN_LIB:$LIBSTDCXX" python/venv/bin/python -m pytest \
            python/tests/test_sanitizer_ownership.py -v