GH-36411: [C++][Python] Use meson-python for PyArrow build system

.github/workflows/dev.yml

@@ -103,7 +103,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit openssl
-          pip install "cython>=3.1" setuptools pytest requests setuptools-scm
+          pip install "cython>=3.1" meson-python pytest requests setuptools-scm
       - name: Run Release Test
         shell: bash
         run: |

.github/workflows/python.yml

@@ -118,7 +118,7 @@ jobs:
         with:
           python-version: 3.12
       - name: Setup Archery
-        run: pip install -e dev/archery[docker]
+        run: pip install -e dev/archery[docker] meson-python
       - name: Execute Docker Build
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
@@ -175,7 +175,7 @@ jobs:
       ARROW_BUILD_TESTS: OFF
       PYARROW_TEST_LARGE_MEMORY: ON
       # Current oldest supported version according to https://endoflife.date/macos
-      MACOSX_DEPLOYMENT_TARGET: 12.0
+      MACOSX_DEPLOYMENT_TARGET: "12.0"
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v6
@@ -288,11 +288,31 @@ jobs:
         env:
           # We can invalidate the current cache by updating this.
           CACHE_VERSION: "2025-09-16.1"
+      - name: Install conda
+        shell: cmd
+        run: |
+          curl -L https://github.com/conda-forge/miniforge/releases/download/25.11.0-0/Miniforge3-25.11.0-0-Windows-x86_64.exe -o miniforge.exe
+          start /wait "" .\miniforge.exe /S /D=C:\miniforge
+          del .\miniforge.exe
+          C:\miniforge\Scripts\conda.exe init cmd.exe
+          echo C:\miniforge\Scripts;C:\miniforge\Library\bin;C:\miniforge\condabin>> %GITHUB_PATH%
+      - name: Create conda environment
+        shell: cmd
+        run: |
+          conda env create -n pyarrow-dev -f .\ci\conda_env_cpp.txt -y
+          conda env update -n pyarrow-dev -f .\ci\conda_env_python.txt
       - name: Build Arrow C++ and PyArrow
         shell: cmd
+        env:
+          PYTHON_CMD: "python"
         run: |
-          call "ci\scripts\python_build.bat" %cd% "${{ steps.path-info.outputs.usr-windows-dir }}"
+          call activate pyarrow-dev || exit /B 1
+          conda list
+          call "ci\scripts\python_build.bat" %cd% "%CONDA_PREFIX%"
       - name: Test PyArrow
         shell: cmd
+        env:
+          PYTHON_CMD: "python"
         run: |
+          call activate pyarrow-dev
           call "ci\scripts\python_test.bat" %cd%

ci/conda_env_cpp.txt

@@ -35,7 +35,7 @@ libboost-devel
 libgrpc
 libprotobuf
 libutf8proc
-lz4-c
+lz4-c>=1.10.0
 make
 meson
 ninja

ci/conda_env_python.txt

@@ -24,9 +24,9 @@ cython>=3.1
 cloudpickle
 fsspec
 hypothesis
+meson-python
 numpy>=1.16.6
 pytest
 pytest-faulthandler
 s3fs>=2023.10.0
-setuptools>=77
 setuptools_scm>=8

ci/conda_env_sphinx.txt

@@ -21,6 +21,7 @@ cython>3.1.1
 doxygen
 ipython
 linkify-it-py
+meson-python
 # We can't install linuxdoc by conda. We install linuxdoc by pip in
 # ci/dockerfiles/conda-python-pandas.dockerfile.
 # linuxdoc

ci/scripts/python_build.bat

@@ -110,30 +110,110 @@ ccache -sv
 
 echo "=== Building Python ==="
 set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
-set PYARROW_BUILD_VERBOSE=1
-set PYARROW_BUNDLE_ARROW_CPP=ON
-set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
-set PYARROW_WITH_ACERO=%ARROW_ACERO%
-set PYARROW_WITH_DATASET=%ARROW_DATASET%
-set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
-set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
-set PYARROW_WITH_GCS=%ARROW_GCS%
-set PYARROW_WITH_HDFS=%ARROW_HDFS%
-set PYARROW_WITH_ORC=%ARROW_ORC%
-set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
-set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
-set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
-set PYARROW_WITH_S3=%ARROW_S3%
-set ARROW_HOME=%CMAKE_INSTALL_PREFIX%
-set CMAKE_PREFIX_PATH=%CMAKE_INSTALL_PREFIX%
+if %ARROW_ACERO% == ON (
+    set PYARROW_WITH_ACERO=enabled
+) else if %ARROW_ACERO% == OFF (
+    set PYARROW_WITH_ACERO=disabled
+) else (
+    set PYARROW_WITH_ACERO=auto
+)
+if %ARROW_DATASET% == ON (
+    set PYARROW_WITH_DATASET=enabled
+) else if %ARROW_DATASET% == OFF (
+    set PYARROW_WITH_DATASET=disabled
+) else (
+    set PYARROW_WITH_DATASET=auto
+)
+if %ARROW_FLIGHT% == ON (
+    set PYARROW_WITH_FLIGHT=enabled
+) else if %ARROW_FLIGHT% == OFF (
+    set PYARROW_WITH_FLIGHT=disabled
+) else (
+    set PYARROW_WITH_FLIGHT=auto
+)
+if %ARROW_GANDIVA% == ON (
+    set PYARROW_WITH_GANDIVA=enabled
+) else if %ARROW_GANDIVA% == OFF (
+    set PYARROW_WITH_GANDIVA=disabled
+) else (
+    set PYARROW_WITH_GANDIVA=auto
+)
+if %ARROW_GCS% == ON (
+    set PYARROW_WITH_GCS=enabled
+) else if %ARROW_GCS% == OFF (
+    set PYARROW_WITH_GCS=disabled
+) else (
+    set PYARROW_WITH_GCS=auto
+)
+if %ARROW_HDFS% == ON (
+    set PYARROW_WITH_HDFS=enabled
+) else if %ARROW_HDFS% == OFF (
+    set PYARROW_WITH_HDFS=disabled
+) else (
+    set PYARROW_WITH_HDFS=auto
+)
+if %ARROW_ORC% == ON (
+    set PYARROW_WITH_ORC=enabled
+) else if %ARROW_ORC% == OFF (
+    set PYARROW_WITH_ORC=disabled
+) else (
+    set PYARROW_WITH_ORC=auto
+)
+if %ARROW_PARQUET% == ON (
+    set PYARROW_WITH_PARQUET=enabled
+) else if %ARROW_PARQUET% == OFF (
+    set PYARROW_WITH_PARQUET=disabled
+) else (
+    set PYARROW_WITH_PARQUET=auto
+)
+if %PARQUET_REQUIRE_ENCRYPTION% == ON (
+    set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
+) else if %ARROW_ACERO% == OFF (
+    set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
+) else (
+    set PYARROW_WITH_PARQUET_ENCRYPTION=auto
+)
+if %ARROW_SUBSTRAIT% == ON (
+    set PYARROW_WITH_SUBSTRAIT=enabled
+) else if %ARROW_SUBSTRAIT% == OFF (
+    set PYARROW_WITH_SUBSTRAIT=disabled
+) else (
+    set PYARROW_WITH_SUBSTRAIT=auto
+)
+if %ARROW_S3% == ON (
+    set PYARROW_WITH_S3=enabled
+) else if %ARROW_S3% == OFF (
+    set PYARROW_WITH_S3=disabled
+) else (
+    set PYARROW_WITH_S3=auto
+)
+if %CMAKE_BUILD_TYPE% == Release (
+    set MESON_BUILD_TYPE=release
+) else (
+    set MESON_BUILD_TYPE=debug
+)
 
 pushd %SOURCE_DIR%\python
 
 @REM Install Python build dependencies
 %PYTHON_CMD% -m pip install --upgrade pip || exit /B 1
 %PYTHON_CMD% -m pip install -r requirements-build.txt || exit /B 1
+%PYTHON_CMD% -m pip install build || exit /B 1
 
 @REM Build PyArrow
-%PYTHON_CMD% -m pip install --no-deps --no-build-isolation -vv . || exit /B 1
+%PYTHON_CMD% -m build --wheel --no-isolation . ^
+    -Csetup-args="-Dbuildtype=%MESON_BUILD_TYPE%" ^
+    -Csetup-args="-Dacero=%PYARROW_WITH_ACERO%" ^
+    -Csetup-args="-Ddataset=%PYARROW_WITH_DATASET%" ^
+    -Csetup-args="-Dflight=%PYARROW_WITH_FLIGHT%" ^
+    -Csetup-args="-Dgandiva=%PYARROW_WITH_GANDIVA%" ^
+    -Csetup-args="-Dgcs=%PYARROW_WITH_GCS%" ^
+    -Csetup-args="-Dhdfs=%PYARROW_WITH_HDFS%" ^
+    -Csetup-args="-Dorc=%PYARROW_WITH_ORC%" ^
+    -Csetup-args="-Dparquet=%PYARROW_WITH_PARQUET%" ^
+    -Csetup-args="-Dparquet_require_encryption=%PYARROW_WITH_PARQUET_ENCRYPTION%" ^
+    -Csetup-args="-Dsubstrait=%PYARROW_WITH_SUBSTRAIT%" ^
+    -Csetup-args="-Ds3=%PYARROW_WITH_S3%" || exit /B 1
+%PYTHON_CMD% -m pip install --no-index --find-links .\dist\ pyarrow
 
 popd

ci/scripts/python_build.sh

@@ -39,44 +39,76 @@ if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
   . "${ARROW_PYTHON_VENV}/bin/activate"
 fi
 
-case "$(uname)" in
-  Linux)
-    n_jobs=$(nproc)
-    ;;
-  Darwin)
-    n_jobs=$(sysctl -n hw.ncpu)
-    ;;
-  MINGW*)
-    n_jobs=${NUMBER_OF_PROCESSORS:-1}
-    ;;
-  *)
-    n_jobs=${NPROC:-1}
-    ;;
-esac
-
 if [ -n "${CONDA_PREFIX}" ]; then
   echo -e "===\n=== Conda environment for build\n==="
   conda list
 fi
 
-export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
-export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
-
-export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
-export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
-export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
-export PYARROW_WITH_DATASET=${ARROW_DATASET:-ON}
-export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT:-OFF}
-export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA:-OFF}
-export PYARROW_WITH_GCS=${ARROW_GCS:-OFF}
-export PYARROW_WITH_HDFS=${ARROW_HDFS:-ON}
-export PYARROW_WITH_ORC=${ARROW_ORC:-OFF}
-export PYARROW_WITH_PARQUET=${ARROW_PARQUET:-OFF}
-export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
-export PYARROW_WITH_S3=${ARROW_S3:-OFF}
-export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}
-
-export PYARROW_PARALLEL=${n_jobs}
+PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                     esac)
+PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                     esac)
+PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                    esac)
+PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "enabled" ;;
+                       esac)
+PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                      esac)
+PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                       esac)
+PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                   esac)
+PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "enabled" ;;
+                    esac)
+PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                   esac)
+PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                       esac)
+PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "enabled" ;;
+                                  esac)
+PYARROW_WITH_S3=$(case "$ARROW_S3" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                  esac)
+PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
+                         ON) echo "enabled" ;;
+                         OFF) echo "disabled" ;;
+                         *) echo "auto" ;;
+                    esac)
 
 : "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
 export CMAKE_PREFIX_PATH
@@ -93,7 +125,40 @@ pushd "${python_build_dir}"
 #   on Debian/Ubuntu (ARROW-15243).
 # - Cannot use build isolation as we want to use specific dependency versions
 #   (e.g. Numpy, Pandas) on some CI jobs.
-${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv .
+
+# The conda compilers package adds flags for debugging and optimization
+# that are unnecessary in a CI context
+OLD_CFLAGS=$CFLAGS
+OLD_CPPFLAGS=$CPPFLAGS
+OLD_CXXFLAGS=$CXXFLAGS
+export CFLAGS=
+export CPPFLAGS=
+export CXXFLAGS=
+
+BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
+BUILD_TYPE=${BUILD_TYPE,,}  # Meson requires lowercase values
+
+${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv . \
+                  -Csetup-args="-Dbuildtype=${BUILD_TYPE}" \
+                  -Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
+                  -Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
+                  -Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
+                  -Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
+                  -Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
+                  -Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
+                  -Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
+                  -Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
+                  -Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
+                  -Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
+                  -Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
+                  -Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
+                  -Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
+                  -Ccompile-args="-v" \
+                  -Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"
+
+export CFLAGS=$OLD_CFLAGS
+export CPPFLAGS=$OLD_CPPFLAGS
+export CXXFLAGS=$OLD_CXXFLAGS
 popd
 
 if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then

ci/scripts/python_sdist_build.sh

@@ -23,5 +23,9 @@ source_dir=${1}/python
 
 pushd "${source_dir}"
 export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
-${PYTHON:-python} setup.py sdist
+# Meson dist must be run from a VCS, so initiate a dummy repo
+git init .
+git add --all .
+git commit -m "dummy commit for meson dist"
+${PYTHON:-python} -m build --sdist .
 popd

cpp/cmake_modules/FindSnappyAlt.cmake

@@ -75,10 +75,6 @@ if(ARROW_SNAPPY_USE_SHARED)
        "${CMAKE_SHARED_LIBRARY_PREFIX}snappy${CMAKE_SHARED_LIBRARY_SUFFIX}")
 else()
   set(SNAPPY_STATIC_LIB_NAME_BASE "snappy")
-  if(MSVC)
-    set(SNAPPY_STATIC_LIB_NAME_BASE
-        "${SNAPPY_STATIC_LIB_NAME_BASE}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}")
-  endif()
   set(SNAPPY_LIB_NAMES
       "${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}"
   )

cpp/cmake_modules/Findutf8proc.cmake

@@ -73,9 +73,6 @@ if(ARROW_UTF8PROC_USE_SHARED)
   list(APPEND utf8proc_LIB_NAMES
        "${CMAKE_SHARED_LIBRARY_PREFIX}utf8proc${CMAKE_SHARED_LIBRARY_SUFFIX}")
 else()
-  if(MSVC AND NOT DEFINED utf8proc_MSVC_STATIC_LIB_SUFFIX)
-    set(utf8proc_MSVC_STATIC_LIB_SUFFIX "_static")
-  endif()
   set(utf8proc_STATIC_LIB_SUFFIX
       "${utf8proc_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(utf8proc_LIB_NAMES