From b2e142b057b25b162972474fb3bb85f0bb0acfb6 Mon Sep 17 00:00:00 2001 From: Grufoony Date: Fri, 5 Dec 2025 16:25:50 +0100 Subject: [PATCH 01/16] Download a pre-built tbb --- CMakeLists.txt | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b447bb70..054cb908 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,17 +130,16 @@ if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) endif() # Get TBB -set(TBB_TEST - OFF - CACHE BOOL "Disable TBB tests" FORCE) +set(TBB_TEST OFF CACHE BOOL "Disable TBB tests" FORCE) +set(TBB_EXAMPLES OFF CACHE BOOL "Disable TBB examples" FORCE) +set(TBB_STRICT OFF CACHE BOOL "Disable TBB strict mode" FORCE) + FetchContent_Declare( tbb - GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB - GIT_TAG v2022.3.0) -FetchContent_GetProperties(tbb) -if(NOT tbb_POPULATED) - FetchContent_MakeAvailable(tbb) -endif() + GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB.git + GIT_TAG v2022.3.0 +) +FetchContent_MakeAvailable(tbb) add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) @@ -177,6 +176,12 @@ install( if(BUILD_PYTHON_BINDINGS) include(FetchContent) + # Check if Doxygen is available for documentation generation + find_package(Doxygen REQUIRED) + if(NOT DOXYGEN_FOUND) + message(FATAL_ERROR "Doxygen is required to build the docstrings.") + endif() + # Get pybind11 FetchContent_Declare( pybind11 From aa49e6ddb473f0d3e3408c3f17dbb5c1a7735492 Mon Sep 17 00:00:00 2001 From: Grufoony Date: Fri, 5 Dec 2025 17:26:39 +0100 Subject: [PATCH 02/16] These should not be necessary anymore --- .github/workflows/binding.yml | 4 ++-- CMakeLists.txt | 8 ++++++++ setup.py | 26 ++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index f2772285..b42cee95 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -32,7 +32,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen - name: Install system dependencies (macOS) if: matrix.os == 'macos-latest' @@ -189,7 +189,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen - name: Install development dependencies run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 054cb908..ca48c201 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() if(BUILD_PYTHON_BINDINGS) set(DSF_BUILD_PIC ON) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() # Set the C++ standard @@ -133,6 +134,8 @@ endif() set(TBB_TEST OFF CACHE BOOL "Disable TBB tests" FORCE) set(TBB_EXAMPLES OFF CACHE BOOL "Disable TBB examples" FORCE) set(TBB_STRICT OFF CACHE BOOL "Disable TBB strict mode" FORCE) +# set(TBB_BUILD_SHARED ON CACHE BOOL "Build TBB as shared library" FORCE) +# set(TBB_BUILD_TBBMALLOC OFF CACHE BOOL "Disable TBB malloc" FORCE) FetchContent_Declare( tbb @@ -225,6 +228,11 @@ if(BUILD_PYTHON_BINDINGS) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "${HOMEBREW_LIB}" INSTALL_RPATH_USE_LINK_PATH TRUE) + elseif(UNIX) + set_target_properties( + dsf_python_module + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN") endif() endif() diff --git a/setup.py b/setup.py index c3eda7ed..814e1e58 100644 --- a/setup.py +++ b/setup.py @@ -123,6 +123,32 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) + # Copy TBB shared library if it exists (Linux only) + if platform.system() == "Linux": + print(f"Searching for TBB shared libraries in {build_temp}...") + # Look for libtbb.so* recursively + tbb_libs = list(build_temp.glob("**/libtbb.so*")) + # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built + tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) + + if tbb_libs: + print(f"Found TBB libraries: {tbb_libs}") + for lib in tbb_libs: + # We only want the real shared object, not symlinks if possible, + # but copying everything matching the pattern is safer to ensure we get the versioned one. + # However, we need to be careful not to overwrite if multiple matches found. + # Usually we want the one that the linker linked against. + # Since we set RPATH to $ORIGIN, we need the library in the same dir as the extension. + + # Avoid copying if it's a symlink pointing to something we already copied? + # simpler: just copy all of them. + dest = extdir / lib.name + if not dest.exists(): + shutil.copy2(lib, dest) + print(f"Copied {lib} to {dest}") + else: + print("Warning: No TBB shared libraries found to copy.") + def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" From ddc619bc9886d32b08faf7547c58e3093dd684d7 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 10:30:00 +0100 Subject: [PATCH 03/16] Is this necessary? --- setup.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/setup.py b/setup.py index 814e1e58..c3eda7ed 100644 --- a/setup.py +++ b/setup.py @@ -123,32 +123,6 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) - # Copy TBB shared library if it exists (Linux only) - if platform.system() == "Linux": - print(f"Searching for TBB shared libraries in {build_temp}...") - # Look for libtbb.so* recursively - tbb_libs = list(build_temp.glob("**/libtbb.so*")) - # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built - tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) - - if tbb_libs: - print(f"Found TBB libraries: {tbb_libs}") - for lib in tbb_libs: - # We only want the real shared object, not symlinks if possible, - # but copying everything matching the pattern is safer to ensure we get the versioned one. - # However, we need to be careful not to overwrite if multiple matches found. - # Usually we want the one that the linker linked against. - # Since we set RPATH to $ORIGIN, we need the library in the same dir as the extension. - - # Avoid copying if it's a symlink pointing to something we already copied? - # simpler: just copy all of them. - dest = extdir / lib.name - if not dest.exists(): - shutil.copy2(lib, dest) - print(f"Copied {lib} to {dest}") - else: - print("Warning: No TBB shared libraries found to copy.") - def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" From a6d96c906bd266167693907929ebc1c276afee05 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 10:34:09 +0100 Subject: [PATCH 04/16] Yes it is --- setup.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/setup.py b/setup.py index c3eda7ed..6df1b794 100644 --- a/setup.py +++ b/setup.py @@ -123,6 +123,32 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) + # Copy TBB shared library if it exists (Linux only) + if platform.system() == "Linux": + print(f"Searching for TBB shared libraries in {build_temp}...") + # Look for libtbb.so* recursively + tbb_libs = list(build_temp.glob("**/libtbb.so*")) + # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built + tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) + + if tbb_libs: + print(f"Found TBB libraries: {tbb_libs}") + for lib in tbb_libs: + # We only want the real shared object, not symlinks if possible, + # but copying everything matching the pattern is safer to ensure we get the versioned one. + # However, we need to be careful not to overwrite if multiple matches found. + # Usually we want the one that the linker linked against. + # Since we set RPATH to $ORIGIN, we need the library in the same dir as the extension. + + # Avoid copying if it's a symlink pointing to something we already copied? + # simpler: just copy all of them. + dest = extdir / lib.name + if not dest.exists(): + shutil.copy2(lib, dest) + print(f"Copied {lib} to {dest}") + else: + print("Warning: No TBB shared libraries found to copy.") + def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" From eeca1e35edbe7b6101c63d10da1e2a54e96eaa38 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 10:34:53 +0100 Subject: [PATCH 05/16] Not necessary --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca48c201..197ba3b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,6 @@ if(NOT CMAKE_BUILD_TYPE) endif() if(BUILD_PYTHON_BINDINGS) set(DSF_BUILD_PIC ON) - set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() # Set the C++ standard From 6470f0430a1536abebafed69d5552ead9064ff21 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 10:55:25 +0100 Subject: [PATCH 06/16] What about this? --- CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 197ba3b3..ad854b47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,11 +227,11 @@ if(BUILD_PYTHON_BINDINGS) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "${HOMEBREW_LIB}" INSTALL_RPATH_USE_LINK_PATH TRUE) - elseif(UNIX) - set_target_properties( - dsf_python_module - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "$ORIGIN") + # elseif(UNIX) + # set_target_properties( + # dsf_python_module + # PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + # INSTALL_RPATH "$ORIGIN") endif() endif() From 4c5aeb90d1f56c391e269162c064fb7953136375 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 11:01:57 +0100 Subject: [PATCH 07/16] Yes it is --- CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ad854b47..197ba3b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,11 +227,11 @@ if(BUILD_PYTHON_BINDINGS) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "${HOMEBREW_LIB}" INSTALL_RPATH_USE_LINK_PATH TRUE) - # elseif(UNIX) - # set_target_properties( - # dsf_python_module - # PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - # INSTALL_RPATH "$ORIGIN") + elseif(UNIX) + set_target_properties( + dsf_python_module + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN") endif() endif() From 920d40eab1dabe277311f60c8fe276d31b4f356c Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 11:03:39 +0100 Subject: [PATCH 08/16] Remove apt installations --- .github/workflows/cmake_examples.yml | 2 +- .github/workflows/pypi.yml | 4 ++-- .github/workflows/pytest.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cmake_examples.yml b/.github/workflows/cmake_examples.yml index 70bef7c4..80fbbe80 100644 --- a/.github/workflows/cmake_examples.yml +++ b/.github/workflows/cmake_examples.yml @@ -23,7 +23,7 @@ jobs: run: | if [ ${{ matrix.os }} == 'ubuntu-latest' ]; then sudo apt update - sudo apt install libtbb-dev libsimdjson-dev binutils + sudo apt install binutils elif [ ${{ matrix.os }} == 'macos-latest' ]; then brew install tbb simdjson fi diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index e5edddda..d9f3d2c4 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -79,7 +79,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen libtbb-dev libfmt-dev libspdlog-dev libsimdjson-dev + sudo apt install -y cmake build-essential doxygen - name: Install build dependencies run: | @@ -174,7 +174,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen - name: Install build dependencies run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 17a89619..213f63c8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen python -m pip install --upgrade pip pip install pytest pytest-cov From 5c2e0f2b69306ebb65ba54f8baa7a31c4c899055 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 11:19:40 +0100 Subject: [PATCH 09/16] Also this should be removed --- .github/workflows/pypi.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index d9f3d2c4..e7e2f5b8 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -127,7 +127,7 @@ jobs: - name: Install system dependencies run: | - brew install cmake tbb spdlog simdjson doxygen + brew install cmake doxygen - name: Install build dependencies run: | @@ -136,9 +136,6 @@ jobs: - name: Set macOS environment variables run: | - echo "CPLUS_INCLUDE_PATH=$(brew --prefix tbb)/include:$(brew --prefix fmt)/include:$(brew --prefix spdlog)/include:$(brew --prefix simdjson)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - echo "CMAKE_PREFIX_PATH=$(brew --prefix tbb);$(brew --prefix fmt);$(brew --prefix spdlog);$(brew --prefix simdjson)" >> $GITHUB_ENV echo "ARCHFLAGS=-arch $(uname -m)" >> $GITHUB_ENV echo "_PYTHON_HOST_PLATFORM=macosx-$(sw_vers -productVersion | cut -d. -f1)-$(uname -m)" >> $GITHUB_ENV From a30d23748c7f19ad292e7b65ff305ab711ebb2f1 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 11:28:50 +0100 Subject: [PATCH 10/16] Proviaml --- .github/workflows/binding.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index b42cee95..61a856cc 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -37,22 +37,13 @@ jobs: - name: Install system dependencies (macOS) if: matrix.os == 'macos-latest' run: | - brew install cmake tbb spdlog simdjson doxygen - echo "CPLUS_INCLUDE_PATH=$(brew --prefix tbb)/include:$(brew --prefix fmt)/include:$(brew --prefix spdlog)/include:$(brew --prefix simdjson)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV + brew install cmake doxygen - name: Install system dependencies (Windows) if: matrix.os == 'windows-latest' run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install - # Install vcpkg for spdlog and tbb on Windows - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg install spdlog:x64-windows tbb:x64-windows - echo "VCPKG_ROOT=$env:GITHUB_WORKSPACE\vcpkg" >> $env:GITHUB_ENV - name: Upgrade pip and install build dependencies run: | From 6a45c5627fa387926a8b66d1337f0719f520f984 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 11:45:00 +0100 Subject: [PATCH 11/16] Try to fix macos bindings --- CMakeLists.txt | 2 +- setup.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 197ba3b3..0bfd8d5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,7 +225,7 @@ if(BUILD_PYTHON_BINDINGS) set_target_properties( dsf_python_module PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "${HOMEBREW_LIB}" + INSTALL_RPATH "${HOMEBREW_LIB};@loader_path" INSTALL_RPATH_USE_LINK_PATH TRUE) elseif(UNIX) set_target_properties( diff --git a/setup.py b/setup.py index 6df1b794..af24cd45 100644 --- a/setup.py +++ b/setup.py @@ -123,13 +123,19 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) - # Copy TBB shared library if it exists (Linux only) - if platform.system() == "Linux": + # Copy TBB shared library if it exists (Linux and macOS) + if platform.system() == "Linux" or platform.system() == "Darwin": print(f"Searching for TBB shared libraries in {build_temp}...") - # Look for libtbb.so* recursively - tbb_libs = list(build_temp.glob("**/libtbb.so*")) - # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built - tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) + + tbb_libs = [] + if platform.system() == "Linux": + # Look for libtbb.so* recursively + tbb_libs = list(build_temp.glob("**/libtbb.so*")) + # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built + tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) + else: # macOS + # Look for libtbb.dylib* recursively + tbb_libs = list(build_temp.glob("**/libtbb*.dylib")) if tbb_libs: print(f"Found TBB libraries: {tbb_libs}") @@ -138,7 +144,7 @@ def build_extension(self, ext: CMakeExtension): # but copying everything matching the pattern is safer to ensure we get the versioned one. # However, we need to be careful not to overwrite if multiple matches found. # Usually we want the one that the linker linked against. - # Since we set RPATH to $ORIGIN, we need the library in the same dir as the extension. + # Since we set RPATH to $ORIGIN (Linux) or @loader_path (macOS), we need the library in the same dir as the extension. # Avoid copying if it's a symlink pointing to something we already copied? # simpler: just copy all of them. From 5bcdeb95f0a966323c4c858c244f13bc004b1741 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 11:53:30 +0100 Subject: [PATCH 12/16] Last things to remove --- .github/workflows/cmake_examples.yml | 2 -- .github/workflows/cmake_tests.yml | 7 ------- .github/workflows/codeql.yml | 17 +---------------- 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/cmake_examples.yml b/.github/workflows/cmake_examples.yml index 80fbbe80..e1e0b15a 100644 --- a/.github/workflows/cmake_examples.yml +++ b/.github/workflows/cmake_examples.yml @@ -24,8 +24,6 @@ jobs: if [ ${{ matrix.os }} == 'ubuntu-latest' ]; then sudo apt update sudo apt install binutils - elif [ ${{ matrix.os }} == 'macos-latest' ]; then - brew install tbb simdjson fi - uses: actions/checkout@v4 diff --git a/.github/workflows/cmake_tests.yml b/.github/workflows/cmake_tests.yml index 8f521d02..458bd0b5 100644 --- a/.github/workflows/cmake_tests.yml +++ b/.github/workflows/cmake_tests.yml @@ -24,12 +24,6 @@ jobs: sudo apt update sudo apt install -y lcov gcovr build-essential cmake libhwloc-dev - - name: Install dependencies on macOS - if: matrix.os == 'macos-latest' - run: | - brew update - brew install tbb simdjson - - name: Install dependencies on Windows (vcpkg) if: matrix.os == 'windows-latest' shell: powershell @@ -56,7 +50,6 @@ jobs: mkdir -p build cd build cmake .. \ - -DCMAKE_PREFIX_PATH="$(brew --prefix tbb):$(brew --prefix simdjson)" \ -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/install cmake --build . -j$(sysctl -n hw.ncpu) --config Debug diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3b1daac1..4b306b84 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,17 +23,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install TBB (MacOS & Ubuntu) - run: | - if [[ "$RUNNER_OS" == "macOS" ]]; then - brew install tbb spdlog simdjson - echo "CPLUS_INCLUDE_PATH=$(brew --prefix tbb)/include:$(brew --prefix fmt)/include:$(brew --prefix spdlog)/include:$(brew --prefix simdjson)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - elif [[ "$RUNNER_OS" == "Linux" ]]; then - sudo apt update && sudo apt install -y libtbb-dev libspdlog-dev libsimdjson-dev - fi - - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: @@ -42,11 +31,7 @@ jobs: - name: Build C++ run: | mkdir -p build && cd build - if [[ "$RUNNER_OS" == "macOS" ]]; then - cmake .. -DCMAKE_PREFIX_PATH="$(brew --prefix tbb);$(brew --prefix fmt);$(brew --prefix spdlog)" - else - cmake .. - fi + cmake .. cmake --build . sudo make install From eb31c4a0533629325eb950c99d891fcc9adae781 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 12:00:47 +0100 Subject: [PATCH 13/16] Add windows support for python --- .github/workflows/binding.yml | 2 +- .github/workflows/pypi.yml | 39 ++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index 61a856cc..16a63662 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] # , windows-latest --- IGNORE --- + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.12"] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index e7e2f5b8..5f13cd71 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -153,6 +153,43 @@ jobs: name: wheel-macos-${{ matrix.python-version }} path: wheelhouse/*.whl + build-wheels-windows: + name: Build wheels on Windows (Python ${{ matrix.python-version }}) + needs: [check-version] + runs-on: windows-latest + if: needs.check-version.outputs.should_build == 'true' + strategy: + matrix: + python-version: ['3.10', '3.12'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + choco install doxygen.install + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build wheel setuptools pybind11-stubgen + + - name: Build wheel + run: python -m build --wheel + + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-windows-${{ matrix.python-version }} + path: dist/*.whl + build-sdist: name: Build source distribution needs: [check-version] @@ -189,7 +226,7 @@ jobs: publish: name: Publish to PyPI - needs: [check-version, build-wheels-linux, build-wheels-macos, build-sdist] + needs: [check-version, build-wheels-linux, build-wheels-macos, build-wheels-windows, build-sdist] runs-on: ubuntu-latest if: needs.check-version.outputs.should_build == 'true' From ed113548676449cc327f2cdca7e13c778baf5d8f Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 12:10:48 +0100 Subject: [PATCH 14/16] Fix? --- CMakeLists.txt | 7 +++---- setup.py | 32 ++++++++++++++++++++++++++++++++ src/dsf/__init__.py | 15 +++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bfd8d5f..19ddb225 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,15 +195,14 @@ if(BUILD_PYTHON_BINDINGS) endif() # Add the Python binding module - add_library(dsf_python_module MODULE src/dsf/bindings.cpp) + pybind11_add_module(dsf_python_module src/dsf/bindings.cpp) # Ensure the Python module name has no 'lib' prefix on Unix systems - set_target_properties(dsf_python_module PROPERTIES PREFIX "" OUTPUT_NAME - "dsf_cpp") + set_target_properties(dsf_python_module PROPERTIES OUTPUT_NAME "dsf_cpp") # Link the pybind11 module with your static library and pybind11 target_link_libraries( - dsf_python_module PRIVATE dsf pybind11::module pybind11::headers TBB::tbb + dsf_python_module PRIVATE dsf pybind11::headers TBB::tbb spdlog::spdlog) target_compile_definitions(dsf_python_module PRIVATE SPDLOG_USE_STD_FORMAT) diff --git a/setup.py b/setup.py index af24cd45..ed4d20d0 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,11 @@ def build_extension(self, ext: CMakeExtension): "-DBUILD_PYTHON_BINDINGS=ON", ] + if platform.system() == "Windows": + cmake_args += [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" + ] + # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS try: @@ -155,6 +160,33 @@ def build_extension(self, ext: CMakeExtension): else: print("Warning: No TBB shared libraries found to copy.") + elif platform.system() == "Windows": + print(f"Searching for TBB DLLs in {build_temp}...") + # Look for tbb*.dll recursively + tbb_dlls = list(build_temp.glob("**/tbb*.dll")) + + if tbb_dlls: + print(f"Found TBB DLLs: {tbb_dlls}") + # We want to copy them to the 'dsf' package directory so we can load them in __init__.py + # extdir is where dsf_cpp.pyd is (site-packages root usually) + # We want site-packages/dsf/ + + # self.build_lib is usually the root of the build (e.g. build/lib.win...) + # So we can construct the path to dsf package + dsf_pkg_dir = Path(self.build_lib) / "dsf" + dsf_pkg_dir.mkdir(parents=True, exist_ok=True) + + # Also copy to source directory for editable installs + source_dsf_dir = Path(__file__).parent / "src" / "dsf" + + for dll in tbb_dlls: + print(f"Copying {dll} to {dsf_pkg_dir}") + shutil.copy2(dll, dsf_pkg_dir) + print(f"Copying {dll} to {source_dsf_dir}") + shutil.copy2(dll, source_dsf_dir) + else: + print("Warning: No TBB DLLs found. This might cause import errors.") + def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" diff --git a/src/dsf/__init__.py b/src/dsf/__init__.py index 4a7c999d..2a70a963 100644 --- a/src/dsf/__init__.py +++ b/src/dsf/__init__.py @@ -1,3 +1,18 @@ +import sys +import os + +# On Windows, we need to explicitly load the bundled TBB DLLs +if sys.platform == "win32": + import glob + import ctypes + # Look for tbb dlls in the same directory as this __init__.py + _dll_dir = os.path.dirname(__file__) + for _dll in glob.glob(os.path.join(_dll_dir, "tbb*.dll")): + try: + ctypes.CDLL(_dll) + except Exception as e: + print(f"Warning: Failed to load {_dll}: {e}") + from dsf_cpp import __version__, LogLevel, get_log_level, set_log_level, mobility, mdt from .python.cartography import ( From f3bf332dc8302dc1f2dc7f47423a7fccae9164b2 Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 13:05:51 +0100 Subject: [PATCH 15/16] Formatting --- setup.py | 12 +++++------- src/dsf/__init__.py | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index ed4d20d0..e1c5e735 100644 --- a/setup.py +++ b/setup.py @@ -79,9 +79,7 @@ def build_extension(self, ext: CMakeExtension): ] if platform.system() == "Windows": - cmake_args += [ - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" - ] + cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS @@ -164,21 +162,21 @@ def build_extension(self, ext: CMakeExtension): print(f"Searching for TBB DLLs in {build_temp}...") # Look for tbb*.dll recursively tbb_dlls = list(build_temp.glob("**/tbb*.dll")) - + if tbb_dlls: print(f"Found TBB DLLs: {tbb_dlls}") # We want to copy them to the 'dsf' package directory so we can load them in __init__.py # extdir is where dsf_cpp.pyd is (site-packages root usually) # We want site-packages/dsf/ - + # self.build_lib is usually the root of the build (e.g. build/lib.win...) # So we can construct the path to dsf package dsf_pkg_dir = Path(self.build_lib) / "dsf" dsf_pkg_dir.mkdir(parents=True, exist_ok=True) - + # Also copy to source directory for editable installs source_dsf_dir = Path(__file__).parent / "src" / "dsf" - + for dll in tbb_dlls: print(f"Copying {dll} to {dsf_pkg_dir}") shutil.copy2(dll, dsf_pkg_dir) diff --git a/src/dsf/__init__.py b/src/dsf/__init__.py index 2a70a963..e35d3d92 100644 --- a/src/dsf/__init__.py +++ b/src/dsf/__init__.py @@ -5,6 +5,7 @@ if sys.platform == "win32": import glob import ctypes + # Look for tbb dlls in the same directory as this __init__.py _dll_dir = os.path.dirname(__file__) for _dll in glob.glob(os.path.join(_dll_dir, "tbb*.dll")): From 6f83c8f4ffcfa96240ba78409a867914caf6676d Mon Sep 17 00:00:00 2001 From: grufoony Date: Sat, 6 Dec 2025 13:06:34 +0100 Subject: [PATCH 16/16] Bump version --- src/dsf/dsf.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsf/dsf.hpp b/src/dsf/dsf.hpp index b17cc1fa..5395674a 100644 --- a/src/dsf/dsf.hpp +++ b/src/dsf/dsf.hpp @@ -6,7 +6,7 @@ static constexpr uint8_t DSF_VERSION_MAJOR = 4; static constexpr uint8_t DSF_VERSION_MINOR = 5; -static constexpr uint8_t DSF_VERSION_PATCH = 0; +static constexpr uint8_t DSF_VERSION_PATCH = 1; static auto const DSF_VERSION = std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH);