diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index f2772285..16a63662 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] # , windows-latest --- IGNORE --- + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.10", "3.12"] runs-on: ${{ matrix.os }} @@ -32,27 +32,18 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen - name: Install system dependencies (macOS) if: matrix.os == 'macos-latest' run: | - brew install cmake tbb spdlog simdjson doxygen - echo "CPLUS_INCLUDE_PATH=$(brew --prefix tbb)/include:$(brew --prefix fmt)/include:$(brew --prefix spdlog)/include:$(brew --prefix simdjson)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV + brew install cmake doxygen - name: Install system dependencies (Windows) if: matrix.os == 'windows-latest' run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install - # Install vcpkg for spdlog and tbb on Windows - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg install spdlog:x64-windows tbb:x64-windows - echo "VCPKG_ROOT=$env:GITHUB_WORKSPACE\vcpkg" >> $env:GITHUB_ENV - name: Upgrade pip and install build dependencies run: | @@ -189,7 +180,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen - name: Install development dependencies run: | diff --git a/.github/workflows/cmake_examples.yml b/.github/workflows/cmake_examples.yml index 70bef7c4..e1e0b15a 100644 --- a/.github/workflows/cmake_examples.yml +++ b/.github/workflows/cmake_examples.yml @@ -23,9 +23,7 @@ jobs: run: | if [ ${{ matrix.os }} == 'ubuntu-latest' ]; then sudo apt update - sudo apt install libtbb-dev libsimdjson-dev binutils - elif [ ${{ matrix.os }} == 'macos-latest' ]; then - brew install tbb simdjson + sudo apt install binutils fi - uses: actions/checkout@v4 diff --git a/.github/workflows/cmake_tests.yml b/.github/workflows/cmake_tests.yml index 8f521d02..458bd0b5 100644 --- a/.github/workflows/cmake_tests.yml +++ b/.github/workflows/cmake_tests.yml @@ -24,12 +24,6 @@ jobs: sudo apt update sudo apt install -y lcov gcovr build-essential cmake libhwloc-dev - - name: Install dependencies on macOS - if: matrix.os == 'macos-latest' - run: | - brew update - brew install tbb simdjson - - name: Install dependencies on Windows (vcpkg) if: matrix.os == 'windows-latest' shell: powershell @@ -56,7 +50,6 @@ jobs: mkdir -p build cd build cmake .. \ - -DCMAKE_PREFIX_PATH="$(brew --prefix tbb):$(brew --prefix simdjson)" \ -DCMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/install cmake --build . -j$(sysctl -n hw.ncpu) --config Debug diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3b1daac1..4b306b84 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,17 +23,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install TBB (MacOS & Ubuntu) - run: | - if [[ "$RUNNER_OS" == "macOS" ]]; then - brew install tbb spdlog simdjson - echo "CPLUS_INCLUDE_PATH=$(brew --prefix tbb)/include:$(brew --prefix fmt)/include:$(brew --prefix spdlog)/include:$(brew --prefix simdjson)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - elif [[ "$RUNNER_OS" == "Linux" ]]; then - sudo apt update && sudo apt install -y libtbb-dev libspdlog-dev libsimdjson-dev - fi - - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: @@ -42,11 +31,7 @@ jobs: - name: Build C++ run: | mkdir -p build && cd build - if [[ "$RUNNER_OS" == "macOS" ]]; then - cmake .. -DCMAKE_PREFIX_PATH="$(brew --prefix tbb);$(brew --prefix fmt);$(brew --prefix spdlog)" - else - cmake .. - fi + cmake .. cmake --build . sudo make install diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index e5edddda..5f13cd71 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -79,7 +79,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen libtbb-dev libfmt-dev libspdlog-dev libsimdjson-dev + sudo apt install -y cmake build-essential doxygen - name: Install build dependencies run: | @@ -127,7 +127,7 @@ jobs: - name: Install system dependencies run: | - brew install cmake tbb spdlog simdjson doxygen + brew install cmake doxygen - name: Install build dependencies run: | @@ -136,9 +136,6 @@ jobs: - name: Set macOS environment variables run: | - echo "CPLUS_INCLUDE_PATH=$(brew --prefix tbb)/include:$(brew --prefix fmt)/include:$(brew --prefix spdlog)/include:$(brew --prefix simdjson)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix tbb)/lib:$(brew --prefix fmt)/lib:$(brew --prefix spdlog)/lib:$(brew --prefix simdjson)/lib" >> $GITHUB_ENV - echo "CMAKE_PREFIX_PATH=$(brew --prefix tbb);$(brew --prefix fmt);$(brew --prefix spdlog);$(brew --prefix simdjson)" >> $GITHUB_ENV echo "ARCHFLAGS=-arch $(uname -m)" >> $GITHUB_ENV echo "_PYTHON_HOST_PLATFORM=macosx-$(sw_vers -productVersion | cut -d. -f1)-$(uname -m)" >> $GITHUB_ENV @@ -156,6 +153,43 @@ jobs: name: wheel-macos-${{ matrix.python-version }} path: wheelhouse/*.whl + build-wheels-windows: + name: Build wheels on Windows (Python ${{ matrix.python-version }}) + needs: [check-version] + runs-on: windows-latest + if: needs.check-version.outputs.should_build == 'true' + strategy: + matrix: + python-version: ['3.10', '3.12'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + run: | + choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' + choco install doxygen.install + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build wheel setuptools pybind11-stubgen + + - name: Build wheel + run: python -m build --wheel + + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-windows-${{ matrix.python-version }} + path: dist/*.whl + build-sdist: name: Build source distribution needs: [check-version] @@ -174,7 +208,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen - name: Install build dependencies run: | @@ -192,7 +226,7 @@ jobs: publish: name: Publish to PyPI - needs: [check-version, build-wheels-linux, build-wheels-macos, build-sdist] + needs: [check-version, build-wheels-linux, build-wheels-macos, build-wheels-windows, build-sdist] runs-on: ubuntu-latest if: needs.check-version.outputs.should_build == 'true' diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 17a89619..213f63c8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y cmake build-essential libtbb-dev libspdlog-dev libsimdjson-dev doxygen + sudo apt install -y cmake build-essential doxygen python -m pip install --upgrade pip pip install pytest pytest-cov diff --git a/CMakeLists.txt b/CMakeLists.txt index b447bb70..19ddb225 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,17 +130,18 @@ if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) endif() # Get TBB -set(TBB_TEST - OFF - CACHE BOOL "Disable TBB tests" FORCE) +set(TBB_TEST OFF CACHE BOOL "Disable TBB tests" FORCE) +set(TBB_EXAMPLES OFF CACHE BOOL "Disable TBB examples" FORCE) +set(TBB_STRICT OFF CACHE BOOL "Disable TBB strict mode" FORCE) +# set(TBB_BUILD_SHARED ON CACHE BOOL "Build TBB as shared library" FORCE) +# set(TBB_BUILD_TBBMALLOC OFF CACHE BOOL "Disable TBB malloc" FORCE) + FetchContent_Declare( tbb - GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB - GIT_TAG v2022.3.0) -FetchContent_GetProperties(tbb) -if(NOT tbb_POPULATED) - FetchContent_MakeAvailable(tbb) -endif() + GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB.git + GIT_TAG v2022.3.0 +) +FetchContent_MakeAvailable(tbb) add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) @@ -177,6 +178,12 @@ install( if(BUILD_PYTHON_BINDINGS) include(FetchContent) + # Check if Doxygen is available for documentation generation + find_package(Doxygen REQUIRED) + if(NOT DOXYGEN_FOUND) + message(FATAL_ERROR "Doxygen is required to build the docstrings.") + endif() + # Get pybind11 FetchContent_Declare( pybind11 @@ -188,15 +195,14 @@ if(BUILD_PYTHON_BINDINGS) endif() # Add the Python binding module - add_library(dsf_python_module MODULE src/dsf/bindings.cpp) + pybind11_add_module(dsf_python_module src/dsf/bindings.cpp) # Ensure the Python module name has no 'lib' prefix on Unix systems - set_target_properties(dsf_python_module PROPERTIES PREFIX "" OUTPUT_NAME - "dsf_cpp") + set_target_properties(dsf_python_module PROPERTIES OUTPUT_NAME "dsf_cpp") # Link the pybind11 module with your static library and pybind11 target_link_libraries( - dsf_python_module PRIVATE dsf pybind11::module pybind11::headers TBB::tbb + dsf_python_module PRIVATE dsf pybind11::headers TBB::tbb spdlog::spdlog) target_compile_definitions(dsf_python_module PRIVATE SPDLOG_USE_STD_FORMAT) @@ -218,8 +224,13 @@ if(BUILD_PYTHON_BINDINGS) set_target_properties( dsf_python_module PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "${HOMEBREW_LIB}" + INSTALL_RPATH "${HOMEBREW_LIB};@loader_path" INSTALL_RPATH_USE_LINK_PATH TRUE) + elseif(UNIX) + set_target_properties( + dsf_python_module + PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "$ORIGIN") endif() endif() diff --git a/setup.py b/setup.py index c3eda7ed..e1c5e735 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,9 @@ def build_extension(self, ext: CMakeExtension): "-DBUILD_PYTHON_BINDINGS=ON", ] + if platform.system() == "Windows": + cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] + # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS try: @@ -123,6 +126,65 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) + # Copy TBB shared library if it exists (Linux and macOS) + if platform.system() == "Linux" or platform.system() == "Darwin": + print(f"Searching for TBB shared libraries in {build_temp}...") + + tbb_libs = [] + if platform.system() == "Linux": + # Look for libtbb.so* recursively + tbb_libs = list(build_temp.glob("**/libtbb.so*")) + # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built + tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) + else: # macOS + # Look for libtbb.dylib* recursively + tbb_libs = list(build_temp.glob("**/libtbb*.dylib")) + + if tbb_libs: + print(f"Found TBB libraries: {tbb_libs}") + for lib in tbb_libs: + # We only want the real shared object, not symlinks if possible, + # but copying everything matching the pattern is safer to ensure we get the versioned one. + # However, we need to be careful not to overwrite if multiple matches found. + # Usually we want the one that the linker linked against. + # Since we set RPATH to $ORIGIN (Linux) or @loader_path (macOS), we need the library in the same dir as the extension. + + # Avoid copying if it's a symlink pointing to something we already copied? + # simpler: just copy all of them. + dest = extdir / lib.name + if not dest.exists(): + shutil.copy2(lib, dest) + print(f"Copied {lib} to {dest}") + else: + print("Warning: No TBB shared libraries found to copy.") + + elif platform.system() == "Windows": + print(f"Searching for TBB DLLs in {build_temp}...") + # Look for tbb*.dll recursively + tbb_dlls = list(build_temp.glob("**/tbb*.dll")) + + if tbb_dlls: + print(f"Found TBB DLLs: {tbb_dlls}") + # We want to copy them to the 'dsf' package directory so we can load them in __init__.py + # extdir is where dsf_cpp.pyd is (site-packages root usually) + # We want site-packages/dsf/ + + # self.build_lib is usually the root of the build (e.g. build/lib.win...) + # So we can construct the path to dsf package + dsf_pkg_dir = Path(self.build_lib) / "dsf" + dsf_pkg_dir.mkdir(parents=True, exist_ok=True) + + # Also copy to source directory for editable installs + source_dsf_dir = Path(__file__).parent / "src" / "dsf" + + for dll in tbb_dlls: + print(f"Copying {dll} to {dsf_pkg_dir}") + shutil.copy2(dll, dsf_pkg_dir) + print(f"Copying {dll} to {source_dsf_dir}") + shutil.copy2(dll, source_dsf_dir) + else: + print("Warning: No TBB DLLs found. This might cause import errors.") + def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" diff --git a/src/dsf/__init__.py b/src/dsf/__init__.py index 4a7c999d..e35d3d92 100644 --- a/src/dsf/__init__.py +++ b/src/dsf/__init__.py @@ -1,3 +1,19 @@ +import sys +import os + +# On Windows, we need to explicitly load the bundled TBB DLLs +if sys.platform == "win32": + import glob + import ctypes + + # Look for tbb dlls in the same directory as this __init__.py + _dll_dir = os.path.dirname(__file__) + for _dll in glob.glob(os.path.join(_dll_dir, "tbb*.dll")): + try: + ctypes.CDLL(_dll) + except Exception as e: + print(f"Warning: Failed to load {_dll}: {e}") + from dsf_cpp import __version__, LogLevel, get_log_level, set_log_level, mobility, mdt from .python.cartography import ( diff --git a/src/dsf/dsf.hpp b/src/dsf/dsf.hpp index b17cc1fa..5395674a 100644 --- a/src/dsf/dsf.hpp +++ b/src/dsf/dsf.hpp @@ -6,7 +6,7 @@ static constexpr uint8_t DSF_VERSION_MAJOR = 4; static constexpr uint8_t DSF_VERSION_MINOR = 5; -static constexpr uint8_t DSF_VERSION_PATCH = 0; +static constexpr uint8_t DSF_VERSION_PATCH = 1; static auto const DSF_VERSION = std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH);