diff --git a/.github/workflows/benchmark_release.yml b/.github/workflows/benchmark_release.yml index c3d0737e..1f436c05 100644 --- a/.github/workflows/benchmark_release.yml +++ b/.github/workflows/benchmark_release.yml @@ -17,7 +17,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y build-essential cmake libhwloc-dev + sudo apt install -y build-essential cmake libtbb-dev - name: Build in Release mode with benchmarks working-directory: ${{ github.workspace }} diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index 16a63662..31855414 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -32,18 +32,24 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install system dependencies (macOS) if: matrix.os == 'macos-latest' run: | - brew install cmake doxygen + brew install cmake doxygen tbb - name: Install system dependencies (Windows) if: matrix.os == 'windows-latest' run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install + git clone https://github.com/microsoft/vcpkg.git vcpkg + cd vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg install tbb:x64-windows + echo "CMAKE_TOOLCHAIN_FILE=$env:GITHUB_WORKSPACE\vcpkg\scripts\buildsystems\vcpkg.cmake" >> $env:GITHUB_ENV + echo "VCPKG_TARGET_TRIPLET=x64-windows" >> $env:GITHUB_ENV - name: Upgrade pip and install build dependencies run: | @@ -180,7 +186,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install development dependencies run: | diff --git a/.github/workflows/cmake_examples.yml b/.github/workflows/cmake_examples.yml index e1e0b15a..327d07cb 100644 --- a/.github/workflows/cmake_examples.yml +++ b/.github/workflows/cmake_examples.yml @@ -23,7 +23,9 @@ jobs: run: | if [ ${{ matrix.os }} == 'ubuntu-latest' ]; then sudo apt update - sudo apt install binutils + sudo apt install binutils libtbb-dev + elif [ ${{ matrix.os }} == 'macos-latest' ]; then + brew install tbb fi - uses: actions/checkout@v4 diff --git a/.github/workflows/cmake_tests.yml b/.github/workflows/cmake_tests.yml index 458bd0b5..a133047a 100644 --- a/.github/workflows/cmake_tests.yml +++ b/.github/workflows/cmake_tests.yml @@ -22,7 +22,11 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y lcov gcovr build-essential cmake libhwloc-dev + sudo apt install -y lcov gcovr build-essential cmake libtbb-dev + + - name: Install dependencies on macOS + if: matrix.os == 'macos-latest' + run: brew install tbb - name: Install dependencies on Windows (vcpkg) if: matrix.os == 'windows-latest' @@ -31,6 +35,7 @@ jobs: git clone https://github.com/microsoft/vcpkg.git vcpkg cd vcpkg .\bootstrap-vcpkg.bat + .\vcpkg install tbb:x64-windows echo "VCPKG_ROOT=$env:GITHUB_WORKSPACE\vcpkg" >> $env:GITHUB_ENV echo "VCPKG_INSTALLED=$env:GITHUB_WORKSPACE\vcpkg\installed\x64-windows" >> $env:GITHUB_ENV diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4b306b84..2912f58c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -28,6 +28,11 @@ jobs: with: languages: ${{ matrix.language }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y libtbb-dev + - name: Build C++ run: | mkdir -p build && cd build diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 5f13cd71..2388927c 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -79,7 +79,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install build dependencies run: | @@ -127,7 +127,7 @@ jobs: - name: Install system dependencies run: | - brew install cmake doxygen + brew install cmake doxygen tbb - name: Install build dependencies run: | @@ -175,6 +175,12 @@ jobs: run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install + git clone https://github.com/microsoft/vcpkg.git vcpkg + cd vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg install tbb:x64-windows + echo "CMAKE_TOOLCHAIN_FILE=$env:GITHUB_WORKSPACE\vcpkg\scripts\buildsystems\vcpkg.cmake" >> $env:GITHUB_ENV + echo "VCPKG_TARGET_TRIPLET=x64-windows" >> $env:GITHUB_ENV - name: Install build dependencies run: | @@ -208,7 +214,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install build dependencies run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 213f63c8..7f34a151 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev python -m pip install --upgrade pip pip install pytest pytest-cov diff --git a/CMakeLists.txt b/CMakeLists.txt index c57586f5..eda7e843 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,19 +129,8 @@ FetchContent_GetProperties(simdjson) if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) endif() -# Get TBB -set(TBB_TEST OFF CACHE BOOL "Disable TBB tests" FORCE) -set(TBB_EXAMPLES OFF CACHE BOOL "Disable TBB examples" FORCE) -set(TBB_STRICT OFF CACHE BOOL "Disable TBB strict mode" FORCE) -# set(TBB_BUILD_SHARED ON CACHE BOOL "Build TBB as shared library" FORCE) -# set(TBB_BUILD_TBBMALLOC OFF CACHE BOOL "Disable TBB malloc" FORCE) - -FetchContent_Declare( - tbb - GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB.git - GIT_TAG v2022.3.0 -) -FetchContent_MakeAvailable(tbb) +# Check if the user has TBB installed +find_package(TBB REQUIRED CONFIG) add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) @@ -209,29 +198,6 @@ if(BUILD_PYTHON_BINDINGS) target_include_directories(dsf_python_module PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) - if(APPLE) - # Try to detect Homebrew prefix dynamically - execute_process( - COMMAND brew --prefix - OUTPUT_VARIABLE HOMEBREW_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) - if(HOMEBREW_PREFIX) - set(HOMEBREW_LIB "${HOMEBREW_PREFIX}/lib") - else() - # Fallback to default Homebrew locations - set(HOMEBREW_LIB "/opt/homebrew/lib;/usr/local/lib") - endif() - set_target_properties( - dsf_python_module - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "${HOMEBREW_LIB};@loader_path" - INSTALL_RPATH_USE_LINK_PATH TRUE) - elseif(UNIX) - set_target_properties( - dsf_python_module - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "$ORIGIN") - endif() endif() # Tests diff --git a/README.md b/README.md index ce5d2aac..fdfdcb94 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,11 @@ print(dsf.__version__) The project requires `C++20` or greater, `cmake`, `tbb` `simdjson`, `spdlog` and `rapidcsv`. To install requirements on Ubuntu: ```shell -sudo apt install cmake libhwloc-dev +sudo apt install cmake libtbb-dev ``` To install requirements on macOS: ```shell -brew install cmake +brew install cmake tbb ``` Utilities are written in python. To install their dependencies: @@ -68,7 +68,7 @@ cmake --install build ## Installation (Python) If you want to use the library from Python, you can build the Python bindings using [pybind11](https://github.com/pybind/pybind11). Make sure you have Doxygen installed to generate the docstrings: ```shell -sudo apt install doxygen +sudo apt install doxygen libtbb-dev ``` Then, the installation is automatic via `pip`: diff --git a/setup.py b/setup.py index e1c5e735..7294397e 100644 --- a/setup.py +++ b/setup.py @@ -80,13 +80,18 @@ def build_extension(self, ext: CMakeExtension): if platform.system() == "Windows": cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] + if "CMAKE_TOOLCHAIN_FILE" in os.environ: + cmake_args.append( + f"-DCMAKE_TOOLCHAIN_FILE={os.environ['CMAKE_TOOLCHAIN_FILE']}" + ) + if "VCPKG_TARGET_TRIPLET" in os.environ: + cmake_args.append( + f"-DVCPKG_TARGET_TRIPLET={os.environ['VCPKG_TARGET_TRIPLET']}" + ) # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS try: - tbb_prefix = subprocess.check_output( - ["brew", "--prefix", "tbb"], text=True - ).strip() fmt_prefix = subprocess.check_output( ["brew", "--prefix", "fmt"], text=True ).strip() @@ -94,7 +99,7 @@ def build_extension(self, ext: CMakeExtension): ["brew", "--prefix", "spdlog"], text=True ).strip() - cmake_prefix_path = f"{tbb_prefix};{fmt_prefix};{spdlog_prefix}" + cmake_prefix_path = f"{fmt_prefix};{spdlog_prefix}" cmake_args.append(f"-DCMAKE_PREFIX_PATH={cmake_prefix_path}") print(f"Added macOS Homebrew prefix paths: {cmake_prefix_path}") @@ -126,65 +131,6 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) - # Copy TBB shared library if it exists (Linux and macOS) - if platform.system() == "Linux" or platform.system() == "Darwin": - print(f"Searching for TBB shared libraries in {build_temp}...") - - tbb_libs = [] - if platform.system() == "Linux": - # Look for libtbb.so* recursively - tbb_libs = list(build_temp.glob("**/libtbb.so*")) - # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built - tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) - else: # macOS - # Look for libtbb.dylib* recursively - tbb_libs = list(build_temp.glob("**/libtbb*.dylib")) - - if tbb_libs: - print(f"Found TBB libraries: {tbb_libs}") - for lib in tbb_libs: - # We only want the real shared object, not symlinks if possible, - # but copying everything matching the pattern is safer to ensure we get the versioned one. - # However, we need to be careful not to overwrite if multiple matches found. - # Usually we want the one that the linker linked against. - # Since we set RPATH to $ORIGIN (Linux) or @loader_path (macOS), we need the library in the same dir as the extension. - - # Avoid copying if it's a symlink pointing to something we already copied? - # simpler: just copy all of them. - dest = extdir / lib.name - if not dest.exists(): - shutil.copy2(lib, dest) - print(f"Copied {lib} to {dest}") - else: - print("Warning: No TBB shared libraries found to copy.") - - elif platform.system() == "Windows": - print(f"Searching for TBB DLLs in {build_temp}...") - # Look for tbb*.dll recursively - tbb_dlls = list(build_temp.glob("**/tbb*.dll")) - - if tbb_dlls: - print(f"Found TBB DLLs: {tbb_dlls}") - # We want to copy them to the 'dsf' package directory so we can load them in __init__.py - # extdir is where dsf_cpp.pyd is (site-packages root usually) - # We want site-packages/dsf/ - - # self.build_lib is usually the root of the build (e.g. build/lib.win...) - # So we can construct the path to dsf package - dsf_pkg_dir = Path(self.build_lib) / "dsf" - dsf_pkg_dir.mkdir(parents=True, exist_ok=True) - - # Also copy to source directory for editable installs - source_dsf_dir = Path(__file__).parent / "src" / "dsf" - - for dll in tbb_dlls: - print(f"Copying {dll} to {dsf_pkg_dir}") - shutil.copy2(dll, dsf_pkg_dir) - print(f"Copying {dll} to {source_dsf_dir}") - shutil.copy2(dll, source_dsf_dir) - else: - print("Warning: No TBB DLLs found. This might cause import errors.") - def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" diff --git a/src/dsf/dsf.hpp b/src/dsf/dsf.hpp index a6d056df..2a85bc44 100644 --- a/src/dsf/dsf.hpp +++ b/src/dsf/dsf.hpp @@ -6,7 +6,7 @@ static constexpr uint8_t DSF_VERSION_MAJOR = 4; static constexpr uint8_t DSF_VERSION_MINOR = 5; -static constexpr uint8_t DSF_VERSION_PATCH = 2; +static constexpr uint8_t DSF_VERSION_PATCH = 3; static auto const DSF_VERSION = std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH);