From b81d14dece58d2f6bc7fadeaf2a79529f2a878ff Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 11:19:15 +0100 Subject: [PATCH 1/7] Init implementation --- .github/workflows/benchmark_release.yml | 2 +- .github/workflows/binding.yml | 5 +- .github/workflows/cmake_examples.yml | 4 +- .github/workflows/cmake_tests.yml | 7 ++- .github/workflows/codeql.yml | 5 ++ .github/workflows/pypi.yml | 7 +-- CMakeLists.txt | 61 ++++++++++------------- setup.py | 64 +------------------------ 8 files changed, 48 insertions(+), 107 deletions(-) diff --git a/.github/workflows/benchmark_release.yml b/.github/workflows/benchmark_release.yml index c3d0737e..1f436c05 100644 --- a/.github/workflows/benchmark_release.yml +++ b/.github/workflows/benchmark_release.yml @@ -17,7 +17,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y build-essential cmake libhwloc-dev + sudo apt install -y build-essential cmake libtbb-dev - name: Build in Release mode with benchmarks working-directory: ${{ github.workspace }} diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index 16a63662..5abbc724 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -32,18 +32,19 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install system dependencies (macOS) if: matrix.os == 'macos-latest' run: | - brew install cmake doxygen + brew install cmake doxygen tbb - name: Install system dependencies (Windows) if: matrix.os == 'windows-latest' run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install + choco install tbb - name: Upgrade pip and install build dependencies run: | diff --git a/.github/workflows/cmake_examples.yml b/.github/workflows/cmake_examples.yml index e1e0b15a..327d07cb 100644 --- a/.github/workflows/cmake_examples.yml +++ b/.github/workflows/cmake_examples.yml @@ -23,7 +23,9 @@ jobs: run: | if [ ${{ matrix.os }} == 'ubuntu-latest' ]; then sudo apt update - sudo apt install binutils + sudo apt install binutils libtbb-dev + elif [ ${{ matrix.os }} == 'macos-latest' ]; then + brew install tbb fi - uses: actions/checkout@v4 diff --git a/.github/workflows/cmake_tests.yml b/.github/workflows/cmake_tests.yml index 458bd0b5..f7b3b208 100644 --- a/.github/workflows/cmake_tests.yml +++ b/.github/workflows/cmake_tests.yml @@ -22,7 +22,11 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y lcov gcovr build-essential cmake libhwloc-dev + sudo apt install -y lcov gcovr build-essential cmake libhwloc-dev libtbb-dev + + - name: Install dependencies on macOS + if: matrix.os == 'macos-latest' + run: brew install tbb - name: Install dependencies on Windows (vcpkg) if: matrix.os == 'windows-latest' @@ -31,6 +35,7 @@ jobs: git clone https://github.com/microsoft/vcpkg.git vcpkg cd vcpkg .\bootstrap-vcpkg.bat + .\vcpkg install tbb:x64-windows echo "VCPKG_ROOT=$env:GITHUB_WORKSPACE\vcpkg" >> $env:GITHUB_ENV echo "VCPKG_INSTALLED=$env:GITHUB_WORKSPACE\vcpkg\installed\x64-windows" >> $env:GITHUB_ENV diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4b306b84..2912f58c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -28,6 +28,11 @@ jobs: with: languages: ${{ matrix.language }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y libtbb-dev + - name: Build C++ run: | mkdir -p build && cd build diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 5f13cd71..c8f40574 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -79,7 +79,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install build dependencies run: | @@ -127,7 +127,7 @@ jobs: - name: Install system dependencies run: | - brew install cmake doxygen + brew install cmake doxygen tbb - name: Install build dependencies run: | @@ -175,6 +175,7 @@ jobs: run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install + choco install tbb - name: Install build dependencies run: | @@ -208,7 +209,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install build dependencies run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index c57586f5..885ca293 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,19 +129,8 @@ FetchContent_GetProperties(simdjson) if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) endif() -# Get TBB -set(TBB_TEST OFF CACHE BOOL "Disable TBB tests" FORCE) -set(TBB_EXAMPLES OFF CACHE BOOL "Disable TBB examples" FORCE) -set(TBB_STRICT OFF CACHE BOOL "Disable TBB strict mode" FORCE) -# set(TBB_BUILD_SHARED ON CACHE BOOL "Build TBB as shared library" FORCE) -# set(TBB_BUILD_TBBMALLOC OFF CACHE BOOL "Disable TBB malloc" FORCE) - -FetchContent_Declare( - tbb - GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB.git - GIT_TAG v2022.3.0 -) -FetchContent_MakeAvailable(tbb) +# Check if the user has TBB installed +find_package(TBB REQUIRED) add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) @@ -209,29 +198,29 @@ if(BUILD_PYTHON_BINDINGS) target_include_directories(dsf_python_module PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) - if(APPLE) - # Try to detect Homebrew prefix dynamically - execute_process( - COMMAND brew --prefix - OUTPUT_VARIABLE HOMEBREW_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) - if(HOMEBREW_PREFIX) - set(HOMEBREW_LIB "${HOMEBREW_PREFIX}/lib") - else() - # Fallback to default Homebrew locations - set(HOMEBREW_LIB "/opt/homebrew/lib;/usr/local/lib") - endif() - set_target_properties( - dsf_python_module - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "${HOMEBREW_LIB};@loader_path" - INSTALL_RPATH_USE_LINK_PATH TRUE) - elseif(UNIX) - set_target_properties( - dsf_python_module - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "$ORIGIN") - endif() + # if(APPLE) + # # Try to detect Homebrew prefix dynamically + # execute_process( + # COMMAND brew --prefix + # OUTPUT_VARIABLE HOMEBREW_PREFIX + # OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + # if(HOMEBREW_PREFIX) + # set(HOMEBREW_LIB "${HOMEBREW_PREFIX}/lib") + # else() + # # Fallback to default Homebrew locations + # set(HOMEBREW_LIB "/opt/homebrew/lib;/usr/local/lib") + # endif() + # set_target_properties( + # dsf_python_module + # PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + # INSTALL_RPATH "${HOMEBREW_LIB};@loader_path" + # INSTALL_RPATH_USE_LINK_PATH TRUE) + # elseif(UNIX) + # set_target_properties( + # dsf_python_module + # PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE + # INSTALL_RPATH "$ORIGIN") + # endif() endif() # Tests diff --git a/setup.py b/setup.py index e1c5e735..b522cc3e 100644 --- a/setup.py +++ b/setup.py @@ -84,9 +84,6 @@ def build_extension(self, ext: CMakeExtension): # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS try: - tbb_prefix = subprocess.check_output( - ["brew", "--prefix", "tbb"], text=True - ).strip() fmt_prefix = subprocess.check_output( ["brew", "--prefix", "fmt"], text=True ).strip() @@ -94,7 +91,7 @@ def build_extension(self, ext: CMakeExtension): ["brew", "--prefix", "spdlog"], text=True ).strip() - cmake_prefix_path = f"{tbb_prefix};{fmt_prefix};{spdlog_prefix}" + cmake_prefix_path = f"{fmt_prefix};{spdlog_prefix}" cmake_args.append(f"-DCMAKE_PREFIX_PATH={cmake_prefix_path}") print(f"Added macOS Homebrew prefix paths: {cmake_prefix_path}") @@ -126,65 +123,6 @@ def build_extension(self, ext: CMakeExtension): cwd=build_temp, ) - # Copy TBB shared library if it exists (Linux and macOS) - if platform.system() == "Linux" or platform.system() == "Darwin": - print(f"Searching for TBB shared libraries in {build_temp}...") - - tbb_libs = [] - if platform.system() == "Linux": - # Look for libtbb.so* recursively - tbb_libs = list(build_temp.glob("**/libtbb.so*")) - # Also look for libtbb_debug.so* if we are in debug mode or if that's what was built - tbb_libs.extend(list(build_temp.glob("**/libtbb_debug.so*"))) - else: # macOS - # Look for libtbb.dylib* recursively - tbb_libs = list(build_temp.glob("**/libtbb*.dylib")) - - if tbb_libs: - print(f"Found TBB libraries: {tbb_libs}") - for lib in tbb_libs: - # We only want the real shared object, not symlinks if possible, - # but copying everything matching the pattern is safer to ensure we get the versioned one. - # However, we need to be careful not to overwrite if multiple matches found. - # Usually we want the one that the linker linked against. - # Since we set RPATH to $ORIGIN (Linux) or @loader_path (macOS), we need the library in the same dir as the extension. - - # Avoid copying if it's a symlink pointing to something we already copied? - # simpler: just copy all of them. - dest = extdir / lib.name - if not dest.exists(): - shutil.copy2(lib, dest) - print(f"Copied {lib} to {dest}") - else: - print("Warning: No TBB shared libraries found to copy.") - - elif platform.system() == "Windows": - print(f"Searching for TBB DLLs in {build_temp}...") - # Look for tbb*.dll recursively - tbb_dlls = list(build_temp.glob("**/tbb*.dll")) - - if tbb_dlls: - print(f"Found TBB DLLs: {tbb_dlls}") - # We want to copy them to the 'dsf' package directory so we can load them in __init__.py - # extdir is where dsf_cpp.pyd is (site-packages root usually) - # We want site-packages/dsf/ - - # self.build_lib is usually the root of the build (e.g. build/lib.win...) - # So we can construct the path to dsf package - dsf_pkg_dir = Path(self.build_lib) / "dsf" - dsf_pkg_dir.mkdir(parents=True, exist_ok=True) - - # Also copy to source directory for editable installs - source_dsf_dir = Path(__file__).parent / "src" / "dsf" - - for dll in tbb_dlls: - print(f"Copying {dll} to {dsf_pkg_dir}") - shutil.copy2(dll, dsf_pkg_dir) - print(f"Copying {dll} to {source_dsf_dir}") - shutil.copy2(dll, source_dsf_dir) - else: - print("Warning: No TBB DLLs found. This might cause import errors.") - def pre_build(self): """Extracts doxygen documentation from XML files and creates a C++ unordered_map""" From efed39f65af20470e19c4134d055c8e6332f4f14 Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 13:41:20 +0100 Subject: [PATCH 2/7] Boh non va --- .github/workflows/binding.yml | 2 +- .github/workflows/cmake_tests.yml | 2 +- CMakeLists.txt | 25 +------------------------ 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index 5abbc724..a598f385 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -181,7 +181,7 @@ jobs: - name: Install system dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev - name: Install development dependencies run: | diff --git a/.github/workflows/cmake_tests.yml b/.github/workflows/cmake_tests.yml index f7b3b208..a133047a 100644 --- a/.github/workflows/cmake_tests.yml +++ b/.github/workflows/cmake_tests.yml @@ -22,7 +22,7 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | sudo apt update - sudo apt install -y lcov gcovr build-essential cmake libhwloc-dev libtbb-dev + sudo apt install -y lcov gcovr build-essential cmake libtbb-dev - name: Install dependencies on macOS if: matrix.os == 'macos-latest' diff --git a/CMakeLists.txt b/CMakeLists.txt index 885ca293..eda7e843 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,7 @@ if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) endif() # Check if the user has TBB installed -find_package(TBB REQUIRED) +find_package(TBB REQUIRED CONFIG) add_library(dsf STATIC ${SOURCES}) target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT) @@ -198,29 +198,6 @@ if(BUILD_PYTHON_BINDINGS) target_include_directories(dsf_python_module PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) - # if(APPLE) - # # Try to detect Homebrew prefix dynamically - # execute_process( - # COMMAND brew --prefix - # OUTPUT_VARIABLE HOMEBREW_PREFIX - # OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) - # if(HOMEBREW_PREFIX) - # set(HOMEBREW_LIB "${HOMEBREW_PREFIX}/lib") - # else() - # # Fallback to default Homebrew locations - # set(HOMEBREW_LIB "/opt/homebrew/lib;/usr/local/lib") - # endif() - # set_target_properties( - # dsf_python_module - # PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - # INSTALL_RPATH "${HOMEBREW_LIB};@loader_path" - # INSTALL_RPATH_USE_LINK_PATH TRUE) - # elseif(UNIX) - # set_target_properties( - # dsf_python_module - # PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - # INSTALL_RPATH "$ORIGIN") - # endif() endif() # Tests From 0b73a60df1e8947946501e89a07a859d0eee4f37 Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 13:53:03 +0100 Subject: [PATCH 3/7] Fix windows --- .github/workflows/binding.yml | 7 ++++++- .github/workflows/pypi.yml | 7 ++++++- setup.py | 4 ++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/binding.yml b/.github/workflows/binding.yml index a598f385..31855414 100644 --- a/.github/workflows/binding.yml +++ b/.github/workflows/binding.yml @@ -44,7 +44,12 @@ jobs: run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install - choco install tbb + git clone https://github.com/microsoft/vcpkg.git vcpkg + cd vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg install tbb:x64-windows + echo "CMAKE_TOOLCHAIN_FILE=$env:GITHUB_WORKSPACE\vcpkg\scripts\buildsystems\vcpkg.cmake" >> $env:GITHUB_ENV + echo "VCPKG_TARGET_TRIPLET=x64-windows" >> $env:GITHUB_ENV - name: Upgrade pip and install build dependencies run: | diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index c8f40574..2388927c 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -175,7 +175,12 @@ jobs: run: | choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' choco install doxygen.install - choco install tbb + git clone https://github.com/microsoft/vcpkg.git vcpkg + cd vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg install tbb:x64-windows + echo "CMAKE_TOOLCHAIN_FILE=$env:GITHUB_WORKSPACE\vcpkg\scripts\buildsystems\vcpkg.cmake" >> $env:GITHUB_ENV + echo "VCPKG_TARGET_TRIPLET=x64-windows" >> $env:GITHUB_ENV - name: Install build dependencies run: | diff --git a/setup.py b/setup.py index b522cc3e..5bc03dd7 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,10 @@ def build_extension(self, ext: CMakeExtension): if platform.system() == "Windows": cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] + if "CMAKE_TOOLCHAIN_FILE" in os.environ: + cmake_args.append(f"-DCMAKE_TOOLCHAIN_FILE={os.environ['CMAKE_TOOLCHAIN_FILE']}") + if "VCPKG_TARGET_TRIPLET" in os.environ: + cmake_args.append(f"-DVCPKG_TARGET_TRIPLET={os.environ['VCPKG_TARGET_TRIPLET']}") # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS From 7380f39a2836c5df8f6fa0ba0fde0d445b8b9332 Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 14:11:57 +0100 Subject: [PATCH 4/7] Formatting --- setup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5bc03dd7..7294397e 100644 --- a/setup.py +++ b/setup.py @@ -81,9 +81,13 @@ def build_extension(self, ext: CMakeExtension): if platform.system() == "Windows": cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] if "CMAKE_TOOLCHAIN_FILE" in os.environ: - cmake_args.append(f"-DCMAKE_TOOLCHAIN_FILE={os.environ['CMAKE_TOOLCHAIN_FILE']}") + cmake_args.append( + f"-DCMAKE_TOOLCHAIN_FILE={os.environ['CMAKE_TOOLCHAIN_FILE']}" + ) if "VCPKG_TARGET_TRIPLET" in os.environ: - cmake_args.append(f"-DVCPKG_TARGET_TRIPLET={os.environ['VCPKG_TARGET_TRIPLET']}") + cmake_args.append( + f"-DVCPKG_TARGET_TRIPLET={os.environ['VCPKG_TARGET_TRIPLET']}" + ) # Add macOS-specific CMake prefix paths for Homebrew dependencies if platform.system() == "Darwin": # macOS From c7d652c1a74f6376dca76e1d77f6661745762d98 Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 14:12:19 +0100 Subject: [PATCH 5/7] Update Readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ce5d2aac..fdfdcb94 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,11 @@ print(dsf.__version__) The project requires `C++20` or greater, `cmake`, `tbb` `simdjson`, `spdlog` and `rapidcsv`. To install requirements on Ubuntu: ```shell -sudo apt install cmake libhwloc-dev +sudo apt install cmake libtbb-dev ``` To install requirements on macOS: ```shell -brew install cmake +brew install cmake tbb ``` Utilities are written in python. To install their dependencies: @@ -68,7 +68,7 @@ cmake --install build ## Installation (Python) If you want to use the library from Python, you can build the Python bindings using [pybind11](https://github.com/pybind/pybind11). Make sure you have Doxygen installed to generate the docstrings: ```shell -sudo apt install doxygen +sudo apt install doxygen libtbb-dev ``` Then, the installation is automatic via `pip`: From db0331d06e7ccf714afd01117728c6fd92b93b03 Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 14:12:52 +0100 Subject: [PATCH 6/7] Bump version --- src/dsf/dsf.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsf/dsf.hpp b/src/dsf/dsf.hpp index a6d056df..2a85bc44 100644 --- a/src/dsf/dsf.hpp +++ b/src/dsf/dsf.hpp @@ -6,7 +6,7 @@ static constexpr uint8_t DSF_VERSION_MAJOR = 4; static constexpr uint8_t DSF_VERSION_MINOR = 5; -static constexpr uint8_t DSF_VERSION_PATCH = 2; +static constexpr uint8_t DSF_VERSION_PATCH = 3; static auto const DSF_VERSION = std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH); From 8a40bd339a4c3dc5d13a6351b83150400a4816f3 Mon Sep 17 00:00:00 2001 From: grufoony Date: Mon, 22 Dec 2025 14:28:12 +0100 Subject: [PATCH 7/7] Missing tbb --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 213f63c8..7f34a151 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y cmake build-essential doxygen + sudo apt install -y cmake build-essential doxygen libtbb-dev python -m pip install --upgrade pip pip install pytest pytest-cov