From 9348a0933647ff5d21cfc576e6a57bc89ab70445 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 27 May 2025 17:58:33 -0400 Subject: [PATCH 1/5] optionally capture by reference in vector API --- CMakeLists.txt | 10 ++++ singularity-eos/eos/eos_base.hpp | 47 ++++++++++++------- .../packages/singularity-eos/package.py | 3 ++ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d09423743c..0af3d29a3c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,9 @@ cmake_dependent_option( SINGULARITY_USE_KOKKOSKERNELS "Use KokkosKernels for LA routines" ON "SINGULARITY_USE_KOKKOS;SINGULARITY_BUILD_CLOSURE" OFF) +option(SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE + "Capture by reference in vector API on CPUs when possible" ON) + # extra build options option(SINGULARITY_BUILD_PYTHON "Compile Python bindings" OFF) option(SINGULARITY_BUILD_EXAMPLES "Compile examples" OFF) @@ -595,6 +598,13 @@ if (SINGULARITY_STRICT_WARNINGS) target_compile_options(singularity-eos_Interface INTERFACE -Wall -Werror -Wno-unknown-pragmas) endif() +if (SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) + message(WARNING "Vector capture by reference is ON for CPU builds without Kokkos. " + "This is not technically valid Kokkos and may cause issues in edge cases. " + "To disable, set -DSINGULARITY_VECTOR_CAPTURE_BY_REFERENCE=OFF.") + target_compile_options(singularity-eos_Interface INTERFACE + SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) +endif() if(TARGET singularity-eos_Library) target_compile_options(singularity-eos_Library PRIVATE ${xlfix}) diff --git a/singularity-eos/eos/eos_base.hpp b/singularity-eos/eos/eos_base.hpp index 2eff349b450..15aa28cdd83 100644 --- a/singularity-eos/eos/eos_base.hpp +++ b/singularity-eos/eos/eos_base.hpp @@ -65,6 +65,20 @@ char *StrCat(char *destination, const char *source) { } } // namespace impl +#if defined(SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) && !defined(__CUDACC__) && \ + !defined(__HIPCC__) +#if !defined(PORTABILITY_STRATEGY_KOKKOS) && !defined(PORTABILITY_STRATEGY_NONE) +#error \ + "Unexpected portability strategy! Please set SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE to build in this mode." +#endif // unexpected portability strategy! +// JMM: I am hoping that by not adding any decorators here, we will +// catch problems with unexpected, non-Cuda, non-HIP accelerator +// architectures. +#define SG_VEC_LAMBDA [&] +#else +#define SG_VEC_LAMBDA PORTABLE_LAMBDA +#endif // SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE + // This Macro adds the `using` statements that allow for the base class // VECTOR functionality to overload the scalar implementations in the derived // classes. Do not add functions here that are not overloads of derived class features. @@ -274,7 +288,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { temperatures[i] = copy.TemperatureFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -309,7 +323,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { sies[i] = copy.InternalEnergyFromDensityTemperature(rhos[i], temperatures[i], lambdas[i]); }); @@ -344,7 +358,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { pressures[i] = copy.PressureFromDensityTemperature(rhos[i], temperatures[i], lambdas[i]); }); @@ -377,7 +391,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { pressures[i] = copy.PressureFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -408,7 +422,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { sies[i] = copy.MinInternalEnergyFromDensity(rhos[i], lambdas[i]); }); } @@ -438,7 +452,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { entropies[i] = copy.EntropyFromDensityTemperature(rhos[i], temperatures[i], lambdas[i]); }); @@ -471,7 +485,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { entropies[i] = copy.EntropyFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -503,7 +517,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { cvs[i] = copy.SpecificHeatFromDensityTemperature(rhos[i], temperatures[i], lambdas[i]); }); @@ -538,7 +552,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { cvs[i] = copy.SpecificHeatFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -571,7 +585,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { bmods[i] = copy.BulkModulusFromDensityTemperature(rhos[i], temperatures[i], lambdas[i]); }); @@ -606,7 +620,7 @@ class EosBase { static auto const cname = name.c_str(); const CRTP © = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { bmods[i] = copy.BulkModulusFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -638,7 +652,7 @@ class EosBase { static auto const cname = name.c_str(); CRTP copy = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { gm1s[i] = copy.GruneisenParamFromDensityTemperature(rhos[i], temperatures[i], lambdas[i]); }); @@ -673,7 +687,7 @@ class EosBase { static auto const cname = name.c_str(); CRTP copy = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { gm1s[i] = copy.GruneisenParamFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -707,7 +721,7 @@ class EosBase { static auto const cname = name.c_str(); CRTP copy = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { Gs[i] = copy.GibbsFreeEnergyFromDensityTemperature(rhos[i], Ts[i], lambdas[i]); }); } @@ -739,7 +753,7 @@ class EosBase { static auto const cname = name.c_str(); CRTP copy = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { Gs[i] = copy.GibbsFreeEnergyFromDensityInternalEnergy(rhos[i], sies[i], lambdas[i]); }); @@ -774,7 +788,7 @@ class EosBase { static auto const cname = name.c_str(); CRTP copy = *(static_cast(this)); portableFor( - cname, 0, num, PORTABLE_LAMBDA(const int i) { + cname, 0, num, SG_VEC_LAMBDA(const int i) { copy.FillEos(rhos[i], temps[i], energies[i], presses[i], cvs[i], bmods[i], output, lambdas[i]); }); @@ -1032,5 +1046,6 @@ class EosBase { } // namespace eos_base } // namespace singularity +#undef SG_VEC_LAMBDA #undef SG_MEMBER_FUNC_NAME #endif diff --git a/spack-repo/packages/singularity-eos/package.py b/spack-repo/packages/singularity-eos/package.py index ffc47b8d266..f6fd4cbc724 100644 --- a/spack-repo/packages/singularity-eos/package.py +++ b/spack-repo/packages/singularity-eos/package.py @@ -87,6 +87,8 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): variant("closure", default=True, description="Build closure module") variant("shared", default=False, description="Build shared libs") variant("vandv", default=True, description="Enable V&V EOSs in default Singularity::Variant") + variant("vecref", default=True, + description="When possible capture by reference in vector API. Performance optimization.") plugins = {} @@ -210,6 +212,7 @@ def cmake_args(self): self.define_from_variant("SINGULARITY_USE_SPINER_WITH_HDF5", "hdf5"), self.define_from_variant("BUILD_SHARED_LIBS", "shared"), self.define_from_variant("SINGULARITY_USE_V_AND_V_EOS", "vandv"), + self.define_from_variant("SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE", "vecref"), self.define("SINGULARITY_BUILD_TESTS", self.run_tests), self.define( "SINGULARITY_BUILD_SESAME2SPINER", From acb95a72728f1b75f408b11d7cf39b1cdd1030cd Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 28 May 2025 11:49:35 -0400 Subject: [PATCH 2/5] add to at least one part of the build matrix --- .github/workflows/tests_clang.yml | 1 + CMakeLists.txt | 2 +- doc/sphinx/src/building.rst | 49 ++++++++++--------- .../packages/singularity-eos/package.py | 2 +- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests_clang.yml b/.github/workflows/tests_clang.yml index 1b6c3ef47e6..5b2480a6dc6 100644 --- a/.github/workflows/tests_clang.yml +++ b/.github/workflows/tests_clang.yml @@ -44,6 +44,7 @@ jobs: -DSINGULARITY_TEST_HELMHOLTZ=ON \ -DSINGULARITY_FORCE_SUBMODULE_MODE=ON \ -DSINGULARITY_USE_V_AND_V_EOS=OFF \ + -DSINGULARITY_VECTOR_CAPTURE_BY_REFERENCE=ON \ -DSINGULARITY_USE_KOKKOS=OFF \ -DSINGULARITY_PLUGINS=$(pwd)/../example/plugin \ -DCMAKE_LINKER=ld.gold \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0af3d29a3c3..e30b9489fb9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,7 +70,7 @@ cmake_dependent_option( "SINGULARITY_USE_KOKKOS;SINGULARITY_BUILD_CLOSURE" OFF) option(SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE - "Capture by reference in vector API on CPUs when possible" ON) + "Capture by reference in vector API on CPUs when possible" OFF) # extra build options option(SINGULARITY_BUILD_PYTHON "Compile Python bindings" OFF) diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst index 29a7b2709d8..0f8c6603bf8 100644 --- a/doc/sphinx/src/building.rst +++ b/doc/sphinx/src/building.rst @@ -98,30 +98,31 @@ sections detailing those build modes. The main CMake options to configure building are in the following table: -====================================== ======= =========================================== - Option Default Comment -====================================== ======= =========================================== - ``SINGULARITY_USE_SPINER`` ON Enables EOS objects that use ``spiner``. - ``SINGULARITY_USE_FORTRAN`` ON Enable Fortran API for equation of state. - ``SINGULARITY_USE_KOKKOS`` OFF Uses Kokkos as the portability backend. Currently only Kokkos is supported for GPUs. - ``SINGULARITY_USE_EOSPAC`` OFF Link against EOSPAC. Needed for sesame2spiner and some tests. - ``SINGULARITY_EOSPAC_ENABLE_SHMEM`` OFF Enable shared memory support in EOSPAC backend. - ``SINGULARITY_BUILD_CLOSURE`` OFF Build the mixed cell closure models - ``SINGULARITY_BUILD_TESTS`` OFF Build test infrastructure. - ``SINGULARITY_BUILD_PYTHON`` OFF Build Python bindings. - ``SINGULARITY_BUILD_EXAMPLES`` OFF Build examples of ``singularity-eos`` in use. - ``SINGULARITY_INVERT_AT_SETUP`` OFF For tests, pre-invert eospac tables. - ``SINGULARITY_BETTER_DEBUG_FLAGS`` ON Enables nicer GPU debug flags. May interfere with in-tree builds as a submodule. - ``SINGULARITY_HIDE_MORE_WARNINGS`` OFF Makes warnings less verbose. May interfere with in-tree builds as a submodule. - ``SINGULARITY_FORCE_SUBMODULE_MODE`` OFF Force build in _submodule_ mode. - ``SINGULARITY_USE_TRUE_LOG_GRIDDING`` OFF Use grids that conform to logarithmic spacing. - ``SINGULARITY_USE_SINGLE_LOGS`` OFF Use single precision logarithms (may degrade accuracy). - ``SINGULARITY_NQT_ORDER_1`` OFF For fast logs, use the less accurate but faster 1st-order version. - ``SINGULARITY_NQT_PORTABLE`` OFF For fast logs, use the slower but endianness-independent implementation. - ``SINGULARITY_STRICT_WARNINGS`` OFF For testing. Adds -Wall and -Werror to builds. - ``SINGULARITY_USE_V_AND_V_EOS`` OFF Enables several additional EOS models and adds them to the default variant - ``SINGULARITY_USE_STELLAR_COLLAPSE`` OFF Adds the Stellar Collapse EOS to the default variant -====================================== ======= =========================================== +============================================= ======= =========================================== + Option Default Comment +============================================= ======= =========================================== + ``SINGULARITY_USE_SPINER`` ON Enables EOS objects that use ``spiner``. + ``SINGULARITY_USE_FORTRAN`` ON Enable Fortran API for equation of state. + ``SINGULARITY_USE_KOKKOS`` OFF Uses Kokkos as the portability backend. Currently only Kokkos is supported for GPUs. + ``SINGULARITY_USE_EOSPAC`` OFF Link against EOSPAC. Needed for sesame2spiner and some tests. + ``SINGULARITY_EOSPAC_ENABLE_SHMEM`` OFF Enable shared memory support in EOSPAC backend. + ``SINGULARITY_BUILD_CLOSURE`` OFF Build the mixed cell closure models + ``SINGULARITY_BUILD_TESTS`` OFF Build test infrastructure. + ``SINGULARITY_BUILD_PYTHON`` OFF Build Python bindings. + ``SINGULARITY_BUILD_EXAMPLES`` OFF Build examples of ``singularity-eos`` in use. + ``SINGULARITY_INVERT_AT_SETUP`` OFF For tests, pre-invert eospac tables. + ``SINGULARITY_BETTER_DEBUG_FLAGS`` ON Enables nicer GPU debug flags. May interfere with in-tree builds as a submodule. + ``SINGULARITY_HIDE_MORE_WARNINGS`` OFF Makes warnings less verbose. May interfere with in-tree builds as a submodule. + ``SINGULARITY_FORCE_SUBMODULE_MODE`` OFF Force build in _submodule_ mode. + ``SINGULARITY_USE_TRUE_LOG_GRIDDING`` OFF Use grids that conform to logarithmic spacing. + ``SINGULARITY_USE_SINGLE_LOGS`` OFF Use single precision logarithms (may degrade accuracy). + ``SINGULARITY_NQT_ORDER_1`` OFF For fast logs, use the less accurate but faster 1st-order version. + ``SINGULARITY_NQT_PORTABLE`` OFF For fast logs, use the slower but endianness-independent implementation. + ``SINGULARITY_STRICT_WARNINGS`` OFF For testing. Adds -Wall and -Werror to builds. + ``SINGULARITY_USE_V_AND_V_EOS`` OFF Enables several additional EOS models and adds them to the default variant + ``SINGULARITY_USE_STELLAR_COLLAPSE`` OFF Adds the Stellar Collapse EOS to the default variant + ``SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE`` OFF A performance trick for CPU-only builds. May improve vector-API calls for small vector lengths. Enable at your own risk! +============================================= ======= =========================================== More options are available to modify only if certain other options or variables satisfy certain conditions (*dependent options*). *Dependent diff --git a/spack-repo/packages/singularity-eos/package.py b/spack-repo/packages/singularity-eos/package.py index f6fd4cbc724..c17291cc153 100644 --- a/spack-repo/packages/singularity-eos/package.py +++ b/spack-repo/packages/singularity-eos/package.py @@ -87,7 +87,7 @@ class SingularityEos(CMakePackage, CudaPackage, ROCmPackage): variant("closure", default=True, description="Build closure module") variant("shared", default=False, description="Build shared libs") variant("vandv", default=True, description="Enable V&V EOSs in default Singularity::Variant") - variant("vecref", default=True, + variant("vecref", default=False, description="When possible capture by reference in vector API. Performance optimization.") plugins = {} From dc3f3b5008dfadf28d138b6217edbda7b952015a Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 28 May 2025 12:23:37 -0400 Subject: [PATCH 3/5] fix the cmake. add the changelog --- CHANGELOG.md | 1 + CMakeLists.txt | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0b9693848..c080b3429f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR511]](https://github.com/lanl/singularity-eos/pull/511) Added ability to sometimes capture by reference in vector API ### Fixed (Repair bugs, etc) - [[PR505]](https://github.com/lanl/singularity-eos/pull/505) rename LogType::TRUE to LogType::DOUBLE diff --git a/CMakeLists.txt b/CMakeLists.txt index e30b9489fb9..bb92bafc6ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -364,6 +364,13 @@ endif() if (SINGULARITY_USE_EOSPAC AND SINGULARITY_EOSPAC_ENABLE_SHMEM) target_compile_definitions(singularity-eos_Interface INTERFACE SINGULARITY_EOSPAC_ENABLE_SHARED_MEMORY) endif() +if (SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) + message(WARNING "Vector capture by reference is ON for CPU builds without Kokkos. " + "This is not technically valid Kokkos and may cause issues in edge cases. " + "To disable, set -DSINGULARITY_VECTOR_CAPTURE_BY_REFERENCE=OFF.") + target_compile_definitions(singularity-eos_Interface INTERFACE + SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) +endif() # ------------------------------------------------------------------------------# # Handle dependencies @@ -598,13 +605,6 @@ if (SINGULARITY_STRICT_WARNINGS) target_compile_options(singularity-eos_Interface INTERFACE -Wall -Werror -Wno-unknown-pragmas) endif() -if (SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) - message(WARNING "Vector capture by reference is ON for CPU builds without Kokkos. " - "This is not technically valid Kokkos and may cause issues in edge cases. " - "To disable, set -DSINGULARITY_VECTOR_CAPTURE_BY_REFERENCE=OFF.") - target_compile_options(singularity-eos_Interface INTERFACE - SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) -endif() if(TARGET singularity-eos_Library) target_compile_options(singularity-eos_Library PRIVATE ${xlfix}) From a6575853ce4f567672314710a0e00a0611d5235a Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 28 May 2025 12:25:15 -0400 Subject: [PATCH 4/5] update warning language --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb92bafc6ff..8a3b1b2edaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -365,7 +365,7 @@ if (SINGULARITY_USE_EOSPAC AND SINGULARITY_EOSPAC_ENABLE_SHMEM) target_compile_definitions(singularity-eos_Interface INTERFACE SINGULARITY_EOSPAC_ENABLE_SHARED_MEMORY) endif() if (SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) - message(WARNING "Vector capture by reference is ON for CPU builds without Kokkos. " + message(WARNING "Vector capture by reference is ON for CPU builds. " "This is not technically valid Kokkos and may cause issues in edge cases. " "To disable, set -DSINGULARITY_VECTOR_CAPTURE_BY_REFERENCE=OFF.") target_compile_definitions(singularity-eos_Interface INTERFACE From ebae2f9985b68131bc500aebc73c4f5a292ee74c Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 28 May 2025 12:31:35 -0400 Subject: [PATCH 5/5] added some comments to describe what's happening here --- singularity-eos/eos/eos_base.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/singularity-eos/eos/eos_base.hpp b/singularity-eos/eos/eos_base.hpp index 15aa28cdd83..93b253587ee 100644 --- a/singularity-eos/eos/eos_base.hpp +++ b/singularity-eos/eos/eos_base.hpp @@ -65,11 +65,17 @@ char *StrCat(char *destination, const char *source) { } } // namespace impl +// JMM: This is a little trick to skip copying EOS's when not +// necessary. It changes the semantics of the lambda to capture by +// reference *only* on CPU-only builds. Should buy performance, but is +// a bit of a hack. I consider it dangerous because it changes the +// semantics/memnory model between CPU and GPU, which may be difficult +// to debug. It is thus off by defualt, but can be turned on. #if defined(SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE) && !defined(__CUDACC__) && \ !defined(__HIPCC__) #if !defined(PORTABILITY_STRATEGY_KOKKOS) && !defined(PORTABILITY_STRATEGY_NONE) #error \ - "Unexpected portability strategy! Please set SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE to build in this mode." + "Unexpected portability strategy! Please set SINGULARITY_VECTOR_CAPTURE_BY_REFERENCE=OFF to build in this mode." #endif // unexpected portability strategy! // JMM: I am hoping that by not adding any decorators here, we will // catch problems with unexpected, non-Cuda, non-HIP accelerator